diff --git a/AutoCoverTool/online/inference_one.py b/AutoCoverTool/online/inference_one.py index a73d87d..f60be54 100644 --- a/AutoCoverTool/online/inference_one.py +++ b/AutoCoverTool/online/inference_one.py @@ -1,689 +1,711 @@ """ 单个处理的逻辑 song_id: ---src.mp3 // 源数据，需要提前放进去 ---cache ---vocal.wav // 分离之后产生 ---acc.wav // 分离之后产生 ---vocal_32.wav // 分离之后产生 ---song_id_sp1.wav // 合成之后产生 ---song_id_sp2.wav // 合成之后产生 ---song_id_sp2_d.wav // 降噪之后生成 ---song_id_sp2_dv.wav // 降噪+拉伸之后产生 [占比太高的不产生] ---song_id_sp2_dve442.wav // 手动调整之后产生 ---song_id_sp2_dve442_replace.wav // 替换之后产生 ---song_id_sp2_dve442_replace_mix.wav // 人声+伴奏混合之后产生 ---song_id --acc.mp3 // 44k双声道320k --vocal.mp3 // 44k双声道320k --src.mp3 // 44k双声道320k --song_id_sp2_dv.mp3 // 44k单声道320k ---song_id_out // 对外输出 --src.mp3 // 原始音频 --song_id_sp2_dv_replace_mix.mp3 // 制作完成的音频 环境安装: conda create -n auto_song_cover python=3.9 # 安装demucs环境[进入到ref.music_remover 执行pip install -r requirements.txt] # 安装so_vits_svc环境[进入到ref.so_vits_svc 执行pip install -r requirements.txt] pip install librosa pip install scikit-maad pip install praat-parselmouth pip install matplotlib pip install torchvision pip install madmom pip install torchstat 环境设置: export PATH=$PATH:/data/gpu_env_common/env/bin/ffmpeg/bin export PYTHONPATH=$PWD:$PWD/ref/music_remover/demucs:$PWD/ref/so_vits_svc:$PWD/ref/split_dirty_frame """ import os import time import shutil import random import logging import librosa logging.basicConfig(filename='/tmp/inference.log', level=logging.INFO) gs_err_code_success = 0 gs_err_code_no_src_mp3 = 1 gs_err_code_separate = 2 gs_err_code_trans_32 = 3 gs_err_code_encode_err = 4 gs_err_code_replace_err = 5 gs_err_code_replace_trans_err = 6 gs_err_code_mix_err = 7 gs_err_code_mix_transcode_err = 8 gs_err_code_no_src_dir = 9 gs_err_code_volume_err = 10 gs_err_code_trans2_442 = 11 gs_err_code_reverb = 12 gs_err_code_no_good_choice = 13 gs_err_code_preprocess_vocal = 14 gs_err_code_replace_except_err = 15 gs_denoise_exe = "/opt/soft/bin/denoise_exe" gs_draw_volume_exe = "/opt/soft/bin/draw_volume" gs_simple_mixer_path = "/opt/soft/bin/simple_mixer" gs_rever_path = "/opt/soft/bin/dereverbrate" from ref.music_remover.separate_interface import SeparateInterface from ref.so_vits_svc.inference_main import * from ref.split_dirty_frame.script.process_one import ReplaceVocalFrame, construct_power_fragment class SongCoverInference: def __init__(self): self.work_dir = None self.cache_dir = None self.cid = None self.src_mp3 = None self.vocal_path = None self.vocal_32_path = None self.acc_path = None self.speakers = [ 10414574138721494, 10414574140317353, 1688849864840588, 3634463651, 5629499489839033, 5910973794723621, 6755399374234747, 8162774327817435, 8162774329368194, 1125899914308640, # 以下为男声,包括这个 12384898975368914, 12947848931397021, 3096224748076687, 3096224751151928, 5066549357604730, 5348024335101054, 6755399442719465, 7036874421386111 ] self.speakers2gender = { 10414574138721494: 2, 10414574140317353: 2, 1688849864840588: 2, 3634463651: 2, 5629499489839033: 2, 5910973794723621: 2, 6755399374234747: 2, 8162774327817435: 2, 8162774329368194: 2, 1125899914308640: 1, # 1是男 12384898975368914: 1, 12947848931397021: 1, 3096224748076687: 1, 3096224751151928: 1, 5066549357604730: 1, 5348024335101054: 1, 6755399442719465: 1, 7036874421386111: 1 } self.speakers_model_path = "data/train_users/{}/logs/32k/G_2000.pth" self.speakers_model_config = "data/train_users/{}/config/config.json" st = time.time() self.separate_inst = None logging.info("post process ... ReplaceVocalFrame init sp={}".format(time.time() - st)) self.replace_vocal_frame_inst = None logging.info("SongCoverInference init sp={}".format(time.time() - st)) def separate(self, cid, src_mp3, vocal_path, acc_path): """ 人声伴奏分离 :param cid: :param src_mp3: :param vocal_path: :param acc_path: :return: """ st = time.time() if self.separate_inst is None: self.separate_inst = SeparateInterface() if not self.separate_inst.process(cid, src_mp3, vocal_path, acc_path): return gs_err_code_separate if not os.path.exists(vocal_path) or not os.path.exists(acc_path): return gs_err_code_separate # 转码出一个32k单声道的数据 cmd = "ffmpeg -i {} -ar 32000 -ac 1 -y {} -loglevel fatal".format(vocal_path, self.vocal_32_path) os.system(cmd) if not os.path.exists(self.vocal_32_path): return gs_err_code_trans_32 print("separate:cid={}|sp={}".format(cid, time.time() - st)) return gs_err_code_success def get_start_ms(self, vocal_path): """ 给定原始音频，找一段连续10s的音频 :param vocal_path: :return: """ audio, sr = librosa.load(vocal_path, sr=16000) audio = librosa.util.normalize(audio) # 帧长100ms,帧移10ms,计算能量 power_arr = [] for i in range(0, len(audio) - 1600, 160): power_arr.append(np.sum(np.abs(audio[i:i + 160])) / 160) # 将能量小于等于10的部分做成段 power_arr = construct_power_fragment(power_arr) fragments = [] last_pos = 0 for idx, line in enumerate(power_arr): start = round(float(line[0]) * 0.01, 3) duration = round(float(line[1]) * 0.01, 3) fragments.append([last_pos, start - last_pos]) last_pos = start + duration if last_pos < len(audio) / sr: fragments.append([last_pos, len(audio) / sr - last_pos]) # 合并数据，两者间隔在50ms以内的合并起来 idx = 0 while idx < len(fragments) - 1: if fragments[idx + 1][0] - (fragments[idx][0] + fragments[idx][1]) < 0.05: fragments[idx][1] = fragments[idx + 1][0] + fragments[idx + 1][1] - fragments[idx][0] del fragments[idx + 1] idx -= 1 idx += 1 # out_file = vocal_path + "_power.csv" # with open(out_file, "w") as f: # f.write("Name\tStart\tDuration\tTime Format\tType\n") # for fragment in fragments: # start = round(float(fragment[0]), 3) # duration = round(float(fragment[1]), 3) # strr = "{}\t{}\t{}\t{}\n".format("11", start, duration, "decimal\tCue\t") # f.write(strr) # 筛选出开始的位置 # 1. 连续时长大于10s，当前段长度大于3s # 2. 不可用 # 从0到fragments[idx], 包含idx其中人声段的总和 tot_vocal_duration = [fragments[0][1]] for i in range(1, len(fragments)): tot_vocal_duration.append(tot_vocal_duration[i - 1] + fragments[i][1]) # 计算出任意两段之间非人声占比 for i in range(0, len(fragments)): if fragments[i][1] >= 3: now_tot = 0 if i > 0: now_tot = tot_vocal_duration[i - 1] for j in range(i + 1, len(fragments)): cur_rate = tot_vocal_duration[j] - now_tot cur_rate = cur_rate / (fragments[j][1] + fragments[j][0] - fragments[i][0]) if cur_rate > 0.1: return fragments[i][0] return -1 def inference_speaker(self): """ 推理生成合成后的音频 随机取5个干声,选择占比最小的，并且要求占比小于0.3 :return: """ st = time.time() out_speakers = random.sample(self.speakers, 15) out_songs_dict = {} for speaker in out_speakers: model_path = self.speakers_model_path.format(speaker) config_path = self.speakers_model_config.format(speaker) song_path = os.path.join(self.cache_dir, "{}_{}.wav".format(self.cid, speaker)) try: inf(model_path, config_path, self.vocal_32_path, song_path, "prod") except Exception as ex: logging.info("cid={}, inference_speaker err={}".format(self.cid, ex)) continue if os.path.exists(song_path): if self.replace_vocal_frame_inst is None: self.replace_vocal_frame_inst = ReplaceVocalFrame( "data/models/split_dirty_frame_v5_3_epoch3_852.pth") rate = self.replace_vocal_frame_inst.get_rate(song_path) if rate < 0.3: out_songs_dict[song_path] = rate # 从内部选择占比最低的 out_songs = [] if len(out_songs_dict.keys()) > 0: st_sec = self.get_start_ms(self.vocal_path) song_msg = sorted(out_songs_dict.items(), key=lambda kv: kv[1])[0] out_songs = [song_msg[0]] logging.info("GetRate:cid={},song={},rate={},st_tm={}".format(self.cid, song_msg[0], round(song_msg[1], 2), round(st_sec, 3))) print("GetRate:cid={},song={},rate={},st_tm={}".format(self.cid, song_msg[0], round(song_msg[1], 2), round(st_sec, 3))) # logging.info("inference_speaker len = {} finish sp = {}".format(len(out_songs), time.time() - st)) print("inference_speaker len = {} finish sp = {}".format(len(out_songs), time.time() - st)) return out_songs def get_new_vocal_rate(self, songs): """ 获取人声的比率 :param songs: :return: """ st = time.time() need_to_process_song = [] for song in songs: if self.replace_vocal_frame_inst is None: self.replace_vocal_frame_inst = ReplaceVocalFrame("data/models/split_dirty_frame_v5_3_epoch3_852.pth") rate = self.replace_vocal_frame_inst.get_rate(song) logging.info("{} {} replace_rate={}".format(self.cid, song, rate)) if rate < 1.0: need_to_process_song.append(song) logging.info( "get_new_vocal_rate belen = {} len = {} finish sp = {}".format(len(songs), len(need_to_process_song), time.time() - st)) return need_to_process_song def preprocess_vocal(self, songs, vocal_path): """ 1. 降噪 2. 拉伸 :param songs: :param vocal_path: 参考的音频信号 :return: """ st = time.time() dv_out_list = [] for song in songs: denoise_path = str(song).replace(".wav", "_d.wav") cmd = "{} {} {}".format(gs_denoise_exe, song, denoise_path) os.system(cmd) if not os.path.exists(denoise_path): print("{} {} ERROR denoise".format(self.cid, song)) continue # 拉伸 volume_path = str(song).replace(".wav", "_dv.wav") cmd = "{} {} {} {}".format(gs_draw_volume_exe, denoise_path, vocal_path, volume_path) os.system(cmd) if not os.path.exists(volume_path): print("{} {} ERROR denoise".format(self.cid, volume_path)) continue dv_out_list.append(volume_path) print( "preprocess_vocal belen = {} len = {} finish sp = {}".format(len(songs), len(dv_out_list), time.time() - st)) return dv_out_list def output(self, dv_out_list): """ 对外输出数据 :param dv_out_list: :return: """ st = time.time() out_dir = os.path.join(self.work_dir, self.cid) if os.path.exists(out_dir): shutil.rmtree(out_dir) os.makedirs(out_dir) # 拷贝数据 dst_mp3_path = os.path.join(out_dir, "src_mp3") dst_acc_path = os.path.join(out_dir, "acc.mp3") dst_vocal_path = os.path.join(out_dir, "vocal.mp3") shutil.copyfile(self.src_mp3, dst_mp3_path) cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(self.acc_path, dst_acc_path) os.system(cmd) if not os.path.exists(dst_acc_path): return gs_err_code_encode_err cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(self.vocal_path, dst_vocal_path) os.system(cmd) if not os.path.exists(dst_vocal_path): return gs_err_code_encode_err # 将所有数据放到out_dir中，用于给人工标注 for dv_wav in dv_out_list: dv_wav_name = str(dv_wav).split("/")[-1].replace(".wav", "_441.mp3") dst_dv_path = os.path.join(out_dir, dv_wav_name) cmd = "ffmpeg -i {} -ar 44100 -ac 1 -ab 320k -y {} -loglevel fatal".format(dv_wav, dst_dv_path) os.system(cmd) if not os.path.exists(dst_dv_path): print("{} encode err!".format(cmd)) continue logging.info( "preprocess_vocal output sp = {}".format(time.time() - st)) def process_one(self, cid, work_dir, enable_output=False): logging.info("\nstart:cid={},work_dir={}----------------------->>>>>>>>".format(cid, work_dir)) self.cid = cid self.work_dir = work_dir # 所有不对外交付的，全部放到这里 self.cache_dir = os.path.join(work_dir, "cache") if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) os.makedirs(self.cache_dir) self.src_mp3 = os.path.join(self.work_dir, "src.mp3") if not os.path.exists(self.src_mp3): return gs_err_code_no_src_mp3 self.vocal_path = os.path.join(self.cache_dir, "vocal.wav") self.vocal_32_path = os.path.join(self.cache_dir, "vocal_32.wav") self.acc_path = os.path.join(self.cache_dir, "acc.wav") if not os.path.exists(self.vocal_32_path): logging.info("start separate ... {} {} {}".format(self.src_mp3, self.vocal_path, self.acc_path)) err = self.separate(cid, self.src_mp3, self.vocal_path, self.acc_path) if err != gs_err_code_success: return err, None, None logging.info("start inference_speaker ...") out_songs = self.inference_speaker() dv_out_list = self.preprocess_vocal(out_songs, self.vocal_path) if len(dv_out_list) == 0: return gs_err_code_no_good_choice, None, None mix_mp3_path = None gender = -1 if enable_output: self.output(dv_out_list) else: # 默认全部处理一遍 for dv_out_path in dv_out_list: src_path = dv_out_path.replace("_dv.wav", ".wav") err, mix_mp3_path = self.after_process(self.cid, self.work_dir, src_path, dv_out_path, self.vocal_path, self.acc_path, True, False) if err != gs_err_code_success: logging.info("after_process err {}".format(err)) # 取出性别属性 if err == gs_err_code_success and mix_mp3_path is not None: gender = self.speakers2gender[int(str(os.path.basename(mix_mp3_path)).split("_")[1])] logging.info("finish:cid={},work_dir={}----------------------->>>>>>>>".format(cid, work_dir)) return gs_err_code_success, mix_mp3_path, gender def reverb_by_vocal(self, file): st = time.time() file_442 = file.replace(".wav", "_442.wav") if not os.path.exists(file_442): cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {}".format(file, file_442) os.system(cmd) if not os.path.exists(file_442): return gs_err_code_trans2_442, None file_dst = file.replace(".wav", "_442_dr.wav") cmd = "{} {} {} {}".format(gs_rever_path, self.vocal_path, file_442, file_dst) os.system(cmd) if not os.path.exists(file_dst): return gs_err_code_reverb, None print("cid = {}, reverb_by_vocal sp={}".format(self.cid, time.time() - st)) return gs_err_code_success, file_dst def after_process(self, cid, work_dir, in_file, effect_file, vocal_file, acc_file, need_draw=True, need_reverb=True): """ 后处理逻辑 将处理好的音频进行替换，然后和伴奏进行混合，最后进行编码 :return: """ if need_reverb: # 抓取混响 err, effect_file = self.reverb_by_vocal(in_file) if err != gs_err_code_success: return err, None if need_draw: # 增加一个拉伸的步骤 volume_path = str(effect_file).replace(".wav", "_dv.wav") cmd = "{} {} {} {}".format(gs_draw_volume_exe, effect_file, vocal_file, volume_path) print(cmd) os.system(cmd) if not os.path.exists(volume_path): print("{} {} ERROR draw volume".format(self.cid, volume_path)) return gs_err_code_volume_err, None effect_file = volume_path st = time.time() self.cid = cid self.work_dir = work_dir self.src_mp3 = os.path.join(self.work_dir, "src.mp3") if not os.path.exists(self.work_dir): return gs_err_code_no_src_dir self.replace_vocal_frame_inst.process(in_file, effect_file, vocal_file) dst_path = effect_file + "_replace.wav" if not os.path.exists(dst_path): return gs_err_code_replace_err, None print("replace_vocal_frame_inst sp = {}".format(time.time() - st)) # 转码 dst_path_442 = dst_path.replace("_replace.wav", "_replace442.wav") cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} -loglevel fatal".format(dst_path, dst_path_442) os.system(cmd) if not os.path.exists(dst_path_442): return gs_err_code_replace_trans_err, None # 合并转码后再做一次拉伸，保证响度 volume_path = str(dst_path_442).replace(".wav", "_dv.wav") cmd = "{} {} {} {}".format(gs_draw_volume_exe, dst_path_442, vocal_file, volume_path) print(cmd) os.system(cmd) if not os.path.exists(volume_path): print("{} {} ERROR draw volume".format(self.cid, volume_path)) return gs_err_code_volume_err, None dst_path_442 = volume_path # 混合 mix_path = dst_path_442.replace("_replace442.wav", "_replace442_mix.wav") cmd = "{} {} {} {}".format(gs_simple_mixer_path, dst_path_442, acc_file, mix_path) print("{}".format(cmd)) os.system(cmd) if not os.path.exists(mix_path): return gs_err_code_mix_err, None # 编码为mp3 output_dir = os.path.join(self.work_dir, self.cid + "_out") if not os.path.exists(output_dir): os.makedirs(output_dir) name = str(mix_path).replace("_replace442_mix.wav", "_replace442_mix.mp3").split("/")[-1] mix_path_mp3 = os.path.join(output_dir, name) cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(mix_path, mix_path_mp3) os.system(cmd) if not os.path.exists(mix_path_mp3): return gs_err_code_mix_transcode_err, None # 拷贝src到output_dir # shutil.copyfile(self.src_mp3, os.path.join(output_dir, "src.mp3")) # logging.info("after_process sp = {}".format(time.time() - st)) return gs_err_code_success, mix_path_mp3 ####################################新对外接口############################################################ def prepare_env(self, cid, work_dir, create_dir=False): self.cid = cid self.work_dir = work_dir # 所有不对外交付的，全部放到这里 self.cache_dir = os.path.join(work_dir, "cache") if create_dir: if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) os.makedirs(self.cache_dir) self.src_mp3 = os.path.join(self.work_dir, "src.mp3") if not os.path.exists(self.src_mp3): return gs_err_code_no_src_mp3 self.vocal_path = os.path.join(self.cache_dir, "vocal.wav") self.vocal_32_path = os.path.join(self.cache_dir, "vocal_32.wav") self.acc_path = os.path.join(self.cache_dir, "acc.wav") return gs_err_code_success def generate_svc_file(self, cid, work_dir): """ :param cid: :param work_dir: :return:err_code, 生成出的svc的文件名称 """ err = self.prepare_env(cid, work_dir, create_dir=True) if err != gs_err_code_success: return err, None # 音源分离 if not os.path.exists(self.vocal_32_path): st = time.time() err = self.separate(cid, self.src_mp3, self.vocal_path, self.acc_path) logging.info("cid={},separate,sp={}".format(self.cid, time.time() - st)) if err != gs_err_code_success: return err, None # 生成svc,只保留一个最佳的 st = time.time() out_songs = self.inference_speaker() if len(out_songs) == 0: return gs_err_code_no_good_choice, None logging.info("cid={},inference_speaker,{},sp={}".format(self.cid, out_songs[0], time.time() - st)) return gs_err_code_success, out_songs[0] def effect(self, cid, work_dir, svc_file): st = time.time() err = self.prepare_env(cid, work_dir) if err != gs_err_code_success: return err, None logging.info("cid={},effect_and_mix,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 预处理人声 dv_out_list = self.preprocess_vocal([svc_file], self.vocal_path) if len(dv_out_list) == 0: return gs_err_code_preprocess_vocal, None svc_file = dv_out_list[0] # 做音效 st = time.time() err, effect_file = self.reverb_by_vocal(svc_file) if err != gs_err_code_success: return err, None logging.info("cid={},reverb_by_vocal,{},sp={}".format(self.cid, svc_file, time.time() - st)) return err, effect_file def mix(self, cid, work_dir, svc_file, effect_file): """ 做音效以及合并 :param cid: :param work_dir: :param svc_file: :param effect_file: :return: err_code, 完成的mp3文件 """ st = time.time() err = self.prepare_env(cid, work_dir) if err != gs_err_code_success: return err, None logging.info("cid={},effect_and_mix,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 拉伸 st = time.time() volume_path = str(effect_file).replace(".wav", "_dv.wav") cmd = "{} {} {} {}".format(gs_draw_volume_exe, effect_file, self.vocal_path, volume_path) os.system(cmd) if not os.path.exists(volume_path): print("{} {} ERROR draw volume".format(self.cid, volume_path)) return gs_err_code_volume_err, None effect_file = volume_path logging.info("cid={},draw_volume,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 替换 st = time.time() try: if self.replace_vocal_frame_inst is None: self.replace_vocal_frame_inst = ReplaceVocalFrame("data/models/split_dirty_frame_v5_3_epoch3_852.pth") self.replace_vocal_frame_inst.process(svc_file, effect_file, self.vocal_path) except Exception as ex: logging.info("{},replace_vocal_frame_inst, {}", self.cid, ex) return gs_err_code_replace_except_err, None dst_path = effect_file + "_replace.wav" if not os.path.exists(dst_path): return gs_err_code_replace_err, None logging.info("cid={},replace_vocal_frame_inst,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 转码 st = time.time() dst_path_442 = dst_path.replace("_replace.wav", "_replace442.wav") cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} -loglevel fatal".format(dst_path, dst_path_442) os.system(cmd) if not os.path.exists(dst_path_442): return gs_err_code_replace_trans_err, None logging.info("cid={},transcode,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 合并转码后再做一次拉伸，保证响度 st = time.time() volume_path = str(dst_path_442).replace("_replace442.wav", "_replace442_dv.wav") cmd = "{} {} {} {}".format(gs_draw_volume_exe, dst_path_442, self.vocal_path, volume_path) os.system(cmd) if not os.path.exists(volume_path): print("{} {} ERROR draw volume".format(self.cid, volume_path)) return gs_err_code_volume_err, None dst_path_442 = volume_path logging.info("cid={},draw_volume2,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 混合 st = time.time() mix_path = dst_path_442.replace("_replace442_dv.wav", "_replace442_dv_mix.wav") cmd = "{} {} {} {}".format(gs_simple_mixer_path, dst_path_442, self.acc_path, mix_path) os.system(cmd) if not os.path.exists(mix_path): return gs_err_code_mix_err, None logging.info("cid={},mixer,{},sp={}".format(self.cid, svc_file, time.time() - st)) # 编码为mp3 st = time.time() output_dir = os.path.join(self.work_dir, self.cid + "_out") if not os.path.exists(output_dir): os.makedirs(output_dir) name = str(mix_path).replace("_replace442_dv_mix.wav", "_replace442_dv_mix.mp3").split("/")[-1] mix_path_mp3 = os.path.join(output_dir, name) cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(mix_path, mix_path_mp3) print(cmd) os.system(cmd) if not os.path.exists(mix_path_mp3): return gs_err_code_mix_transcode_err, None logging.info("cid={},encode,{},sp={}".format(self.cid, svc_file, time.time() - st)) return gs_err_code_success, mix_path_mp3 def get_gender(self, svc_file): return self.speakers2gender[int(os.path.basename(svc_file.replace(".wav", "")).split("_")[1])] def process_one_logic(self, cid, work_dir): """ 搞成两部分: 1. 分离数据+5次推理，获取最佳结果,并保存 2. 利用最佳结果做音效以及合并 :return: """ err, svc_file = self.generate_svc_file(cid, work_dir) gender = -1 if err != gs_err_code_success: return err, svc_file, gender, gender = self.get_gender(svc_file) err, effect_file = self.effect(cid, work_dir, svc_file) if err != gs_err_code_success: return err, svc_file, gender err, mix_mp3_path = self.mix(cid, work_dir, svc_file, effect_file) return err, mix_mp3_path, gender def test(): arr = [ # "611752105020343687", # "611752105023532439", # "611752105030419688", # "611752105030485748", # "611752105030485685", "dzq", ] base_dir = "/data/rsync/jianli.yang/AutoCoverTool/data/test" s_inst = SongCoverInference() for cid in arr: st = time.time() # err, mix_mp3, gender = s_inst.process_one(cid, os.path.join(base_dir, cid), False) err, mix_mp3, gender = s_inst.process_one_logic(cid, os.path.join(base_dir, cid)) print(mix_mp3, gender) print("cid={} RealFinish err={} sp={}".format(cid, err, time.time() - st)) +def test_gene_svc(): + base_dir = "/data/rsync/jianli.yang/AutoCoverTool/data/test" + cid = "clean" + work_dir = os.path.join(base_dir, cid) + st = time.time() + speaker = "1125899914308640" + speakers_model_path = "data/train_users/{}/logs/32k/G_2000.pth" + speakers_model_config = "data/train_users/{}/config/config.json" + model_path = speakers_model_path.format(speaker) + config_path = speakers_model_config.format(speaker) + + # 缓存目录: + cache_dir = os.path.join(work_dir, "cache") + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + os.makedirs(cache_dir) + song_path = os.path.join(cache_dir, "{}_{}.wav".format(cid, speaker)) + vocal_path = os.path.join(work_dir, "vocal_32.wav") + inf(model_path, config_path, vocal_path, song_path, "prod") + print("finish....") + + if __name__ == '__main__': test() diff --git a/AutoCoverTool/online/tone_shift_one.py b/AutoCoverTool/online/tone_shift_one.py index 2abe3e0..64f6236 100644 --- a/AutoCoverTool/online/tone_shift_one.py +++ b/AutoCoverTool/online/tone_shift_one.py @@ -1,173 +1,216 @@ """ 变调的方式做处理 1. 下载 2. 分离 3. 针对于人声变调+2，伴奏+1 4. 合成 """ import os +import json import shutil +import librosa import logging +import numpy as np from ref.music_remover.separate_interface import SeparateInterface from online.inference_worker import upload_file2cos, gs_state_use, gs_state_finish, gs_state_default from online.common import * logging.basicConfig(filename='/tmp/tone_shift_one.log', level=logging.INFO) gs_tone_shift_exe = "/opt/soft/bin/tone_shift_exe" gs_simple_mixer_path = "/opt/soft/bin/simple_mixer" gs_err_code_success = 0 gs_err_code_tone_shift = 1 gs_err_code_mix = 2 gs_err_code_transcode = 3 gs_err_code_upload = 4 gs_err_code_download = 5 gs_err_code_trans_to_mp3 = 6 gs_err_code_separate = 7 +gs_err_code_duration_too_long = 8 +gs_err_code_duration_no_vocal = 9 + + +def exec_cmd(cmd): + r = os.popen(cmd) + text = r.read() + r.close() + return text + + +def get_d(audio_path): + cmd = "ffprobe -v quiet -print_format json -show_format -show_streams {}".format(audio_path) + data = exec_cmd(cmd) + data = json.loads(data) + # 返回秒 + return float(data["format"]["duration"]) + + +def get_mean_power(audio_path): + sr = 44100 + audio, sr = librosa.load(audio_path, sr=sr, mono=True) + mm = np.mean(np.abs(audio)) + return mm class ToneShift: def __init__(self): self.separate_inst = SeparateInterface() def update_state(self, song_id, state): sql = "update svc_queue_table set state={},update_time={} where song_id = {}". \ format(state, int(time.time()), song_id) banned_user_map['db'] = "av_db" update_db(sql, banned_user_map) def get_one_data(self): sql = "select song_id, url from svc_queue_table where state = 0 and song_src=3 order by create_time desc limit 1" banned_user_map["db"] = "av_db" data = get_data_by_mysql(sql, banned_user_map) if len(data) == 0: return None, None song_id, song_url = data[0] if song_id != "": self.update_state(song_id, gs_state_use) return str(song_id), song_url def pre_process(self, work_dir, song_url): """ 创建文件夹，下载数据 :return: """ ext = str(song_url).split(".")[-1] dst_file = "{}/src_origin.{}".format(work_dir, ext) cmd = "wget {} -O {}".format(song_url, dst_file) print(cmd) os.system(cmd) if not os.path.exists(dst_file): return gs_err_code_download + + duration = get_d(dst_file) + print("Duration:", dst_file, duration) + if duration > 20 * 60: + return gs_err_code_duration_too_long + dst_mp3_file = "{}/src.mp3".format(work_dir) cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} ".format(dst_file, dst_mp3_file) os.system(cmd) if not os.path.exists(dst_mp3_file): return gs_err_code_trans_to_mp3 return gs_err_code_success def tone_shift_one(self, in_file, dst_file, pitch): cmd = "{} {} {} {}".format(gs_tone_shift_exe, in_file, dst_file, pitch) os.system(cmd) return os.path.exists(dst_file) def mix(self, cid, vocal_path, acc_path, tp): if tp == 1: vocal_pitch = 2 acc_pitch = 0 else: vocal_pitch = -2 acc_pitch = 0 vocal_path_2 = vocal_path.replace(".wav", "_{}.wav".format(vocal_pitch)) acc_path_2 = acc_path.replace(".wav", "_{}.wav".format(acc_pitch)) err = self.tone_shift_one(vocal_path, vocal_path_2, vocal_pitch) if not err: return gs_err_code_tone_shift, None err = self.tone_shift_one(acc_path, acc_path_2, acc_pitch) if not err: return gs_err_code_tone_shift, None base_dir = os.path.dirname(vocal_path) mix_path = "{}/mix_{}_{}.wav".format(base_dir, vocal_pitch, acc_pitch) cmd = "{} {} {} {}".format(gs_simple_mixer_path, vocal_path_2, acc_path_2, mix_path) print("exec_cmd={}".format(cmd)) os.system(cmd) if not os.path.exists(mix_path): return gs_err_code_mix, None # 转码 mix_path_mp3 = mix_path.replace(".wav", ".mp3") cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(mix_path, mix_path_mp3) os.system(cmd) if not os.path.exists(mix_path_mp3): return gs_err_code_transcode, None # 上传到cos mix_name = os.path.basename(mix_path_mp3) key = "av_res/svc_res_tone_shift/{}/{}".format(str(cid), mix_name) if not upload_file2cos(key, mix_path_mp3): return gs_err_code_upload, None return gs_err_code_success, key def process_one(self, cid, work_dir): """ :param cid: :param work_dir: :return: """ src_mp3 = os.path.join(work_dir, "src.mp3") vocal_path = os.path.join(work_dir, "vocal.wav") acc_path = os.path.join(work_dir, "acc.wav") if not self.separate_inst.process(cid, src_mp3, vocal_path, acc_path): return gs_err_code_separate, [] if not os.path.exists(vocal_path) or not os.path.exists(acc_path): return gs_err_code_separate, [] + # 当人声的平均能量小于一定值时，则认为无人声(0.01是经验值判定，样本分析来看) + # 无人声的样本[0.0056, 0.0003], 有人声的样本(目前最小)[0.046, 0.049] + print("power:{},{}".format(cid, get_mean_power(vocal_path))) + if get_mean_power(vocal_path) < 0.02: + return gs_err_code_duration_no_vocal, [] err, type1_mix_mp3 = self.mix(cid, vocal_path, acc_path, 1) if err != gs_err_code_success: return err, [] err, type2_mix_mp3 = self.mix(cid, vocal_path, acc_path, 2) if err != gs_err_code_success: return err, [] return gs_err_code_success, [type1_mix_mp3, type2_mix_mp3] def process_worker(self): logging.info("start process_worker .....") base_dir = "/tmp/tone_shift_one" if not os.path.exists(base_dir): os.makedirs(base_dir) while True: worker_st = time.time() cid, song_url = self.get_one_data() + if cid is None: + time.sleep(5) + logging.info("get one data is None ...") + continue + work_dir = os.path.join(base_dir, str(cid)) if os.path.exists(work_dir): shutil.rmtree(work_dir) os.makedirs(work_dir) err = self.pre_process(work_dir, song_url) if err != gs_err_code_success: return err st = time.time() err, data = self.process_one(str(cid), work_dir) logging.info("process_finish,{},{}".format(cid, time.time() - st)) if err == gs_err_code_success and len(data) == 2: sql = "update svc_queue_table set state={},update_time={},svc_url=\"{}\" where song_id = {}". \ format(gs_state_finish, int(time.time()), ",".join(data), str(cid)) banned_user_map['db'] = "av_db" update_db(sql, banned_user_map) else: self.update_state(str(cid), -err) shutil.rmtree(work_dir) logging.info("process_finish,{},{}".format(cid, time.time() - worker_st)) if __name__ == '__main__': ts = ToneShift() ts.process_worker() diff --git a/AutoCoverTool/ref/tools/mixer/tone_shift.cpp b/AutoCoverTool/ref/tools/mixer/tone_shift.cpp index 7e4e410..afbefab 100644 --- a/AutoCoverTool/ref/tools/mixer/tone_shift.cpp +++ b/AutoCoverTool/ref/tools/mixer/tone_shift.cpp @@ -1,245 +1,250 @@ // // Created by yangjianli on 2019-09-09. // /** * 输入一个音频和伴奏自动进行混合 * gated_loudness 当前音量 * gain 预期增益 */ #include "iostream" #include "WaveFile.h" #include "math.h" #include "ebur128.h" #include "AudioMixer.h" #include "alimiter.h" #include "waves/inc/WaveFile.h" #include "CAudioEffectsChainApi.h" #include "string" #include "ae_server/CAeServer.h" #include #include #include #include #include #include "denoise/webrtc/include/WebrtcDenoise.h" #define PROC_LEN 1024 #define DEFAULT_BASELINE_DB (float)-14.57f int short2float(short *pInBuf, int nLen, float *pOutBuf) { for (int i = 0; i < nLen; i++) { pOutBuf[i] = pInBuf[i] * 1.0 / 32768; } return 0; } int float2short(float *pInBuf, int nLen, short *pOutBuf) { for (int i = 0; i < nLen; i++) { pOutBuf[i] = int(pInBuf[i] * 32768); } return 0; } /** * 获取增益 * @param nChannel * @param nSampleRate * @param pData * @param nLength * @param gain * @return */ int ebur128_whole(int nChannel, int nSampleRate, short *pData, const int nLength, double &gated_loudness, double &gain) { printf("ebur128_init start .. %d\n", nLength); ebur128_state *st = NULL; st = ebur128_init(nChannel, nSampleRate, EBUR128_MODE_I); if (NULL == st) { return -1; } int nPos = 0; int nTmpLength = 0; int nRet; printf("process start ..\n"); while (nPos < nLength) { nTmpLength = PROC_LEN; if (nLength - nPos < PROC_LEN) { nTmpLength = nLength - nPos; } nRet = ebur128_add_frames_short(st, pData + nPos, nTmpLength / nChannel); if (nRet != 0) { return -2; } nPos += nTmpLength; } printf("process ok..\n"); gated_loudness = -1; ebur128_loudness_global(st, &gated_loudness); float db = (DEFAULT_BASELINE_DB - gated_loudness) / 20.f; gain = pow(10, db); printf("gated_loudness = %f db = %f gain = %f\n", gated_loudness, db, gain); ebur128_destroy(&st); return 0; } /** * 混合音频和伴奏 * @param pVocalIn * @param pAccIn * @param nLength * @param gainVocal * @param gainAcc * @param pOutBuf * @return */ int mix(float *pVocalIn, float *pAccIn, int nLength, double gainVocal, double gainAcc, float *pOutBuf, int nSampleRate, int nChannel, int nDelay, std::string effect_file) { CAudioMixer *cAudioMixer = new CAudioMixer(); cAudioMixer->init(nSampleRate, nChannel); cAudioMixer->set_acc_delay(nDelay); cAudioMixer->set_vocal_volume(int(gainVocal * 50)); cAudioMixer->set_acc_volume(int(gainAcc * 50)); int nPos = 0; int nStep = 1024; float *fTmp = new float[nStep]; cAudioMixer->reset(); nPos = 0; nStep = 1024; int cnt = 0; CAeServer cAeServer; cAeServer.init(nSampleRate, nChannel, nStep / nChannel); AE_PARAMS_IM_EFFECT im_params = { .effect_path = effect_file, }; cAeServer.set_params(AE_TYPE_IM_EFFECT, (void *) &im_params); while (nPos < nLength) { if (nLength - nPos < nStep) { nStep = nLength - nPos; } cnt++; cAeServer.process(pVocalIn + nPos, pVocalIn + nPos, nStep); cAudioMixer->process(pVocalIn + nPos, pAccIn + nPos, pOutBuf + nPos, nStep); nPos += nStep; } cAeServer.uninit(); delete cAudioMixer; delete[] fTmp; return 0; } int denoise_webrtc(short *pInBuf, int nLength, int nChannel, int nSampleRate) { CWebrtcDenoise cWebrtcDenoise; cWebrtcDenoise.init(nSampleRate, nChannel); float *pTmp = new float[nLength]; for (int i = 0; i < nLength; i++) { pTmp[i] = pInBuf[i] * 1.0 / 32768; } cWebrtcDenoise.set_level(kHigh); int nStep = 512 * nChannel; for (int i = 0; i < nStep; i++) { pTmp[i] = pTmp[i] * i * 1.0 / nStep; } for (int i = 0, cnt = 0; i < nLength; i += nStep, cnt++) { if (nLength - i < nStep) continue; cWebrtcDenoise.process(pTmp + i, nStep); } for (int i = 0; i < nLength; i++) { pInBuf[i] = short(pTmp[i] * 32768); } delete[] pTmp; return 0; } double calc_power_rate(float *in_data, int32_t in_len, float *ref_data, int32_t ref_len) { double in_power = 0; double ref_power = 0; int32_t min_len = in_len > ref_len ? ref_len : in_len; for (int i = 0; i < min_len; i++) { in_power += (in_data[i]) * (in_data[i]); ref_power += (ref_data[i]) * (ref_data[i]); } return ref_power / in_power; } int main(int argc, char *argv[]) { if (argc != 4) { printf("input error! example: ./main vocal_path dst_path pitch!\n"); return -1; } std::string vocal_path = argv[1]; std::string dst_path = argv[2]; float pitch = strtod(argv[3], NULL); // 读取人声 CWaveFile *oWaveFile = new CWaveFile(vocal_path.c_str(), false); float *pVocalBuf = new float[oWaveFile->GetTotalFrames() * oWaveFile->GetChannels()]; oWaveFile->ReadFrameAsfloat(pVocalBuf, oWaveFile->GetTotalFrames()); int nStep = 1024; int nLength = oWaveFile->GetTotalFrames() * oWaveFile->GetChannels(); CAeServer cAeServer; cAeServer.init(oWaveFile->GetSampleRate(), oWaveFile->GetChannels(), nStep / oWaveFile->GetChannels()); AEToneShiftParam ae_param; ae_param.max_shift = 12; ae_param.min_shift = -12; ae_param.tone_shift = pitch; cAeServer.set_params(AE_TYPE_TONE_SHIFT, &ae_param); int nPos = 0; while (nPos < nLength) { if (nLength - nPos < nStep) { nStep = nLength - nPos; } cAeServer.process(pVocalBuf + nPos, pVocalBuf + nPos, nStep); nPos += nStep; } + + // 剔除84ms延迟 + int latency_pos = int(cAeServer.get_latency_ms() * oWaveFile->GetSampleRate() / 1000.0) * oWaveFile->GetChannels(); + printf("latency_pos=%d\n", latency_pos); cAeServer.uninit(); + //写入文件 printf("write2file nLength:%d path:%s!\n", oWaveFile->GetTotalFrames() * oWaveFile->GetChannels(), dst_path.c_str()); CWaveFile out_wav = CWaveFile(dst_path.c_str(), true); out_wav.SetChannels(oWaveFile->GetChannels()); out_wav.SetSampleRate(oWaveFile->GetSampleRate()); out_wav.SetSampleFormat(SF_IEEE_FLOAT); out_wav.SetupDone(); - out_wav.WriteFrame(pVocalBuf, oWaveFile->GetTotalFrames()); + out_wav.WriteFrame(pVocalBuf+latency_pos, oWaveFile->GetTotalFrames()); delete oWaveFile; delete[] pVocalBuf; return 0; } \ No newline at end of file diff --git a/AutoCoverTool/script/get_song_url.py b/AutoCoverTool/script/get_song_url.py index 1e5314f..b3b5541 100644 --- a/AutoCoverTool/script/get_song_url.py +++ b/AutoCoverTool/script/get_song_url.py @@ -1,280 +1,761 @@ """ 获取歌曲的地址 # song_src=2 是来源108和109的歌曲，未被洗过的 # song_src=1 是曲库给的 # song_src=3 # 用于轻变调的 """ from script.common import * from copy import deepcopy from online.common import update_db def get_url_by_song_id(song_id): sql = "select task_url,starmaker_songid from silence where starmaker_songid = {} order by task_id limit 1".format( song_id) ban = deepcopy(banned_user_map) ban["db"] = "starmaker_musicbook" data = get_data_by_mysql(sql, ban) if len(data) > 0: return data[0][0] return None def process(): arr = [ - "611752105030484885", - "611752105029543722", - "611752105030556608", - "611752105030585154", - "611752105030556609", - "611752105029054060", - "611752105028975148", - "611752105030558172", - "611752105028778344", - "611752105030556613", - "611752105029290698", - "611752105030556605", - "611752105027484924", - "611752105030559472", - "611752105030534293", - "611752105027148644", - "611752105029292630", - "611752105026900917", - "611752105027103140", - "611752105030589795", - "611752105026915170", - "611752105030534289", - "611752105026751742", - "611752105026452638", - "611752105025979421", - "611752105025817810", - "611752105026536899", - "611752105030534282", - "611752105030534285", - "611752105030559474", - "611752105025219762", - "611752105025034426", - "611752105024938926", - "611752105029648740", - "611752105029675859", - "611752105024598727", - "611752105030548412", - "611752105030487271", - "611752105029648743", - "611752105023692976", - "611752105024135802", - "611752105023616288", - "611752105023255629", - "611752105022728286", - "611752105023206033", - "611752105023091102", - "611752105029792918", - "611752105022729259", - "611752105030487512", - "611752105022842120", - "611752105022842054", - "611752105022785621", - "611752105022840550", - "611752105022838205", - "611752105022839189", - "611752105022835751", - "611752105022818025", - "611752105022797521", - "611752105022784390", - "611752105028820609", - "611752105030488595", - "611752105030517536", - "611752105030501857", - "611752105030478339", - "611752105025957389", - "611752105027484925", - "611752105027484915", - "611752105024415490", - "611752105027854244", - "611752105029527187", - "611752105028870536", - "611752105028444597", - "611752105028778353", - "611752105027877846", - "611752105028906605", - "611752105027781526", - "611752105027877887", - "611752105027795229", - "611752105027734187", - "611752105028820612", - "611752105027626964", - "611752105027460080", - "611752105027507932", - "611752105027611342", - "611752105027435127", - "611752105027307631", - "611752105029648514", - "611752105026874730", - "611752105030591117", - "611752105026437853", - "611752105025541483", - "611752105026536913", - "611752105022647044", - "611752105023440333", - "611752105023460357", - "611752105023604729", - "611752105023510939", - "611752105022842387", - "611752105024230229", - "611752105023674599", - "611752105023160140", - "611752105022647074", - "611752105022615220", - "611752105028408822", - "611752105022816170", - "611752105022772279", - "611752105022614618", - "611752105020417684", - "611752105020382477", - "611752105022780345", - "611752105022780961", - "611752105022837186", - "611752105022778042", - "611752105022775939", - "611752105022764224", - "611752105022781267", - "611752105022839030", - "611752105022767294", - "611752105022784996", - "611752105022775600", - "611752105022780284", - "611752105022768837", - "611752105030590847", - "611752105022780965", - "611752105022779020", - "611752105022777496", - "611752105022781268", - "611752105022785681", - "611752105022779294", - "611752105022823781", - "611752105022780210", - "611752105022774220", - "611752105022768419", - "611752105030590845", - "611752105022835406", - "611752105022774040", - "611752105022783776", - "611752105022781193", - "611752105020390942", - "611752105022783967", - "611752105022763051", - "611752105022780818", - "611752105022835415", - "611752105022782935", - "611752105020402448", - "611752105022781011", - "611752105020384960", - "611752105022779784", - "611752105022781387", - "611752105025580424", - "611752105022765022", - "611752105025492732", - "611752105023683356", - "611752105022842241", - "611752105024231227", - "611752105029291290", - "611752105023104185", - "611752105025565044", - "611752105025458749", - "611752105025458753", - "611752105025090763", - "611752105030590839", - "611752105030534180", - "611752105023908922", - "611752105027326105", - "611752105023725727", - "611752105022647079", - "611752105024082232", - "611752105029648891", - "611752105025504662", - "611752105025496983", - "611752105026716551", - "611752105029648872", - "611752105022614531", - "611752105029041707", - "611752105030483313", - "611752105023219237", - "611752105022842989", - "611752105022746733", - "611752105023162802", - "611752105022729263", - "611752105022777120", - "611752105025584544", - "611752105025458809", - "611752105027648113", - "611752105030590840", - "611752105024183682", - "611752105023086347", - "611752105022839975", - "611752105025348359", - "611752105022781144", - "611752105022647060", - "611752105022728482", - "611752105025840622", - "611752105022836470", - "611752105023246015", - "611752105022838206", - "611752105022780355", - "611752105022768062", - "611752105022777600" + "611752105016642206", + "611752105016665828", + "611752105020332340", + "611752105020332347", + "611752105020390931", + "611752105020417679", + "611752105021442334", + "611752105021459528", + "611752105021810110", + "611752105021916843", + "611752105022312180", + "611752105022614734", + "611752105022615541", + "611752105022615665", + "611752105022616931", + "611752105022647066", + "611752105022647087", + "611752105022652047", + "611752105022700847", + "611752105022728649", + "611752105022728653", + "611752105022729255", + "611752105022733605", + "611752105022736029", + "611752105022740011", + "611752105022742166", + "611752105022743986", + "611752105022746848", + "611752105022748944", + "611752105022749272", + "611752105022749768", + "611752105022751902", + "611752105022752248", + "611752105022754267", + "611752105022754702", + "611752105022755405", + "611752105022757577", + "611752105022758309", + "611752105022758395", + "611752105022761145", + "611752105022761151", + "611752105022764435", + "611752105022764965", + "611752105022766106", + "611752105022766675", + "611752105022774258", + "611752105022776257", + "611752105022776285", + "611752105022776577", + "611752105022776846", + "611752105022777151", + "611752105022777306", + "611752105022778977", + "611752105022779055", + "611752105022779169", + "611752105022779962", + "611752105022780648", + "611752105022784727", + "611752105022785048", + "611752105022785179", + "611752105022785551", + "611752105022811718", + "611752105022814368", + "611752105022815931", + "611752105022819519", + "611752105022824948", + "611752105022828998", + "611752105022833822", + "611752105022835250", + "611752105022838357", + "611752105022838544", + "611752105022838589", + "611752105022838666", + "611752105022839015", + "611752105022839300", + "611752105022839468", + "611752105022839559", + "611752105022892354", + "611752105022911042", + "611752105023134539", + "611752105023142842", + "611752105023588204", + "611752105023588294", + "611752105024204862", + "611752105024546859", + "611752105024598735", + "611752105024608149", + "611752105024728134", + "611752105024938931", + "611752105025198792", + "611752105025327479", + "611752105025458721", + "611752105025458732", + "611752105025458759", + "611752105025458792", + "611752105025458796", + "611752105025458831", + "611752105025534832", + "611752105025538466", + "611752105025565027", + "611752105025584548", + "611752105025586093", + "611752105025720331", + "611752105025741198", + "611752105025741447", + "611752105025817802", + "611752105025835130", + "611752105025879265", + "611752105026152312", + "611752105026205732", + "611752105026281560", + "611752105026343284", + "611752105026388268", + "611752105026421148", + "611752105026433966", + "611752105026536897", + "611752105026536911", + "611752105026580839", + "611752105026648945", + "611752105026663363", + "611752105026736866", + "611752105027067863", + "611752105027112518", + "611752105027186556", + "611752105027189208", + "611752105027189301", + "611752105027326104", + "611752105027460089", + "611752105027484913", + "611752105027588072", + "611752105027611383", + "611752105027690075", + "611752105028183260", + "611752105028470803", + "611752105028507652", + "611752105028528335", + "611752105028809597", + "611752105028815367", + "611752105028820629", + "611752105028820633", + "611752105028827057", + "611752105028837845", + "611752105028878340", + "611752105028906600", + "611752105028944645", + "611752105028958744", + "611752105029006303", + "611752105029006319", + "611752105029059923", + "611752105029078388", + "611752105029090034", + "611752105029209546", + "611752105029243449", + "611752105029272970", + "611752105029291291", + "611752105029291294", + "611752105029291295", + "611752105029291297", + "611752105029291298", + "611752105029291304", + "611752105029395411", + "611752105029432787", + "611752105029570149", + "611752105029570153", + "611752105029570157", + "611752105029648873", + "611752105029648877", + "611752105029648879", + "611752105029953987", + "611752105029954853", + "611752105029955024", + "611752105029955258", + "611752105029956379", + "611752105029956615", + "611752105029990162", + "611752105029990590", + "611752105029991249", + "611752105030103635", + "611752105030119229", + "611752105030124600", + "611752105030483296", + "611752105030485000", + "611752105030485417", + "611752105030485428", + "611752105030485533", + "611752105030485561", + "611752105030485562", + "611752105030485565", + "611752105030485566", + "611752105030485569", + "611752105030485570", + "611752105030485572", + "611752105030485591", + "611752105030485592", + "611752105030485594", + "611752105030485595", + "611752105030485597", + "611752105030485598", + "611752105030485601", + "611752105030485602", + "611752105030485607", + "611752105030485608", + "611752105030485610", + "611752105030485612", + "611752105030485613", + "611752105030485616", + "611752105030485618", + "611752105030485620", + "611752105030485621", + "611752105030485626", + "611752105030485627", + "611752105030485631", + "611752105030485634", + "611752105030485637", + "611752105030485639", + "611752105030485642", + "611752105030485646", + "611752105030485650", + "611752105030485653", + "611752105030485655", + "611752105030485656", + "611752105030485662", + "611752105030485663", + "611752105030485666", + "611752105030485667", + "611752105030485669", + "611752105030485671", + "611752105030485672", + "611752105030485673", + "611752105030485676", + "611752105030485677", + "611752105030485679", + "611752105030485681", + "611752105030485682", + "611752105030485685", + "611752105030485687", + "611752105030485688", + "611752105030485691", + "611752105030485692", + "611752105030485693", + "611752105030485696", + "611752105030485697", + "611752105030485700", + "611752105030485702", + "611752105030485703", + "611752105030485707", + "611752105030485710", + "611752105030485711", + "611752105030485715", + "611752105030485716", + "611752105030485717", + "611752105030485721", + "611752105030485722", + "611752105030485724", + "611752105030485726", + "611752105030485727", + "611752105030485729", + "611752105030485731", + "611752105030485733", + "611752105030485734", + "611752105030485736", + "611752105030485737", + "611752105030485738", + "611752105030485739", + "611752105030485741", + "611752105030485742", + "611752105030485744", + "611752105030485745", + "611752105030485748", + "611752105030485749", + "611752105030485750", + "611752105030485755", + "611752105030485758", + "611752105030485759", + "611752105030485761", + "611752105030485763", + "611752105030485766", + "611752105030485768", + "611752105030485769", + "611752105030485772", + "611752105030485778", + "611752105030485779", + "611752105030485787", + "611752105030485790", + "611752105030485791", + "611752105030485794", + "611752105030485797", + "611752105030485799", + "611752105030488510", + "611752105030488594", + "611752105030488665", + "611752105030488713", + "611752105030488727", + "611752105030488744", + "611752105030488814", + "611752105030488836", + "611752105030488852", + "611752105030488864", + "611752105030488880", + "611752105030488962", + "611752105030488997", + "611752105030489153", + "611752105030489354", + "611752105030489380", + "611752105030489394", + "611752105030489403", + "611752105030489415", + "611752105030562265", + "611752105030562283", + "611752105030562308", + "611752105030562320", + "611752105030562341", + "611752105030562353", + "611752105030562357", + "611752105030562371", + "611752105030562377", + "611752105030562386", + "611752105030562451", + "611752105030562466", + "611752105030562487", + "611752105030562933", + "611752105030562937", + "611752105030562946", + "611752105030563002", + "611752105030563012", + "611752105030563018", + "611752105030563020", + "611752105030563117", + "611752105030563422", + "611752105030563435", + "611752105030563469", + "611752105030563484", + "611752105030563555", + "611752105030563570", + "611752105030563584", + "611752105030563620", + "611752105030563722", + "611752105030563730", + "611752105030563754", + "611752105030590082", + "611752105030590145", + "611752105030590214", + "611752105030590224", + "611752105030590259", + "611752105030590265", + "611752105030590274", + "611752105030590490", + "611752105030590649", + "611752105030590659", + "611752105030590671", + "611752105030590689", + "611752105030590706", + "611752105030590735", + "611752105030590750", + "611752105030590757", + "611752105030590939", + "611752105030590951", + "611752105030590977", + "611752105030590993", + "611752105030591011", + "611752105030591021", + "611752105030591053", + "611752105030591070", + "611752105030591083", + "611752105030591094", + "611752105030591105", + "611752105030591162", + "611752105030591181", + "611752105030591184", + "611752105030591185", + "611752105030591187", + "611752105030591191", + "611752105030591193", + "611752105030591195", + "611752105030591198", + "611752105030591200", + "611752105030591203", + "611752105030591204", + "611752105030591209", + "611752105030591213", + "611752105030591214", + "611752105030591215", + "611752105030591221", + "611752105030593095", + "611752105030593133", + "611752105030593144", + "611752105030593155", + "611752105030593181", + "611752105030593200", + "611752105030593208", + "611752105030593346", + "611752105030593368", + "611752105030593384", + "611752105030593396", + "611752105030593416", + "611752105030593432", + "611752105030593445", + "611752105030593465", + "611752105030593481", + "611752105030593496", + "611752105030593509", + "611752105030593569", + "611752105030593588", + "611752105030593597", + "611752105030593606", + "611752105030593618", + "611752105030593633", + "611752105030593645", + "611752105030593658", + "611752105030593671", + "611752105030593685", + "611752105030593936", + "611752105030593955", + "611752105030593962", + "611752105030593970", + "611752105030593976", + "611752105030593985", + "611752105030593998", + "611752105030594008", + "611752105030594018", + "611752105030594023", + "611752105030594041", + "611752105030594045", + "611752105030594053", + "611752105030594070", + "611752105030594075", + "611752105030594081", + "611752105030594088", + "611752105030594099", + "611752105030594107", + "611752105030594117", + "611752105030594121", + "611752105030594128", + "611752105030594137", + "611752105030594148", + "611752105030594158", + "611752105030594164", + "611752105030594171", + "611752105030594176", + "611752105030594192", + "611752105030594200", + "611752105030594208", + "611752105030594221", + "611752105030594256", + "611752105030594262", + "611752105030594275", + "611752105030594286", + "611752105030594292", + "611752105030594296", + "611752105030594299", + "611752105030594311", + "611752105030594313", + "611752105030594319", + "611752105030594325", + "611752105030594330", + "611752105030594336", + "611752105030594343", + "611752105030594348", + "611752105030594376", + "611752105030594384", + "611752105030595120", + "611752105030595128", + "611752105030595200", + "611752105030595219", + "611752105030595343", + "611752105030595375", + "611752105030595385", + "611752105030595393", + "611752105030595411", + "611752105030595429", + "611752105030595472", + "611752105030595495", + "611752105030595505", + "611752105030595532", + "611752105030595544", + "611752105030595555", + "611752105030595563", + "611752105030595587", + "611752105030595596", + "611752105030595628", + "611752105030595640", + "611752105030595654", + "611752105030595661", + "611752105030595673", + "611752105030595714", + "611752105030595730", + "611752105030595740", + "611752105030595774", + "611752105030595812", + "611752105030595820", + "611752105030595829", + "611752105030595910", + "611752105030595941", + "611752105030595983", + "611752105030596002", + "611752105030596035", + "611752105030596046", + "611752105030596057", + "611752105030596072", + "611752105030596082", + "611752105030596128", + "611752105030596137", + "611752105030596373", + "611752105030596416", + "611752105030596423", + "611752105030596432", + "611752105030596435", + "611752105030596439", + "611752105030596444", + "611752105030596449", + "611752105030596468", + "611752105030596478", + "611752105030596479", + "611752105030596484", + "611752105030596500", + "611752105030596502", + "611752105030596509", + "611752105030596513", + "611752105030596523", + "611752105030596535", + "611752105030596536", + "611752105030596560", + "611752105030596570", + "611752105030596576", + "611752105030596584", + "611752105030596585", + "611752105030596593", + "611752105030596600", + "611752105030596612", + "611752105030596621", + "611752105030596681", + "611752105030596696", + "611752105030596716", + "611752105030596759", + "611752105030596782", + "611752105030596790", + "611752105030596806", + "611752105030596826", + "611752105030596827", + "611752105030596830", + "611752105030596831", + "611752105030596833", + "611752105030596842", + "611752105030596852", + "611752105030596854", + "611752105030596864", + "611752105030596871", + "611752105030596875", + "611752105030596879", + "611752105030596883", + "611752105030596887", + "611752105030596888", + "611752105030596895", + "611752105030596896", + "611752105030596899", + "611752105030596903", + "611752105030596906", + "611752105030596913", + "611752105030596919", + "611752105030596920", + "611752105030596924", + "611752105030596925", + "611752105030596931", + "611752105030596934", + "611752105030596935", + "611752105030596939", + "611752105030596949", + "611752105030596952", + "611752105030596956", + "611752105030596965", + "611752105030596970", + "611752105030596973", + "611752105030596975", + "611752105030596978", + "611752105030596985", + "611752105030596994", + "611752105030596998", + "611752105030596999", + "611752105030597001", + "611752105030597006", + "611752105030597009", + "611752105030597013", + "611752105030597015", + "611752105030597018", + "611752105030597025", + "611752105030597028", + "611752105030597031", + "611752105030597032", + "611752105030597038", + "611752105030597044", + "611752105030597049", + "611752105030597055", + "611752105030597058", + "611752105030597063", + "611752105030597072", + "611752105030597083", + "611752105030597091", + "611752105030597099", + "611752105030597104", + "611752105030597109", + "611752105030597112", + "611752105030597115", + "611752105030597120", + "611752105030597132", + "611752105030597136", + "611752105030597141", + "611752105030597144", + "611752105030597149", + "611752105030597154", + "611752105030597159", + "611752105030597161", + "611752105030597168", + "611752105030597170", + "611752105030597177", + "611752105030597181", + "611752105030597186", + "611752105030597187", + "611752105030597191", + "611752105030597192", + "611752105030597194", + "611752105030597198", + "611752105030597204", + "611752105030597205", + "611752105030597208", + "611752105030597212", + "611752105030597213", + "611752105030597218", + "611752105030597221", + "611752105030597225", + "611752105030597229", + "611752105030597230", + "611752105030597233", + "611752105030597236", + "611752105030597238", + "611752105030597241", + "611752105030597246", + "611752105030597248", + "611752105030597250", + "611752105030597253", + "611752105030597254", + "611752105030597257", + "611752105030597260", + "611752105030597261", + "611752105030597264", + "611752105030597265", + "611752105030597266", + "611752105030597269", + "611752105030597271", + "611752105030597275", + "611752105030597281", + "611752105030597295", + "611752105030597299", + "611752105030597306", + "611752105030597318", + "611752105030597327", + "611752105030597328", + "611752105030597330", + "611752105030597334", + "611752105030597339", + "611752105030597347", + "611752105030597348", + "611752105030597356", + "611752105030597359", + "611752105030597372", + "611752105030597385", + "611752105030597417", + "611752105030597424", + "611752105030597443", + "611752105030597447", + "611752105030597450", + "611752105030597456", + "611752105030597461", + "611752105030597468", + "611752105030597476", + "611752105030597505", + "611752105030598550", + "611752105030598562", + "611752105030598634", + "611752105030598642", + "611752105030598665" ] ban = deepcopy(banned_user_map) ban["db"] = "av_db" for sid in arr: url = get_url_by_song_id(sid) if url is not None: print("out,{},{}".format(url, sid)) # 不在数据库中 sql = "select song_id from svc_queue_table where song_id={}".format(sid) - data = get_data_by_mysql(sql, ban) - if len(data) == 0: - tm = int(time.time()) - sql = "insert INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\",{}, {}, 1)" \ - .format(sid, url, tm, tm) - update_db(sql, ban) + # data = get_data_by_mysql(sql, ban) + # if len(data) == 0: + tm = int(time.time()) + sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\",{}, {}, 3)" \ + .format(sid, url, tm, tm) + update_db(sql, ban) def get_data_from_song(): sql = """ select tb1.song_id, tb1.recording_count from ( select song_id,recording_count from starmaker.song where song_src in (108,109) and song_status = 2 order by recording_count desc ) as tb1 left join ( select song_id from av_db.svc_queue_table ) as tb2 on tb1.song_id = tb2.song_id where tb2.song_id is null - order by tb1.recording_count desc limit 400 + order by tb1.recording_count desc limit 1000 """ ban = deepcopy(banned_user_map) ban_v1 = deepcopy(banned_user_map) ban["db"] = "starmaker_musicbook" ban_v1["db"] = "av_db" data = get_data_by_mysql(sql, ban) for dt in data: sid = dt[0] url = get_url_by_song_id(sid) if url is not None: print("out,{},{}".format(url, sid)) - sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time) VALUES ({}, \"{}\", NOW(), NOW())" \ - .format(sid, url) + tm = int(time.time()) + sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\", {}, {}, 3)" \ + .format(sid, url, tm, tm) update_db(sql, ban_v1) if __name__ == '__main__': # get_data_from_song() process()