Page MenuHomePhabricator

No OneTemporary

diff --git a/AutoCoverTool/online/inference_one.py b/AutoCoverTool/online/inference_one.py
index a73d87d..f60be54 100644
--- a/AutoCoverTool/online/inference_one.py
+++ b/AutoCoverTool/online/inference_one.py
@@ -1,689 +1,711 @@
"""
单个处理的逻辑
song_id:
---src.mp3 // 源数据,需要提前放进去
---cache
---vocal.wav // 分离之后产生
---acc.wav // 分离之后产生
---vocal_32.wav // 分离之后产生
---song_id_sp1.wav // 合成之后产生
---song_id_sp2.wav // 合成之后产生
---song_id_sp2_d.wav // 降噪之后生成
---song_id_sp2_dv.wav // 降噪+拉伸之后产生 [占比太高的不产生]
---song_id_sp2_dve442.wav // 手动调整之后产生
---song_id_sp2_dve442_replace.wav // 替换之后产生
---song_id_sp2_dve442_replace_mix.wav // 人声+伴奏混合之后产生
---song_id
--acc.mp3 // 44k双声道320k
--vocal.mp3 // 44k双声道320k
--src.mp3 // 44k双声道320k
--song_id_sp2_dv.mp3 // 44k单声道320k
---song_id_out // 对外输出
--src.mp3 // 原始音频
--song_id_sp2_dv_replace_mix.mp3 // 制作完成的音频
环境安装:
conda create -n auto_song_cover python=3.9
# 安装demucs环境[进入到ref.music_remover 执行pip install -r requirements.txt]
# 安装so_vits_svc环境[进入到ref.so_vits_svc 执行pip install -r requirements.txt]
pip install librosa
pip install scikit-maad
pip install praat-parselmouth
pip install matplotlib
pip install torchvision
pip install madmom
pip install torchstat
环境设置:
export PATH=$PATH:/data/gpu_env_common/env/bin/ffmpeg/bin
export PYTHONPATH=$PWD:$PWD/ref/music_remover/demucs:$PWD/ref/so_vits_svc:$PWD/ref/split_dirty_frame
"""
import os
import time
import shutil
import random
import logging
import librosa
logging.basicConfig(filename='/tmp/inference.log', level=logging.INFO)
gs_err_code_success = 0
gs_err_code_no_src_mp3 = 1
gs_err_code_separate = 2
gs_err_code_trans_32 = 3
gs_err_code_encode_err = 4
gs_err_code_replace_err = 5
gs_err_code_replace_trans_err = 6
gs_err_code_mix_err = 7
gs_err_code_mix_transcode_err = 8
gs_err_code_no_src_dir = 9
gs_err_code_volume_err = 10
gs_err_code_trans2_442 = 11
gs_err_code_reverb = 12
gs_err_code_no_good_choice = 13
gs_err_code_preprocess_vocal = 14
gs_err_code_replace_except_err = 15
gs_denoise_exe = "/opt/soft/bin/denoise_exe"
gs_draw_volume_exe = "/opt/soft/bin/draw_volume"
gs_simple_mixer_path = "/opt/soft/bin/simple_mixer"
gs_rever_path = "/opt/soft/bin/dereverbrate"
from ref.music_remover.separate_interface import SeparateInterface
from ref.so_vits_svc.inference_main import *
from ref.split_dirty_frame.script.process_one import ReplaceVocalFrame, construct_power_fragment
class SongCoverInference:
def __init__(self):
self.work_dir = None
self.cache_dir = None
self.cid = None
self.src_mp3 = None
self.vocal_path = None
self.vocal_32_path = None
self.acc_path = None
self.speakers = [
10414574138721494,
10414574140317353,
1688849864840588,
3634463651,
5629499489839033,
5910973794723621,
6755399374234747,
8162774327817435,
8162774329368194,
1125899914308640, # 以下为男声,包括这个
12384898975368914,
12947848931397021,
3096224748076687,
3096224751151928,
5066549357604730,
5348024335101054,
6755399442719465,
7036874421386111
]
self.speakers2gender = {
10414574138721494: 2,
10414574140317353: 2,
1688849864840588: 2,
3634463651: 2,
5629499489839033: 2,
5910973794723621: 2,
6755399374234747: 2,
8162774327817435: 2,
8162774329368194: 2,
1125899914308640: 1, # 1是男
12384898975368914: 1,
12947848931397021: 1,
3096224748076687: 1,
3096224751151928: 1,
5066549357604730: 1,
5348024335101054: 1,
6755399442719465: 1,
7036874421386111: 1
}
self.speakers_model_path = "data/train_users/{}/logs/32k/G_2000.pth"
self.speakers_model_config = "data/train_users/{}/config/config.json"
st = time.time()
self.separate_inst = None
logging.info("post process ... ReplaceVocalFrame init sp={}".format(time.time() - st))
self.replace_vocal_frame_inst = None
logging.info("SongCoverInference init sp={}".format(time.time() - st))
def separate(self, cid, src_mp3, vocal_path, acc_path):
"""
人声伴奏分离
:param cid:
:param src_mp3:
:param vocal_path:
:param acc_path:
:return:
"""
st = time.time()
if self.separate_inst is None:
self.separate_inst = SeparateInterface()
if not self.separate_inst.process(cid, src_mp3, vocal_path, acc_path):
return gs_err_code_separate
if not os.path.exists(vocal_path) or not os.path.exists(acc_path):
return gs_err_code_separate
# 转码出一个32k单声道的数据
cmd = "ffmpeg -i {} -ar 32000 -ac 1 -y {} -loglevel fatal".format(vocal_path, self.vocal_32_path)
os.system(cmd)
if not os.path.exists(self.vocal_32_path):
return gs_err_code_trans_32
print("separate:cid={}|sp={}".format(cid, time.time() - st))
return gs_err_code_success
def get_start_ms(self, vocal_path):
"""
给定原始音频,找一段连续10s的音频
:param vocal_path:
:return:
"""
audio, sr = librosa.load(vocal_path, sr=16000)
audio = librosa.util.normalize(audio)
# 帧长100ms,帧移10ms,计算能量
power_arr = []
for i in range(0, len(audio) - 1600, 160):
power_arr.append(np.sum(np.abs(audio[i:i + 160])) / 160)
# 将能量小于等于10的部分做成段
power_arr = construct_power_fragment(power_arr)
fragments = []
last_pos = 0
for idx, line in enumerate(power_arr):
start = round(float(line[0]) * 0.01, 3)
duration = round(float(line[1]) * 0.01, 3)
fragments.append([last_pos, start - last_pos])
last_pos = start + duration
if last_pos < len(audio) / sr:
fragments.append([last_pos, len(audio) / sr - last_pos])
# 合并数据,两者间隔在50ms以内的合并起来
idx = 0
while idx < len(fragments) - 1:
if fragments[idx + 1][0] - (fragments[idx][0] + fragments[idx][1]) < 0.05:
fragments[idx][1] = fragments[idx + 1][0] + fragments[idx + 1][1] - fragments[idx][0]
del fragments[idx + 1]
idx -= 1
idx += 1
# out_file = vocal_path + "_power.csv"
# with open(out_file, "w") as f:
# f.write("Name\tStart\tDuration\tTime Format\tType\n")
# for fragment in fragments:
# start = round(float(fragment[0]), 3)
# duration = round(float(fragment[1]), 3)
# strr = "{}\t{}\t{}\t{}\n".format("11", start, duration, "decimal\tCue\t")
# f.write(strr)
# 筛选出开始的位置
# 1. 连续时长大于10s,当前段长度大于3s
# 2. 不可用
# 从0到fragments[idx], 包含idx其中人声段的总和
tot_vocal_duration = [fragments[0][1]]
for i in range(1, len(fragments)):
tot_vocal_duration.append(tot_vocal_duration[i - 1] + fragments[i][1])
# 计算出任意两段之间非人声占比
for i in range(0, len(fragments)):
if fragments[i][1] >= 3:
now_tot = 0
if i > 0:
now_tot = tot_vocal_duration[i - 1]
for j in range(i + 1, len(fragments)):
cur_rate = tot_vocal_duration[j] - now_tot
cur_rate = cur_rate / (fragments[j][1] + fragments[j][0] - fragments[i][0])
if cur_rate > 0.1:
return fragments[i][0]
return -1
def inference_speaker(self):
"""
推理生成合成后的音频
随机取5个干声,选择占比最小的,并且要求占比小于0.3
:return:
"""
st = time.time()
out_speakers = random.sample(self.speakers, 15)
out_songs_dict = {}
for speaker in out_speakers:
model_path = self.speakers_model_path.format(speaker)
config_path = self.speakers_model_config.format(speaker)
song_path = os.path.join(self.cache_dir, "{}_{}.wav".format(self.cid, speaker))
try:
inf(model_path, config_path, self.vocal_32_path, song_path, "prod")
except Exception as ex:
logging.info("cid={}, inference_speaker err={}".format(self.cid, ex))
continue
if os.path.exists(song_path):
if self.replace_vocal_frame_inst is None:
self.replace_vocal_frame_inst = ReplaceVocalFrame(
"data/models/split_dirty_frame_v5_3_epoch3_852.pth")
rate = self.replace_vocal_frame_inst.get_rate(song_path)
if rate < 0.3:
out_songs_dict[song_path] = rate
# 从内部选择占比最低的
out_songs = []
if len(out_songs_dict.keys()) > 0:
st_sec = self.get_start_ms(self.vocal_path)
song_msg = sorted(out_songs_dict.items(), key=lambda kv: kv[1])[0]
out_songs = [song_msg[0]]
logging.info("GetRate:cid={},song={},rate={},st_tm={}".format(self.cid, song_msg[0], round(song_msg[1], 2),
round(st_sec, 3)))
print("GetRate:cid={},song={},rate={},st_tm={}".format(self.cid, song_msg[0], round(song_msg[1], 2),
round(st_sec, 3)))
# logging.info("inference_speaker len = {} finish sp = {}".format(len(out_songs), time.time() - st))
print("inference_speaker len = {} finish sp = {}".format(len(out_songs), time.time() - st))
return out_songs
def get_new_vocal_rate(self, songs):
"""
获取人声的比率
:param songs:
:return:
"""
st = time.time()
need_to_process_song = []
for song in songs:
if self.replace_vocal_frame_inst is None:
self.replace_vocal_frame_inst = ReplaceVocalFrame("data/models/split_dirty_frame_v5_3_epoch3_852.pth")
rate = self.replace_vocal_frame_inst.get_rate(song)
logging.info("{} {} replace_rate={}".format(self.cid, song, rate))
if rate < 1.0:
need_to_process_song.append(song)
logging.info(
"get_new_vocal_rate belen = {} len = {} finish sp = {}".format(len(songs), len(need_to_process_song),
time.time() - st))
return need_to_process_song
def preprocess_vocal(self, songs, vocal_path):
"""
1. 降噪
2. 拉伸
:param songs:
:param vocal_path: 参考的音频信号
:return:
"""
st = time.time()
dv_out_list = []
for song in songs:
denoise_path = str(song).replace(".wav", "_d.wav")
cmd = "{} {} {}".format(gs_denoise_exe, song, denoise_path)
os.system(cmd)
if not os.path.exists(denoise_path):
print("{} {} ERROR denoise".format(self.cid, song))
continue
# 拉伸
volume_path = str(song).replace(".wav", "_dv.wav")
cmd = "{} {} {} {}".format(gs_draw_volume_exe, denoise_path, vocal_path, volume_path)
os.system(cmd)
if not os.path.exists(volume_path):
print("{} {} ERROR denoise".format(self.cid, volume_path))
continue
dv_out_list.append(volume_path)
print(
"preprocess_vocal belen = {} len = {} finish sp = {}".format(len(songs), len(dv_out_list),
time.time() - st))
return dv_out_list
def output(self, dv_out_list):
"""
对外输出数据
:param dv_out_list:
:return:
"""
st = time.time()
out_dir = os.path.join(self.work_dir, self.cid)
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
# 拷贝数据
dst_mp3_path = os.path.join(out_dir, "src_mp3")
dst_acc_path = os.path.join(out_dir, "acc.mp3")
dst_vocal_path = os.path.join(out_dir, "vocal.mp3")
shutil.copyfile(self.src_mp3, dst_mp3_path)
cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(self.acc_path, dst_acc_path)
os.system(cmd)
if not os.path.exists(dst_acc_path):
return gs_err_code_encode_err
cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(self.vocal_path, dst_vocal_path)
os.system(cmd)
if not os.path.exists(dst_vocal_path):
return gs_err_code_encode_err
# 将所有数据放到out_dir中,用于给人工标注
for dv_wav in dv_out_list:
dv_wav_name = str(dv_wav).split("/")[-1].replace(".wav", "_441.mp3")
dst_dv_path = os.path.join(out_dir, dv_wav_name)
cmd = "ffmpeg -i {} -ar 44100 -ac 1 -ab 320k -y {} -loglevel fatal".format(dv_wav, dst_dv_path)
os.system(cmd)
if not os.path.exists(dst_dv_path):
print("{} encode err!".format(cmd))
continue
logging.info(
"preprocess_vocal output sp = {}".format(time.time() - st))
def process_one(self, cid, work_dir, enable_output=False):
logging.info("\nstart:cid={},work_dir={}----------------------->>>>>>>>".format(cid, work_dir))
self.cid = cid
self.work_dir = work_dir
# 所有不对外交付的,全部放到这里
self.cache_dir = os.path.join(work_dir, "cache")
if os.path.exists(self.cache_dir):
shutil.rmtree(self.cache_dir)
os.makedirs(self.cache_dir)
self.src_mp3 = os.path.join(self.work_dir, "src.mp3")
if not os.path.exists(self.src_mp3):
return gs_err_code_no_src_mp3
self.vocal_path = os.path.join(self.cache_dir, "vocal.wav")
self.vocal_32_path = os.path.join(self.cache_dir, "vocal_32.wav")
self.acc_path = os.path.join(self.cache_dir, "acc.wav")
if not os.path.exists(self.vocal_32_path):
logging.info("start separate ... {} {} {}".format(self.src_mp3, self.vocal_path, self.acc_path))
err = self.separate(cid, self.src_mp3, self.vocal_path, self.acc_path)
if err != gs_err_code_success:
return err, None, None
logging.info("start inference_speaker ...")
out_songs = self.inference_speaker()
dv_out_list = self.preprocess_vocal(out_songs, self.vocal_path)
if len(dv_out_list) == 0:
return gs_err_code_no_good_choice, None, None
mix_mp3_path = None
gender = -1
if enable_output:
self.output(dv_out_list)
else:
# 默认全部处理一遍
for dv_out_path in dv_out_list:
src_path = dv_out_path.replace("_dv.wav", ".wav")
err, mix_mp3_path = self.after_process(self.cid, self.work_dir, src_path, dv_out_path, self.vocal_path,
self.acc_path,
True, False)
if err != gs_err_code_success:
logging.info("after_process err {}".format(err))
# 取出性别属性
if err == gs_err_code_success and mix_mp3_path is not None:
gender = self.speakers2gender[int(str(os.path.basename(mix_mp3_path)).split("_")[1])]
logging.info("finish:cid={},work_dir={}----------------------->>>>>>>>".format(cid, work_dir))
return gs_err_code_success, mix_mp3_path, gender
def reverb_by_vocal(self, file):
st = time.time()
file_442 = file.replace(".wav", "_442.wav")
if not os.path.exists(file_442):
cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {}".format(file, file_442)
os.system(cmd)
if not os.path.exists(file_442):
return gs_err_code_trans2_442, None
file_dst = file.replace(".wav", "_442_dr.wav")
cmd = "{} {} {} {}".format(gs_rever_path, self.vocal_path, file_442, file_dst)
os.system(cmd)
if not os.path.exists(file_dst):
return gs_err_code_reverb, None
print("cid = {}, reverb_by_vocal sp={}".format(self.cid, time.time() - st))
return gs_err_code_success, file_dst
def after_process(self, cid, work_dir, in_file, effect_file, vocal_file, acc_file, need_draw=True,
need_reverb=True):
"""
后处理逻辑
将处理好的音频进行替换,然后和伴奏进行混合,最后进行编码
:return:
"""
if need_reverb:
# 抓取混响
err, effect_file = self.reverb_by_vocal(in_file)
if err != gs_err_code_success:
return err, None
if need_draw:
# 增加一个拉伸的步骤
volume_path = str(effect_file).replace(".wav", "_dv.wav")
cmd = "{} {} {} {}".format(gs_draw_volume_exe, effect_file, vocal_file, volume_path)
print(cmd)
os.system(cmd)
if not os.path.exists(volume_path):
print("{} {} ERROR draw volume".format(self.cid, volume_path))
return gs_err_code_volume_err, None
effect_file = volume_path
st = time.time()
self.cid = cid
self.work_dir = work_dir
self.src_mp3 = os.path.join(self.work_dir, "src.mp3")
if not os.path.exists(self.work_dir):
return gs_err_code_no_src_dir
self.replace_vocal_frame_inst.process(in_file, effect_file, vocal_file)
dst_path = effect_file + "_replace.wav"
if not os.path.exists(dst_path):
return gs_err_code_replace_err, None
print("replace_vocal_frame_inst sp = {}".format(time.time() - st))
# 转码
dst_path_442 = dst_path.replace("_replace.wav", "_replace442.wav")
cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} -loglevel fatal".format(dst_path, dst_path_442)
os.system(cmd)
if not os.path.exists(dst_path_442):
return gs_err_code_replace_trans_err, None
# 合并转码后再做一次拉伸,保证响度
volume_path = str(dst_path_442).replace(".wav", "_dv.wav")
cmd = "{} {} {} {}".format(gs_draw_volume_exe, dst_path_442, vocal_file, volume_path)
print(cmd)
os.system(cmd)
if not os.path.exists(volume_path):
print("{} {} ERROR draw volume".format(self.cid, volume_path))
return gs_err_code_volume_err, None
dst_path_442 = volume_path
# 混合
mix_path = dst_path_442.replace("_replace442.wav", "_replace442_mix.wav")
cmd = "{} {} {} {}".format(gs_simple_mixer_path, dst_path_442, acc_file, mix_path)
print("{}".format(cmd))
os.system(cmd)
if not os.path.exists(mix_path):
return gs_err_code_mix_err, None
# 编码为mp3
output_dir = os.path.join(self.work_dir, self.cid + "_out")
if not os.path.exists(output_dir):
os.makedirs(output_dir)
name = str(mix_path).replace("_replace442_mix.wav", "_replace442_mix.mp3").split("/")[-1]
mix_path_mp3 = os.path.join(output_dir, name)
cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(mix_path, mix_path_mp3)
os.system(cmd)
if not os.path.exists(mix_path_mp3):
return gs_err_code_mix_transcode_err, None
# 拷贝src到output_dir
# shutil.copyfile(self.src_mp3, os.path.join(output_dir, "src.mp3"))
# logging.info("after_process sp = {}".format(time.time() - st))
return gs_err_code_success, mix_path_mp3
####################################新对外接口############################################################
def prepare_env(self, cid, work_dir, create_dir=False):
self.cid = cid
self.work_dir = work_dir
# 所有不对外交付的,全部放到这里
self.cache_dir = os.path.join(work_dir, "cache")
if create_dir:
if os.path.exists(self.cache_dir):
shutil.rmtree(self.cache_dir)
os.makedirs(self.cache_dir)
self.src_mp3 = os.path.join(self.work_dir, "src.mp3")
if not os.path.exists(self.src_mp3):
return gs_err_code_no_src_mp3
self.vocal_path = os.path.join(self.cache_dir, "vocal.wav")
self.vocal_32_path = os.path.join(self.cache_dir, "vocal_32.wav")
self.acc_path = os.path.join(self.cache_dir, "acc.wav")
return gs_err_code_success
def generate_svc_file(self, cid, work_dir):
"""
:param cid:
:param work_dir:
:return:err_code, 生成出的svc的文件名称
"""
err = self.prepare_env(cid, work_dir, create_dir=True)
if err != gs_err_code_success:
return err, None
# 音源分离
if not os.path.exists(self.vocal_32_path):
st = time.time()
err = self.separate(cid, self.src_mp3, self.vocal_path, self.acc_path)
logging.info("cid={},separate,sp={}".format(self.cid, time.time() - st))
if err != gs_err_code_success:
return err, None
# 生成svc,只保留一个最佳的
st = time.time()
out_songs = self.inference_speaker()
if len(out_songs) == 0:
return gs_err_code_no_good_choice, None
logging.info("cid={},inference_speaker,{},sp={}".format(self.cid, out_songs[0], time.time() - st))
return gs_err_code_success, out_songs[0]
def effect(self, cid, work_dir, svc_file):
st = time.time()
err = self.prepare_env(cid, work_dir)
if err != gs_err_code_success:
return err, None
logging.info("cid={},effect_and_mix,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 预处理人声
dv_out_list = self.preprocess_vocal([svc_file], self.vocal_path)
if len(dv_out_list) == 0:
return gs_err_code_preprocess_vocal, None
svc_file = dv_out_list[0]
# 做音效
st = time.time()
err, effect_file = self.reverb_by_vocal(svc_file)
if err != gs_err_code_success:
return err, None
logging.info("cid={},reverb_by_vocal,{},sp={}".format(self.cid, svc_file, time.time() - st))
return err, effect_file
def mix(self, cid, work_dir, svc_file, effect_file):
"""
做音效以及合并
:param cid:
:param work_dir:
:param svc_file:
:param effect_file:
:return: err_code, 完成的mp3文件
"""
st = time.time()
err = self.prepare_env(cid, work_dir)
if err != gs_err_code_success:
return err, None
logging.info("cid={},effect_and_mix,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 拉伸
st = time.time()
volume_path = str(effect_file).replace(".wav", "_dv.wav")
cmd = "{} {} {} {}".format(gs_draw_volume_exe, effect_file, self.vocal_path, volume_path)
os.system(cmd)
if not os.path.exists(volume_path):
print("{} {} ERROR draw volume".format(self.cid, volume_path))
return gs_err_code_volume_err, None
effect_file = volume_path
logging.info("cid={},draw_volume,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 替换
st = time.time()
try:
if self.replace_vocal_frame_inst is None:
self.replace_vocal_frame_inst = ReplaceVocalFrame("data/models/split_dirty_frame_v5_3_epoch3_852.pth")
self.replace_vocal_frame_inst.process(svc_file, effect_file, self.vocal_path)
except Exception as ex:
logging.info("{},replace_vocal_frame_inst, {}", self.cid, ex)
return gs_err_code_replace_except_err, None
dst_path = effect_file + "_replace.wav"
if not os.path.exists(dst_path):
return gs_err_code_replace_err, None
logging.info("cid={},replace_vocal_frame_inst,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 转码
st = time.time()
dst_path_442 = dst_path.replace("_replace.wav", "_replace442.wav")
cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} -loglevel fatal".format(dst_path, dst_path_442)
os.system(cmd)
if not os.path.exists(dst_path_442):
return gs_err_code_replace_trans_err, None
logging.info("cid={},transcode,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 合并转码后再做一次拉伸,保证响度
st = time.time()
volume_path = str(dst_path_442).replace("_replace442.wav", "_replace442_dv.wav")
cmd = "{} {} {} {}".format(gs_draw_volume_exe, dst_path_442, self.vocal_path, volume_path)
os.system(cmd)
if not os.path.exists(volume_path):
print("{} {} ERROR draw volume".format(self.cid, volume_path))
return gs_err_code_volume_err, None
dst_path_442 = volume_path
logging.info("cid={},draw_volume2,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 混合
st = time.time()
mix_path = dst_path_442.replace("_replace442_dv.wav", "_replace442_dv_mix.wav")
cmd = "{} {} {} {}".format(gs_simple_mixer_path, dst_path_442, self.acc_path, mix_path)
os.system(cmd)
if not os.path.exists(mix_path):
return gs_err_code_mix_err, None
logging.info("cid={},mixer,{},sp={}".format(self.cid, svc_file, time.time() - st))
# 编码为mp3
st = time.time()
output_dir = os.path.join(self.work_dir, self.cid + "_out")
if not os.path.exists(output_dir):
os.makedirs(output_dir)
name = str(mix_path).replace("_replace442_dv_mix.wav", "_replace442_dv_mix.mp3").split("/")[-1]
mix_path_mp3 = os.path.join(output_dir, name)
cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(mix_path, mix_path_mp3)
print(cmd)
os.system(cmd)
if not os.path.exists(mix_path_mp3):
return gs_err_code_mix_transcode_err, None
logging.info("cid={},encode,{},sp={}".format(self.cid, svc_file, time.time() - st))
return gs_err_code_success, mix_path_mp3
def get_gender(self, svc_file):
return self.speakers2gender[int(os.path.basename(svc_file.replace(".wav", "")).split("_")[1])]
def process_one_logic(self, cid, work_dir):
"""
搞成两部分:
1. 分离数据+5次推理,获取最佳结果,并保存
2. 利用最佳结果做音效以及合并
:return:
"""
err, svc_file = self.generate_svc_file(cid, work_dir)
gender = -1
if err != gs_err_code_success:
return err, svc_file, gender,
gender = self.get_gender(svc_file)
err, effect_file = self.effect(cid, work_dir, svc_file)
if err != gs_err_code_success:
return err, svc_file, gender
err, mix_mp3_path = self.mix(cid, work_dir, svc_file, effect_file)
return err, mix_mp3_path, gender
def test():
arr = [
# "611752105020343687",
# "611752105023532439",
# "611752105030419688",
# "611752105030485748",
# "611752105030485685",
"dzq",
]
base_dir = "/data/rsync/jianli.yang/AutoCoverTool/data/test"
s_inst = SongCoverInference()
for cid in arr:
st = time.time()
# err, mix_mp3, gender = s_inst.process_one(cid, os.path.join(base_dir, cid), False)
err, mix_mp3, gender = s_inst.process_one_logic(cid, os.path.join(base_dir, cid))
print(mix_mp3, gender)
print("cid={} RealFinish err={} sp={}".format(cid, err, time.time() - st))
+def test_gene_svc():
+ base_dir = "/data/rsync/jianli.yang/AutoCoverTool/data/test"
+ cid = "clean"
+ work_dir = os.path.join(base_dir, cid)
+ st = time.time()
+ speaker = "1125899914308640"
+ speakers_model_path = "data/train_users/{}/logs/32k/G_2000.pth"
+ speakers_model_config = "data/train_users/{}/config/config.json"
+ model_path = speakers_model_path.format(speaker)
+ config_path = speakers_model_config.format(speaker)
+
+ # 缓存目录:
+ cache_dir = os.path.join(work_dir, "cache")
+ if os.path.exists(cache_dir):
+ shutil.rmtree(cache_dir)
+ os.makedirs(cache_dir)
+ song_path = os.path.join(cache_dir, "{}_{}.wav".format(cid, speaker))
+ vocal_path = os.path.join(work_dir, "vocal_32.wav")
+ inf(model_path, config_path, vocal_path, song_path, "prod")
+ print("finish....")
+
+
if __name__ == '__main__':
test()
diff --git a/AutoCoverTool/online/tone_shift_one.py b/AutoCoverTool/online/tone_shift_one.py
index 2abe3e0..64f6236 100644
--- a/AutoCoverTool/online/tone_shift_one.py
+++ b/AutoCoverTool/online/tone_shift_one.py
@@ -1,173 +1,216 @@
"""
变调的方式做处理
1. 下载
2. 分离
3. 针对于人声变调+2,伴奏+1
4. 合成
"""
import os
+import json
import shutil
+import librosa
import logging
+import numpy as np
from ref.music_remover.separate_interface import SeparateInterface
from online.inference_worker import upload_file2cos, gs_state_use, gs_state_finish, gs_state_default
from online.common import *
logging.basicConfig(filename='/tmp/tone_shift_one.log', level=logging.INFO)
gs_tone_shift_exe = "/opt/soft/bin/tone_shift_exe"
gs_simple_mixer_path = "/opt/soft/bin/simple_mixer"
gs_err_code_success = 0
gs_err_code_tone_shift = 1
gs_err_code_mix = 2
gs_err_code_transcode = 3
gs_err_code_upload = 4
gs_err_code_download = 5
gs_err_code_trans_to_mp3 = 6
gs_err_code_separate = 7
+gs_err_code_duration_too_long = 8
+gs_err_code_duration_no_vocal = 9
+
+
+def exec_cmd(cmd):
+ r = os.popen(cmd)
+ text = r.read()
+ r.close()
+ return text
+
+
+def get_d(audio_path):
+ cmd = "ffprobe -v quiet -print_format json -show_format -show_streams {}".format(audio_path)
+ data = exec_cmd(cmd)
+ data = json.loads(data)
+ # 返回秒
+ return float(data["format"]["duration"])
+
+
+def get_mean_power(audio_path):
+ sr = 44100
+ audio, sr = librosa.load(audio_path, sr=sr, mono=True)
+ mm = np.mean(np.abs(audio))
+ return mm
class ToneShift:
def __init__(self):
self.separate_inst = SeparateInterface()
def update_state(self, song_id, state):
sql = "update svc_queue_table set state={},update_time={} where song_id = {}". \
format(state, int(time.time()), song_id)
banned_user_map['db'] = "av_db"
update_db(sql, banned_user_map)
def get_one_data(self):
sql = "select song_id, url from svc_queue_table where state = 0 and song_src=3 order by create_time desc limit 1"
banned_user_map["db"] = "av_db"
data = get_data_by_mysql(sql, banned_user_map)
if len(data) == 0:
return None, None
song_id, song_url = data[0]
if song_id != "":
self.update_state(song_id, gs_state_use)
return str(song_id), song_url
def pre_process(self, work_dir, song_url):
"""
创建文件夹,下载数据
:return:
"""
ext = str(song_url).split(".")[-1]
dst_file = "{}/src_origin.{}".format(work_dir, ext)
cmd = "wget {} -O {}".format(song_url, dst_file)
print(cmd)
os.system(cmd)
if not os.path.exists(dst_file):
return gs_err_code_download
+
+ duration = get_d(dst_file)
+ print("Duration:", dst_file, duration)
+ if duration > 20 * 60:
+ return gs_err_code_duration_too_long
+
dst_mp3_file = "{}/src.mp3".format(work_dir)
cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} ".format(dst_file, dst_mp3_file)
os.system(cmd)
if not os.path.exists(dst_mp3_file):
return gs_err_code_trans_to_mp3
return gs_err_code_success
def tone_shift_one(self, in_file, dst_file, pitch):
cmd = "{} {} {} {}".format(gs_tone_shift_exe, in_file, dst_file, pitch)
os.system(cmd)
return os.path.exists(dst_file)
def mix(self, cid, vocal_path, acc_path, tp):
if tp == 1:
vocal_pitch = 2
acc_pitch = 0
else:
vocal_pitch = -2
acc_pitch = 0
vocal_path_2 = vocal_path.replace(".wav", "_{}.wav".format(vocal_pitch))
acc_path_2 = acc_path.replace(".wav", "_{}.wav".format(acc_pitch))
err = self.tone_shift_one(vocal_path, vocal_path_2, vocal_pitch)
if not err:
return gs_err_code_tone_shift, None
err = self.tone_shift_one(acc_path, acc_path_2, acc_pitch)
if not err:
return gs_err_code_tone_shift, None
base_dir = os.path.dirname(vocal_path)
mix_path = "{}/mix_{}_{}.wav".format(base_dir, vocal_pitch, acc_pitch)
cmd = "{} {} {} {}".format(gs_simple_mixer_path, vocal_path_2, acc_path_2, mix_path)
print("exec_cmd={}".format(cmd))
os.system(cmd)
if not os.path.exists(mix_path):
return gs_err_code_mix, None
# 转码
mix_path_mp3 = mix_path.replace(".wav", ".mp3")
cmd = "ffmpeg -i {} -ab 320k -y {} -loglevel fatal".format(mix_path, mix_path_mp3)
os.system(cmd)
if not os.path.exists(mix_path_mp3):
return gs_err_code_transcode, None
# 上传到cos
mix_name = os.path.basename(mix_path_mp3)
key = "av_res/svc_res_tone_shift/{}/{}".format(str(cid), mix_name)
if not upload_file2cos(key, mix_path_mp3):
return gs_err_code_upload, None
return gs_err_code_success, key
def process_one(self, cid, work_dir):
"""
:param cid:
:param work_dir:
:return:
"""
src_mp3 = os.path.join(work_dir, "src.mp3")
vocal_path = os.path.join(work_dir, "vocal.wav")
acc_path = os.path.join(work_dir, "acc.wav")
if not self.separate_inst.process(cid, src_mp3, vocal_path, acc_path):
return gs_err_code_separate, []
if not os.path.exists(vocal_path) or not os.path.exists(acc_path):
return gs_err_code_separate, []
+ # 当人声的平均能量小于一定值时,则认为无人声(0.01是经验值判定,样本分析来看)
+ # 无人声的样本[0.0056, 0.0003], 有人声的样本(目前最小)[0.046, 0.049]
+ print("power:{},{}".format(cid, get_mean_power(vocal_path)))
+ if get_mean_power(vocal_path) < 0.02:
+ return gs_err_code_duration_no_vocal, []
err, type1_mix_mp3 = self.mix(cid, vocal_path, acc_path, 1)
if err != gs_err_code_success:
return err, []
err, type2_mix_mp3 = self.mix(cid, vocal_path, acc_path, 2)
if err != gs_err_code_success:
return err, []
return gs_err_code_success, [type1_mix_mp3, type2_mix_mp3]
def process_worker(self):
logging.info("start process_worker .....")
base_dir = "/tmp/tone_shift_one"
if not os.path.exists(base_dir):
os.makedirs(base_dir)
while True:
worker_st = time.time()
cid, song_url = self.get_one_data()
+ if cid is None:
+ time.sleep(5)
+ logging.info("get one data is None ...")
+ continue
+
work_dir = os.path.join(base_dir, str(cid))
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
err = self.pre_process(work_dir, song_url)
if err != gs_err_code_success:
return err
st = time.time()
err, data = self.process_one(str(cid), work_dir)
logging.info("process_finish,{},{}".format(cid, time.time() - st))
if err == gs_err_code_success and len(data) == 2:
sql = "update svc_queue_table set state={},update_time={},svc_url=\"{}\" where song_id = {}". \
format(gs_state_finish, int(time.time()), ",".join(data), str(cid))
banned_user_map['db'] = "av_db"
update_db(sql, banned_user_map)
else:
self.update_state(str(cid), -err)
shutil.rmtree(work_dir)
logging.info("process_finish,{},{}".format(cid, time.time() - worker_st))
if __name__ == '__main__':
ts = ToneShift()
ts.process_worker()
diff --git a/AutoCoverTool/ref/tools/mixer/tone_shift.cpp b/AutoCoverTool/ref/tools/mixer/tone_shift.cpp
index 7e4e410..afbefab 100644
--- a/AutoCoverTool/ref/tools/mixer/tone_shift.cpp
+++ b/AutoCoverTool/ref/tools/mixer/tone_shift.cpp
@@ -1,245 +1,250 @@
//
// Created by yangjianli on 2019-09-09.
//
/**
* 输入一个音频和伴奏自动进行混合
* gated_loudness 当前音量
* gain 预期增益
*/
#include "iostream"
#include "WaveFile.h"
#include "math.h"
#include "ebur128.h"
#include "AudioMixer.h"
#include "alimiter.h"
#include "waves/inc/WaveFile.h"
#include "CAudioEffectsChainApi.h"
#include "string"
#include "ae_server/CAeServer.h"
#include <cstdio>
#include <chrono>
#include <iostream>
#include <cstdlib>
#include <sys/time.h>
#include "denoise/webrtc/include/WebrtcDenoise.h"
#define PROC_LEN 1024
#define DEFAULT_BASELINE_DB (float)-14.57f
int short2float(short *pInBuf, int nLen, float *pOutBuf)
{
for (int i = 0; i < nLen; i++)
{
pOutBuf[i] = pInBuf[i] * 1.0 / 32768;
}
return 0;
}
int float2short(float *pInBuf, int nLen, short *pOutBuf)
{
for (int i = 0; i < nLen; i++)
{
pOutBuf[i] = int(pInBuf[i] * 32768);
}
return 0;
}
/**
* 获取增益
* @param nChannel
* @param nSampleRate
* @param pData
* @param nLength
* @param gain
* @return
*/
int ebur128_whole(int nChannel, int nSampleRate, short *pData, const int nLength, double &gated_loudness, double &gain)
{
printf("ebur128_init start .. %d\n", nLength);
ebur128_state *st = NULL;
st = ebur128_init(nChannel, nSampleRate, EBUR128_MODE_I);
if (NULL == st)
{
return -1;
}
int nPos = 0;
int nTmpLength = 0;
int nRet;
printf("process start ..\n");
while (nPos < nLength)
{
nTmpLength = PROC_LEN;
if (nLength - nPos < PROC_LEN)
{
nTmpLength = nLength - nPos;
}
nRet = ebur128_add_frames_short(st, pData + nPos, nTmpLength / nChannel);
if (nRet != 0)
{
return -2;
}
nPos += nTmpLength;
}
printf("process ok..\n");
gated_loudness = -1;
ebur128_loudness_global(st, &gated_loudness);
float db = (DEFAULT_BASELINE_DB - gated_loudness) / 20.f;
gain = pow(10, db);
printf("gated_loudness = %f db = %f gain = %f\n", gated_loudness, db, gain);
ebur128_destroy(&st);
return 0;
}
/**
* 混合音频和伴奏
* @param pVocalIn
* @param pAccIn
* @param nLength
* @param gainVocal
* @param gainAcc
* @param pOutBuf
* @return
*/
int mix(float *pVocalIn, float *pAccIn, int nLength, double gainVocal, double gainAcc, float *pOutBuf,
int nSampleRate, int nChannel, int nDelay, std::string effect_file)
{
CAudioMixer *cAudioMixer = new CAudioMixer();
cAudioMixer->init(nSampleRate, nChannel);
cAudioMixer->set_acc_delay(nDelay);
cAudioMixer->set_vocal_volume(int(gainVocal * 50));
cAudioMixer->set_acc_volume(int(gainAcc * 50));
int nPos = 0;
int nStep = 1024;
float *fTmp = new float[nStep];
cAudioMixer->reset();
nPos = 0;
nStep = 1024;
int cnt = 0;
CAeServer cAeServer;
cAeServer.init(nSampleRate, nChannel, nStep / nChannel);
AE_PARAMS_IM_EFFECT im_params = {
.effect_path = effect_file,
};
cAeServer.set_params(AE_TYPE_IM_EFFECT, (void *) &im_params);
while (nPos < nLength)
{
if (nLength - nPos < nStep)
{
nStep = nLength - nPos;
}
cnt++;
cAeServer.process(pVocalIn + nPos, pVocalIn + nPos, nStep);
cAudioMixer->process(pVocalIn + nPos, pAccIn + nPos, pOutBuf + nPos, nStep);
nPos += nStep;
}
cAeServer.uninit();
delete cAudioMixer;
delete[] fTmp;
return 0;
}
int denoise_webrtc(short *pInBuf, int nLength, int nChannel, int nSampleRate)
{
CWebrtcDenoise cWebrtcDenoise;
cWebrtcDenoise.init(nSampleRate, nChannel);
float *pTmp = new float[nLength];
for (int i = 0; i < nLength; i++)
{
pTmp[i] = pInBuf[i] * 1.0 / 32768;
}
cWebrtcDenoise.set_level(kHigh);
int nStep = 512 * nChannel;
for (int i = 0; i < nStep; i++)
{
pTmp[i] = pTmp[i] * i * 1.0 / nStep;
}
for (int i = 0, cnt = 0; i < nLength; i += nStep, cnt++)
{
if (nLength - i < nStep) continue;
cWebrtcDenoise.process(pTmp + i, nStep);
}
for (int i = 0; i < nLength; i++)
{
pInBuf[i] = short(pTmp[i] * 32768);
}
delete[] pTmp;
return 0;
}
double calc_power_rate(float *in_data, int32_t in_len, float *ref_data, int32_t ref_len)
{
double in_power = 0;
double ref_power = 0;
int32_t min_len = in_len > ref_len ? ref_len : in_len;
for (int i = 0; i < min_len; i++)
{
in_power += (in_data[i]) * (in_data[i]);
ref_power += (ref_data[i]) * (ref_data[i]);
}
return ref_power / in_power;
}
int main(int argc, char *argv[])
{
if (argc != 4)
{
printf("input error! example: ./main vocal_path dst_path pitch!\n");
return -1;
}
std::string vocal_path = argv[1];
std::string dst_path = argv[2];
float pitch = strtod(argv[3], NULL);
// 读取人声
CWaveFile *oWaveFile = new CWaveFile(vocal_path.c_str(), false);
float *pVocalBuf = new float[oWaveFile->GetTotalFrames() * oWaveFile->GetChannels()];
oWaveFile->ReadFrameAsfloat(pVocalBuf, oWaveFile->GetTotalFrames());
int nStep = 1024;
int nLength = oWaveFile->GetTotalFrames() * oWaveFile->GetChannels();
CAeServer cAeServer;
cAeServer.init(oWaveFile->GetSampleRate(), oWaveFile->GetChannels(), nStep / oWaveFile->GetChannels());
AEToneShiftParam ae_param;
ae_param.max_shift = 12;
ae_param.min_shift = -12;
ae_param.tone_shift = pitch;
cAeServer.set_params(AE_TYPE_TONE_SHIFT, &ae_param);
int nPos = 0;
while (nPos < nLength)
{
if (nLength - nPos < nStep)
{
nStep = nLength - nPos;
}
cAeServer.process(pVocalBuf + nPos, pVocalBuf + nPos, nStep);
nPos += nStep;
}
+
+ // 剔除84ms延迟
+ int latency_pos = int(cAeServer.get_latency_ms() * oWaveFile->GetSampleRate() / 1000.0) * oWaveFile->GetChannels();
+ printf("latency_pos=%d\n", latency_pos);
cAeServer.uninit();
+
//写入文件
printf("write2file nLength:%d path:%s!\n", oWaveFile->GetTotalFrames() * oWaveFile->GetChannels(),
dst_path.c_str());
CWaveFile out_wav = CWaveFile(dst_path.c_str(), true);
out_wav.SetChannels(oWaveFile->GetChannels());
out_wav.SetSampleRate(oWaveFile->GetSampleRate());
out_wav.SetSampleFormat(SF_IEEE_FLOAT);
out_wav.SetupDone();
- out_wav.WriteFrame(pVocalBuf, oWaveFile->GetTotalFrames());
+ out_wav.WriteFrame(pVocalBuf+latency_pos, oWaveFile->GetTotalFrames());
delete oWaveFile;
delete[] pVocalBuf;
return 0;
}
\ No newline at end of file
diff --git a/AutoCoverTool/script/get_song_url.py b/AutoCoverTool/script/get_song_url.py
index 1e5314f..b3b5541 100644
--- a/AutoCoverTool/script/get_song_url.py
+++ b/AutoCoverTool/script/get_song_url.py
@@ -1,280 +1,761 @@
"""
获取歌曲的地址
# song_src=2 是来源108和109的歌曲,未被洗过的
# song_src=1 是曲库给的
# song_src=3 # 用于轻变调的
"""
from script.common import *
from copy import deepcopy
from online.common import update_db
def get_url_by_song_id(song_id):
sql = "select task_url,starmaker_songid from silence where starmaker_songid = {} order by task_id limit 1".format(
song_id)
ban = deepcopy(banned_user_map)
ban["db"] = "starmaker_musicbook"
data = get_data_by_mysql(sql, ban)
if len(data) > 0:
return data[0][0]
return None
def process():
arr = [
- "611752105030484885",
- "611752105029543722",
- "611752105030556608",
- "611752105030585154",
- "611752105030556609",
- "611752105029054060",
- "611752105028975148",
- "611752105030558172",
- "611752105028778344",
- "611752105030556613",
- "611752105029290698",
- "611752105030556605",
- "611752105027484924",
- "611752105030559472",
- "611752105030534293",
- "611752105027148644",
- "611752105029292630",
- "611752105026900917",
- "611752105027103140",
- "611752105030589795",
- "611752105026915170",
- "611752105030534289",
- "611752105026751742",
- "611752105026452638",
- "611752105025979421",
- "611752105025817810",
- "611752105026536899",
- "611752105030534282",
- "611752105030534285",
- "611752105030559474",
- "611752105025219762",
- "611752105025034426",
- "611752105024938926",
- "611752105029648740",
- "611752105029675859",
- "611752105024598727",
- "611752105030548412",
- "611752105030487271",
- "611752105029648743",
- "611752105023692976",
- "611752105024135802",
- "611752105023616288",
- "611752105023255629",
- "611752105022728286",
- "611752105023206033",
- "611752105023091102",
- "611752105029792918",
- "611752105022729259",
- "611752105030487512",
- "611752105022842120",
- "611752105022842054",
- "611752105022785621",
- "611752105022840550",
- "611752105022838205",
- "611752105022839189",
- "611752105022835751",
- "611752105022818025",
- "611752105022797521",
- "611752105022784390",
- "611752105028820609",
- "611752105030488595",
- "611752105030517536",
- "611752105030501857",
- "611752105030478339",
- "611752105025957389",
- "611752105027484925",
- "611752105027484915",
- "611752105024415490",
- "611752105027854244",
- "611752105029527187",
- "611752105028870536",
- "611752105028444597",
- "611752105028778353",
- "611752105027877846",
- "611752105028906605",
- "611752105027781526",
- "611752105027877887",
- "611752105027795229",
- "611752105027734187",
- "611752105028820612",
- "611752105027626964",
- "611752105027460080",
- "611752105027507932",
- "611752105027611342",
- "611752105027435127",
- "611752105027307631",
- "611752105029648514",
- "611752105026874730",
- "611752105030591117",
- "611752105026437853",
- "611752105025541483",
- "611752105026536913",
- "611752105022647044",
- "611752105023440333",
- "611752105023460357",
- "611752105023604729",
- "611752105023510939",
- "611752105022842387",
- "611752105024230229",
- "611752105023674599",
- "611752105023160140",
- "611752105022647074",
- "611752105022615220",
- "611752105028408822",
- "611752105022816170",
- "611752105022772279",
- "611752105022614618",
- "611752105020417684",
- "611752105020382477",
- "611752105022780345",
- "611752105022780961",
- "611752105022837186",
- "611752105022778042",
- "611752105022775939",
- "611752105022764224",
- "611752105022781267",
- "611752105022839030",
- "611752105022767294",
- "611752105022784996",
- "611752105022775600",
- "611752105022780284",
- "611752105022768837",
- "611752105030590847",
- "611752105022780965",
- "611752105022779020",
- "611752105022777496",
- "611752105022781268",
- "611752105022785681",
- "611752105022779294",
- "611752105022823781",
- "611752105022780210",
- "611752105022774220",
- "611752105022768419",
- "611752105030590845",
- "611752105022835406",
- "611752105022774040",
- "611752105022783776",
- "611752105022781193",
- "611752105020390942",
- "611752105022783967",
- "611752105022763051",
- "611752105022780818",
- "611752105022835415",
- "611752105022782935",
- "611752105020402448",
- "611752105022781011",
- "611752105020384960",
- "611752105022779784",
- "611752105022781387",
- "611752105025580424",
- "611752105022765022",
- "611752105025492732",
- "611752105023683356",
- "611752105022842241",
- "611752105024231227",
- "611752105029291290",
- "611752105023104185",
- "611752105025565044",
- "611752105025458749",
- "611752105025458753",
- "611752105025090763",
- "611752105030590839",
- "611752105030534180",
- "611752105023908922",
- "611752105027326105",
- "611752105023725727",
- "611752105022647079",
- "611752105024082232",
- "611752105029648891",
- "611752105025504662",
- "611752105025496983",
- "611752105026716551",
- "611752105029648872",
- "611752105022614531",
- "611752105029041707",
- "611752105030483313",
- "611752105023219237",
- "611752105022842989",
- "611752105022746733",
- "611752105023162802",
- "611752105022729263",
- "611752105022777120",
- "611752105025584544",
- "611752105025458809",
- "611752105027648113",
- "611752105030590840",
- "611752105024183682",
- "611752105023086347",
- "611752105022839975",
- "611752105025348359",
- "611752105022781144",
- "611752105022647060",
- "611752105022728482",
- "611752105025840622",
- "611752105022836470",
- "611752105023246015",
- "611752105022838206",
- "611752105022780355",
- "611752105022768062",
- "611752105022777600"
+ "611752105016642206",
+ "611752105016665828",
+ "611752105020332340",
+ "611752105020332347",
+ "611752105020390931",
+ "611752105020417679",
+ "611752105021442334",
+ "611752105021459528",
+ "611752105021810110",
+ "611752105021916843",
+ "611752105022312180",
+ "611752105022614734",
+ "611752105022615541",
+ "611752105022615665",
+ "611752105022616931",
+ "611752105022647066",
+ "611752105022647087",
+ "611752105022652047",
+ "611752105022700847",
+ "611752105022728649",
+ "611752105022728653",
+ "611752105022729255",
+ "611752105022733605",
+ "611752105022736029",
+ "611752105022740011",
+ "611752105022742166",
+ "611752105022743986",
+ "611752105022746848",
+ "611752105022748944",
+ "611752105022749272",
+ "611752105022749768",
+ "611752105022751902",
+ "611752105022752248",
+ "611752105022754267",
+ "611752105022754702",
+ "611752105022755405",
+ "611752105022757577",
+ "611752105022758309",
+ "611752105022758395",
+ "611752105022761145",
+ "611752105022761151",
+ "611752105022764435",
+ "611752105022764965",
+ "611752105022766106",
+ "611752105022766675",
+ "611752105022774258",
+ "611752105022776257",
+ "611752105022776285",
+ "611752105022776577",
+ "611752105022776846",
+ "611752105022777151",
+ "611752105022777306",
+ "611752105022778977",
+ "611752105022779055",
+ "611752105022779169",
+ "611752105022779962",
+ "611752105022780648",
+ "611752105022784727",
+ "611752105022785048",
+ "611752105022785179",
+ "611752105022785551",
+ "611752105022811718",
+ "611752105022814368",
+ "611752105022815931",
+ "611752105022819519",
+ "611752105022824948",
+ "611752105022828998",
+ "611752105022833822",
+ "611752105022835250",
+ "611752105022838357",
+ "611752105022838544",
+ "611752105022838589",
+ "611752105022838666",
+ "611752105022839015",
+ "611752105022839300",
+ "611752105022839468",
+ "611752105022839559",
+ "611752105022892354",
+ "611752105022911042",
+ "611752105023134539",
+ "611752105023142842",
+ "611752105023588204",
+ "611752105023588294",
+ "611752105024204862",
+ "611752105024546859",
+ "611752105024598735",
+ "611752105024608149",
+ "611752105024728134",
+ "611752105024938931",
+ "611752105025198792",
+ "611752105025327479",
+ "611752105025458721",
+ "611752105025458732",
+ "611752105025458759",
+ "611752105025458792",
+ "611752105025458796",
+ "611752105025458831",
+ "611752105025534832",
+ "611752105025538466",
+ "611752105025565027",
+ "611752105025584548",
+ "611752105025586093",
+ "611752105025720331",
+ "611752105025741198",
+ "611752105025741447",
+ "611752105025817802",
+ "611752105025835130",
+ "611752105025879265",
+ "611752105026152312",
+ "611752105026205732",
+ "611752105026281560",
+ "611752105026343284",
+ "611752105026388268",
+ "611752105026421148",
+ "611752105026433966",
+ "611752105026536897",
+ "611752105026536911",
+ "611752105026580839",
+ "611752105026648945",
+ "611752105026663363",
+ "611752105026736866",
+ "611752105027067863",
+ "611752105027112518",
+ "611752105027186556",
+ "611752105027189208",
+ "611752105027189301",
+ "611752105027326104",
+ "611752105027460089",
+ "611752105027484913",
+ "611752105027588072",
+ "611752105027611383",
+ "611752105027690075",
+ "611752105028183260",
+ "611752105028470803",
+ "611752105028507652",
+ "611752105028528335",
+ "611752105028809597",
+ "611752105028815367",
+ "611752105028820629",
+ "611752105028820633",
+ "611752105028827057",
+ "611752105028837845",
+ "611752105028878340",
+ "611752105028906600",
+ "611752105028944645",
+ "611752105028958744",
+ "611752105029006303",
+ "611752105029006319",
+ "611752105029059923",
+ "611752105029078388",
+ "611752105029090034",
+ "611752105029209546",
+ "611752105029243449",
+ "611752105029272970",
+ "611752105029291291",
+ "611752105029291294",
+ "611752105029291295",
+ "611752105029291297",
+ "611752105029291298",
+ "611752105029291304",
+ "611752105029395411",
+ "611752105029432787",
+ "611752105029570149",
+ "611752105029570153",
+ "611752105029570157",
+ "611752105029648873",
+ "611752105029648877",
+ "611752105029648879",
+ "611752105029953987",
+ "611752105029954853",
+ "611752105029955024",
+ "611752105029955258",
+ "611752105029956379",
+ "611752105029956615",
+ "611752105029990162",
+ "611752105029990590",
+ "611752105029991249",
+ "611752105030103635",
+ "611752105030119229",
+ "611752105030124600",
+ "611752105030483296",
+ "611752105030485000",
+ "611752105030485417",
+ "611752105030485428",
+ "611752105030485533",
+ "611752105030485561",
+ "611752105030485562",
+ "611752105030485565",
+ "611752105030485566",
+ "611752105030485569",
+ "611752105030485570",
+ "611752105030485572",
+ "611752105030485591",
+ "611752105030485592",
+ "611752105030485594",
+ "611752105030485595",
+ "611752105030485597",
+ "611752105030485598",
+ "611752105030485601",
+ "611752105030485602",
+ "611752105030485607",
+ "611752105030485608",
+ "611752105030485610",
+ "611752105030485612",
+ "611752105030485613",
+ "611752105030485616",
+ "611752105030485618",
+ "611752105030485620",
+ "611752105030485621",
+ "611752105030485626",
+ "611752105030485627",
+ "611752105030485631",
+ "611752105030485634",
+ "611752105030485637",
+ "611752105030485639",
+ "611752105030485642",
+ "611752105030485646",
+ "611752105030485650",
+ "611752105030485653",
+ "611752105030485655",
+ "611752105030485656",
+ "611752105030485662",
+ "611752105030485663",
+ "611752105030485666",
+ "611752105030485667",
+ "611752105030485669",
+ "611752105030485671",
+ "611752105030485672",
+ "611752105030485673",
+ "611752105030485676",
+ "611752105030485677",
+ "611752105030485679",
+ "611752105030485681",
+ "611752105030485682",
+ "611752105030485685",
+ "611752105030485687",
+ "611752105030485688",
+ "611752105030485691",
+ "611752105030485692",
+ "611752105030485693",
+ "611752105030485696",
+ "611752105030485697",
+ "611752105030485700",
+ "611752105030485702",
+ "611752105030485703",
+ "611752105030485707",
+ "611752105030485710",
+ "611752105030485711",
+ "611752105030485715",
+ "611752105030485716",
+ "611752105030485717",
+ "611752105030485721",
+ "611752105030485722",
+ "611752105030485724",
+ "611752105030485726",
+ "611752105030485727",
+ "611752105030485729",
+ "611752105030485731",
+ "611752105030485733",
+ "611752105030485734",
+ "611752105030485736",
+ "611752105030485737",
+ "611752105030485738",
+ "611752105030485739",
+ "611752105030485741",
+ "611752105030485742",
+ "611752105030485744",
+ "611752105030485745",
+ "611752105030485748",
+ "611752105030485749",
+ "611752105030485750",
+ "611752105030485755",
+ "611752105030485758",
+ "611752105030485759",
+ "611752105030485761",
+ "611752105030485763",
+ "611752105030485766",
+ "611752105030485768",
+ "611752105030485769",
+ "611752105030485772",
+ "611752105030485778",
+ "611752105030485779",
+ "611752105030485787",
+ "611752105030485790",
+ "611752105030485791",
+ "611752105030485794",
+ "611752105030485797",
+ "611752105030485799",
+ "611752105030488510",
+ "611752105030488594",
+ "611752105030488665",
+ "611752105030488713",
+ "611752105030488727",
+ "611752105030488744",
+ "611752105030488814",
+ "611752105030488836",
+ "611752105030488852",
+ "611752105030488864",
+ "611752105030488880",
+ "611752105030488962",
+ "611752105030488997",
+ "611752105030489153",
+ "611752105030489354",
+ "611752105030489380",
+ "611752105030489394",
+ "611752105030489403",
+ "611752105030489415",
+ "611752105030562265",
+ "611752105030562283",
+ "611752105030562308",
+ "611752105030562320",
+ "611752105030562341",
+ "611752105030562353",
+ "611752105030562357",
+ "611752105030562371",
+ "611752105030562377",
+ "611752105030562386",
+ "611752105030562451",
+ "611752105030562466",
+ "611752105030562487",
+ "611752105030562933",
+ "611752105030562937",
+ "611752105030562946",
+ "611752105030563002",
+ "611752105030563012",
+ "611752105030563018",
+ "611752105030563020",
+ "611752105030563117",
+ "611752105030563422",
+ "611752105030563435",
+ "611752105030563469",
+ "611752105030563484",
+ "611752105030563555",
+ "611752105030563570",
+ "611752105030563584",
+ "611752105030563620",
+ "611752105030563722",
+ "611752105030563730",
+ "611752105030563754",
+ "611752105030590082",
+ "611752105030590145",
+ "611752105030590214",
+ "611752105030590224",
+ "611752105030590259",
+ "611752105030590265",
+ "611752105030590274",
+ "611752105030590490",
+ "611752105030590649",
+ "611752105030590659",
+ "611752105030590671",
+ "611752105030590689",
+ "611752105030590706",
+ "611752105030590735",
+ "611752105030590750",
+ "611752105030590757",
+ "611752105030590939",
+ "611752105030590951",
+ "611752105030590977",
+ "611752105030590993",
+ "611752105030591011",
+ "611752105030591021",
+ "611752105030591053",
+ "611752105030591070",
+ "611752105030591083",
+ "611752105030591094",
+ "611752105030591105",
+ "611752105030591162",
+ "611752105030591181",
+ "611752105030591184",
+ "611752105030591185",
+ "611752105030591187",
+ "611752105030591191",
+ "611752105030591193",
+ "611752105030591195",
+ "611752105030591198",
+ "611752105030591200",
+ "611752105030591203",
+ "611752105030591204",
+ "611752105030591209",
+ "611752105030591213",
+ "611752105030591214",
+ "611752105030591215",
+ "611752105030591221",
+ "611752105030593095",
+ "611752105030593133",
+ "611752105030593144",
+ "611752105030593155",
+ "611752105030593181",
+ "611752105030593200",
+ "611752105030593208",
+ "611752105030593346",
+ "611752105030593368",
+ "611752105030593384",
+ "611752105030593396",
+ "611752105030593416",
+ "611752105030593432",
+ "611752105030593445",
+ "611752105030593465",
+ "611752105030593481",
+ "611752105030593496",
+ "611752105030593509",
+ "611752105030593569",
+ "611752105030593588",
+ "611752105030593597",
+ "611752105030593606",
+ "611752105030593618",
+ "611752105030593633",
+ "611752105030593645",
+ "611752105030593658",
+ "611752105030593671",
+ "611752105030593685",
+ "611752105030593936",
+ "611752105030593955",
+ "611752105030593962",
+ "611752105030593970",
+ "611752105030593976",
+ "611752105030593985",
+ "611752105030593998",
+ "611752105030594008",
+ "611752105030594018",
+ "611752105030594023",
+ "611752105030594041",
+ "611752105030594045",
+ "611752105030594053",
+ "611752105030594070",
+ "611752105030594075",
+ "611752105030594081",
+ "611752105030594088",
+ "611752105030594099",
+ "611752105030594107",
+ "611752105030594117",
+ "611752105030594121",
+ "611752105030594128",
+ "611752105030594137",
+ "611752105030594148",
+ "611752105030594158",
+ "611752105030594164",
+ "611752105030594171",
+ "611752105030594176",
+ "611752105030594192",
+ "611752105030594200",
+ "611752105030594208",
+ "611752105030594221",
+ "611752105030594256",
+ "611752105030594262",
+ "611752105030594275",
+ "611752105030594286",
+ "611752105030594292",
+ "611752105030594296",
+ "611752105030594299",
+ "611752105030594311",
+ "611752105030594313",
+ "611752105030594319",
+ "611752105030594325",
+ "611752105030594330",
+ "611752105030594336",
+ "611752105030594343",
+ "611752105030594348",
+ "611752105030594376",
+ "611752105030594384",
+ "611752105030595120",
+ "611752105030595128",
+ "611752105030595200",
+ "611752105030595219",
+ "611752105030595343",
+ "611752105030595375",
+ "611752105030595385",
+ "611752105030595393",
+ "611752105030595411",
+ "611752105030595429",
+ "611752105030595472",
+ "611752105030595495",
+ "611752105030595505",
+ "611752105030595532",
+ "611752105030595544",
+ "611752105030595555",
+ "611752105030595563",
+ "611752105030595587",
+ "611752105030595596",
+ "611752105030595628",
+ "611752105030595640",
+ "611752105030595654",
+ "611752105030595661",
+ "611752105030595673",
+ "611752105030595714",
+ "611752105030595730",
+ "611752105030595740",
+ "611752105030595774",
+ "611752105030595812",
+ "611752105030595820",
+ "611752105030595829",
+ "611752105030595910",
+ "611752105030595941",
+ "611752105030595983",
+ "611752105030596002",
+ "611752105030596035",
+ "611752105030596046",
+ "611752105030596057",
+ "611752105030596072",
+ "611752105030596082",
+ "611752105030596128",
+ "611752105030596137",
+ "611752105030596373",
+ "611752105030596416",
+ "611752105030596423",
+ "611752105030596432",
+ "611752105030596435",
+ "611752105030596439",
+ "611752105030596444",
+ "611752105030596449",
+ "611752105030596468",
+ "611752105030596478",
+ "611752105030596479",
+ "611752105030596484",
+ "611752105030596500",
+ "611752105030596502",
+ "611752105030596509",
+ "611752105030596513",
+ "611752105030596523",
+ "611752105030596535",
+ "611752105030596536",
+ "611752105030596560",
+ "611752105030596570",
+ "611752105030596576",
+ "611752105030596584",
+ "611752105030596585",
+ "611752105030596593",
+ "611752105030596600",
+ "611752105030596612",
+ "611752105030596621",
+ "611752105030596681",
+ "611752105030596696",
+ "611752105030596716",
+ "611752105030596759",
+ "611752105030596782",
+ "611752105030596790",
+ "611752105030596806",
+ "611752105030596826",
+ "611752105030596827",
+ "611752105030596830",
+ "611752105030596831",
+ "611752105030596833",
+ "611752105030596842",
+ "611752105030596852",
+ "611752105030596854",
+ "611752105030596864",
+ "611752105030596871",
+ "611752105030596875",
+ "611752105030596879",
+ "611752105030596883",
+ "611752105030596887",
+ "611752105030596888",
+ "611752105030596895",
+ "611752105030596896",
+ "611752105030596899",
+ "611752105030596903",
+ "611752105030596906",
+ "611752105030596913",
+ "611752105030596919",
+ "611752105030596920",
+ "611752105030596924",
+ "611752105030596925",
+ "611752105030596931",
+ "611752105030596934",
+ "611752105030596935",
+ "611752105030596939",
+ "611752105030596949",
+ "611752105030596952",
+ "611752105030596956",
+ "611752105030596965",
+ "611752105030596970",
+ "611752105030596973",
+ "611752105030596975",
+ "611752105030596978",
+ "611752105030596985",
+ "611752105030596994",
+ "611752105030596998",
+ "611752105030596999",
+ "611752105030597001",
+ "611752105030597006",
+ "611752105030597009",
+ "611752105030597013",
+ "611752105030597015",
+ "611752105030597018",
+ "611752105030597025",
+ "611752105030597028",
+ "611752105030597031",
+ "611752105030597032",
+ "611752105030597038",
+ "611752105030597044",
+ "611752105030597049",
+ "611752105030597055",
+ "611752105030597058",
+ "611752105030597063",
+ "611752105030597072",
+ "611752105030597083",
+ "611752105030597091",
+ "611752105030597099",
+ "611752105030597104",
+ "611752105030597109",
+ "611752105030597112",
+ "611752105030597115",
+ "611752105030597120",
+ "611752105030597132",
+ "611752105030597136",
+ "611752105030597141",
+ "611752105030597144",
+ "611752105030597149",
+ "611752105030597154",
+ "611752105030597159",
+ "611752105030597161",
+ "611752105030597168",
+ "611752105030597170",
+ "611752105030597177",
+ "611752105030597181",
+ "611752105030597186",
+ "611752105030597187",
+ "611752105030597191",
+ "611752105030597192",
+ "611752105030597194",
+ "611752105030597198",
+ "611752105030597204",
+ "611752105030597205",
+ "611752105030597208",
+ "611752105030597212",
+ "611752105030597213",
+ "611752105030597218",
+ "611752105030597221",
+ "611752105030597225",
+ "611752105030597229",
+ "611752105030597230",
+ "611752105030597233",
+ "611752105030597236",
+ "611752105030597238",
+ "611752105030597241",
+ "611752105030597246",
+ "611752105030597248",
+ "611752105030597250",
+ "611752105030597253",
+ "611752105030597254",
+ "611752105030597257",
+ "611752105030597260",
+ "611752105030597261",
+ "611752105030597264",
+ "611752105030597265",
+ "611752105030597266",
+ "611752105030597269",
+ "611752105030597271",
+ "611752105030597275",
+ "611752105030597281",
+ "611752105030597295",
+ "611752105030597299",
+ "611752105030597306",
+ "611752105030597318",
+ "611752105030597327",
+ "611752105030597328",
+ "611752105030597330",
+ "611752105030597334",
+ "611752105030597339",
+ "611752105030597347",
+ "611752105030597348",
+ "611752105030597356",
+ "611752105030597359",
+ "611752105030597372",
+ "611752105030597385",
+ "611752105030597417",
+ "611752105030597424",
+ "611752105030597443",
+ "611752105030597447",
+ "611752105030597450",
+ "611752105030597456",
+ "611752105030597461",
+ "611752105030597468",
+ "611752105030597476",
+ "611752105030597505",
+ "611752105030598550",
+ "611752105030598562",
+ "611752105030598634",
+ "611752105030598642",
+ "611752105030598665"
]
ban = deepcopy(banned_user_map)
ban["db"] = "av_db"
for sid in arr:
url = get_url_by_song_id(sid)
if url is not None:
print("out,{},{}".format(url, sid))
# 不在数据库中
sql = "select song_id from svc_queue_table where song_id={}".format(sid)
- data = get_data_by_mysql(sql, ban)
- if len(data) == 0:
- tm = int(time.time())
- sql = "insert INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\",{}, {}, 1)" \
- .format(sid, url, tm, tm)
- update_db(sql, ban)
+ # data = get_data_by_mysql(sql, ban)
+ # if len(data) == 0:
+ tm = int(time.time())
+ sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\",{}, {}, 3)" \
+ .format(sid, url, tm, tm)
+ update_db(sql, ban)
def get_data_from_song():
sql = """
select tb1.song_id, tb1.recording_count
from (
select song_id,recording_count
from starmaker.song
where song_src in (108,109) and song_status = 2
order by recording_count desc
) as tb1
left join
(
select song_id
from av_db.svc_queue_table
) as tb2
on tb1.song_id = tb2.song_id
where tb2.song_id is null
- order by tb1.recording_count desc limit 400
+ order by tb1.recording_count desc limit 1000
"""
ban = deepcopy(banned_user_map)
ban_v1 = deepcopy(banned_user_map)
ban["db"] = "starmaker_musicbook"
ban_v1["db"] = "av_db"
data = get_data_by_mysql(sql, ban)
for dt in data:
sid = dt[0]
url = get_url_by_song_id(sid)
if url is not None:
print("out,{},{}".format(url, sid))
- sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time) VALUES ({}, \"{}\", NOW(), NOW())" \
- .format(sid, url)
+ tm = int(time.time())
+ sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\", {}, {}, 3)" \
+ .format(sid, url, tm, tm)
update_db(sql, ban_v1)
if __name__ == '__main__':
# get_data_from_song()
process()

File Metadata

Mime Type
text/x-diff
Expires
Sun, Jan 12, 08:30 (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1347154
Default Alt Text
(74 KB)

Event Timeline