Page MenuHomePhabricator
Files F4880299

No OneTemporary
Actions

View Options

diff --git a/AutoCoverTool/ref/online/voice_class_online.py b/AutoCoverTool/ref/online/voice_class_online.py
index 6041c94..eab43a7 100644
--- a/AutoCoverTool/ref/online/voice_class_online.py
+++ b/AutoCoverTool/ref/online/voice_class_online.py
@@ -1,420 +1,420 @@
"""
男女声分类在线工具
1 转码为16bit单声道
2 均衡化
3 模型分类
"""
import os
import sys
import librosa
import shutil
import logging
import time
import torch.nn.functional as F
import numpy as np
from model import *
# from common import bind_kernel
logging.basicConfig(level=logging.INFO)
os.environ["LRU_CACHE_CAPACITY"] = "1"
# torch.set_num_threads(1)
# bind_kernel(1)
"""
临时用一下,全局使用的变量
"""
transcode_time = 0
vb_time = 0
mfcc_time = 0
predict_time = 0
"""
错误码
"""
ERR_CODE_SUCCESS = 0 # 处理成功
ERR_CODE_NO_FILE = -1 # 文件不存在
ERR_CODE_TRANSCODE = -2 # 转码失败
ERR_CODE_VOLUME_BALANCED = -3 # 均衡化失败
ERR_CODE_FEATURE_TOO_SHORT = -4 # 特征文件太短
"""
常量
"""
FRAME_LEN = 128
MFCC_LEN = 80
-EBUR128_BIN = "/opt/soft/bin/standard_audio_no_cut"
+EBUR128_BIN = "/data/gpu_env_common/res/av_svc/bin/standard_audio_no_cut"
# EBUR128_BIN = "/Users/yangjianli/linux/opt/soft/bin/standard_audio_no_cut"
GENDER_FEMALE = 0
GENDER_MALE = 1
GENDER_OTHER = 2
"""
通用函数
"""
def exec_cmd(cmd):
ret = os.system(cmd)
if ret != 0:
return False
return True
"""
业务需要的函数
"""
def get_one_mfcc(file_url):
st = time.time()
data, sr = librosa.load(file_url, sr=16000)
if len(data) < 512:
return []
mfcc = librosa.feature.mfcc(y=data, sr=sr, n_fft=512, hop_length=256, n_mfcc=MFCC_LEN)
mfcc = mfcc.transpose()
print("get_one_mfcc:spend_time={}".format(time.time() - st))
global mfcc_time
mfcc_time += time.time() - st
return mfcc
def volume_balanced(src, dst):
st = time.time()
cmd = "{} {} {}".format(EBUR128_BIN, src, dst)
logging.info(cmd)
exec_cmd(cmd)
if not os.path.exists(dst):
logging.error("volume_balanced:cmd={}".format(cmd))
print("volume_balanced:spend_time={}".format(time.time() - st))
global vb_time
vb_time += time.time() - st
return os.path.exists(dst)
def transcode(src, dst):
st = time.time()
cmd = "ffmpeg -loglevel quiet -i {} -ar 16000 -ac 1 {}".format(src, dst)
logging.info(cmd)
exec_cmd(cmd)
if not os.path.exists(dst):
logging.error("transcode:cmd={}".format(cmd))
print("transcode:spend_time={}".format(time.time() - st))
global transcode_time
transcode_time += time.time() - st
return os.path.exists(dst)
class VoiceClass:
def __init__(self, music_voice_pure_model, music_voice_no_pure_model, gender_pure_model, gender_no_pure_model):
"""
四个模型
:param music_voice_pure_model: 分辨纯净人声/其他
:param music_voice_no_pure_model: 分辨有人声/其他
:param gender_pure_model: 纯净人声分辨男女
:param gender_no_pure_model: 有人声分辨男女
"""
st = time.time()
self.device = "cpu"
self.batch_size = 256
self.music_voice_pure_model = load_model(MusicVoiceV5Model, music_voice_pure_model, self.device)
self.music_voice_no_pure_model = load_model(MusicVoiceV5Model, music_voice_no_pure_model, self.device)
self.gender_pure_model = load_model(MobileNetV2Gender, gender_pure_model, self.device)
self.gender_no_pure_model = load_model(MobileNetV2Gender, gender_no_pure_model, self.device)
logging.info("load model ok ! spend_time={}".format(time.time() - st))
def batch_predict(self, model, features):
st = time.time()
scores = []
with torch.no_grad():
for i in range(0, len(features), self.batch_size):
cur_data = features[i:i + self.batch_size].to(self.device)
predicts = model(cur_data)
predicts_score = F.softmax(predicts, dim=1)
scores.extend(predicts_score.cpu().numpy())
ret = np.array(scores)
global predict_time
predict_time += time.time() - st
return ret
def predict_pure(self, filename, features):
scores = self.batch_predict(self.music_voice_pure_model, features)
new_features = []
for idx, score in enumerate(scores):
if score[0] > 0.5: # 非人声
continue
new_features.append(features[idx].numpy())
# 人声段太少,不能进行处理
# 参数可以改
new_feature_len = len(new_features)
new_feature_rate = len(new_features) / len(features)
if new_feature_len < 4 or new_feature_rate < 0.4:
logging.warning(
"filename={}|predict_pure|other|len={}|rate={}".format(filename, new_feature_len, new_feature_rate)
)
return GENDER_OTHER, -1
new_features = torch.from_numpy(np.array(new_features))
scores = self.batch_predict(self.gender_pure_model, new_features)
f_avg = sum(scores[:, 0]) / len(scores)
m_avg = sum(scores[:, 1]) / len(scores)
female_rate = f_avg / (f_avg + m_avg)
if female_rate > 0.65:
return GENDER_FEMALE, female_rate
if female_rate < 0.12:
return GENDER_MALE, female_rate
logging.warning(
"filename={}|predict_pure|other|len={}|rate={}".format(filename, new_feature_len, new_feature_rate)
)
return GENDER_OTHER, female_rate
def predict_no_pure(self, filename, features):
scores = self.batch_predict(self.music_voice_no_pure_model, features)
new_features = []
for idx, score in enumerate(scores):
if score[0] > 0.5: # 非人声
continue
new_features.append(features[idx].numpy())
# 人声段太少,不能进行处理
# 参数可以改
new_feature_len = len(new_features)
new_feature_rate = len(new_features) / len(features)
if new_feature_len < 4 or new_feature_rate < 0.4:
logging.warning(
"filename={}|predict_no_pure|other|len={}|rate={}".format(filename, new_feature_len, new_feature_rate)
)
return GENDER_OTHER, -1
new_features = torch.from_numpy(np.array(new_features))
scores = self.batch_predict(self.gender_no_pure_model, new_features)
f_avg = sum(scores[:, 0]) / len(scores)
m_avg = sum(scores[:, 1]) / len(scores)
female_rate = f_avg / (f_avg + m_avg)
if female_rate > 0.75:
return GENDER_FEMALE, female_rate
if female_rate < 0.1:
return GENDER_MALE, female_rate
logging.warning(
"filename={}|predict_no_pure|other|len={}|rate={}".format(filename, new_feature_len, new_feature_rate)
)
return GENDER_OTHER, female_rate
def predict(self, filename, features):
st = time.time()
new_features = []
for i in range(FRAME_LEN, len(features), FRAME_LEN):
new_features.append(features[i - FRAME_LEN: i])
new_features = torch.from_numpy(np.array(new_features))
gender, rate = self.predict_pure(filename, new_features)
if gender == GENDER_OTHER:
logging.info("start no pure process...")
return self.predict_no_pure(filename, new_features)
print("predict|spend_time={}".format(time.time() - st))
return gender, rate
def process_one_logic(self, filename, file_path, cache_dir):
tmp_wav = os.path.join(cache_dir, "tmp.wav")
tmp_vb_wav = os.path.join(cache_dir, "tmp_vb.wav")
if not transcode(file_path, tmp_wav):
return ERR_CODE_TRANSCODE
if not volume_balanced(tmp_wav, tmp_vb_wav):
return ERR_CODE_VOLUME_BALANCED
features = get_one_mfcc(tmp_vb_wav)
if len(features) < FRAME_LEN:
logging.error("feature too short|file_path={}".format(file_path))
return ERR_CODE_FEATURE_TOO_SHORT
return self.predict(filename, features)
def process_one(self, file_path):
base_dir = os.path.dirname(file_path)
filename = os.path.splitext(file_path)[0]
cache_dir = os.path.join(base_dir, filename + "_cache")
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
ret = self.process_one_logic(filename, file_path, cache_dir)
shutil.rmtree(cache_dir)
return ret
def process(self, file_path):
gender, female_rate = self.process_one(file_path)
logging.info("{}|gender={}|female_rate={}".format(file_path, gender, female_rate))
return gender, female_rate
def process_by_feature(self, feature_file):
"""
直接处理特征文件
:param feature_file:
:return:
"""
filename = os.path.splitext(feature_file)[0]
features = np.load(feature_file)
gender, female_rate = self.predict(filename, features)
return gender, female_rate
def test_all_feature():
import glob
base_dir = "/data/datasets/music_voice_dataset_full/feature_online_data_v3"
female = glob.glob(os.path.join(base_dir, "female/*feature.npy"))
male = glob.glob(os.path.join(base_dir, "male/*feature.npy"))
other = glob.glob(os.path.join(base_dir, "other/*feature.npy"))
model_path = "/data/jianli.yang/voice_classification/online/models"
music_voice_pure_model = os.path.join(model_path, "voice_005_rec_v5.pth")
music_voice_no_pure_model = os.path.join(model_path, "voice_10_v5.pth")
gender_pure_model = os.path.join(model_path, "gender_8k_ratev5_v6_adam.pth")
gender_no_pure_model = os.path.join(model_path, "gender_8k_v6_adam.pth")
vc = VoiceClass(music_voice_pure_model, music_voice_no_pure_model, gender_pure_model, gender_no_pure_model)
tot_st = time.time()
ret_map = {
0: {0: 0, 1: 0, 2: 0},
1: {0: 0, 1: 0, 2: 0},
2: {0: 0, 1: 0, 2: 0}
}
for file in female:
st = time.time()
print("------------------------------>>>>>")
gender, female_score = vc.process_by_feature(file)
ret_map[0][gender] += 1
if gender != 0:
print("err:female->{}|{}|{}".format(gender, file, female_score))
print("process|spend_tm=={}".format(time.time() - st))
for file in male:
st = time.time()
print("------------------------------>>>>>")
gender, female_score = vc.process_by_feature(file)
ret_map[1][gender] += 1
if gender != 1:
print("err:male->{}|{}|{}".format(gender, file, female_score))
print("process|spend_tm=={}".format(time.time() - st))
for file in other:
st = time.time()
print("------------------------------>>>>>")
gender, female_score = vc.process_by_feature(file)
ret_map[2][gender] += 1
if gender != 2:
print("err:other->{}|{}|{}".format(gender, file, female_score))
print("process|spend_tm=={}".format(time.time() - st))
global transcode_time, vb_time, mfcc_time, predict_time
print("spend_time:tot={}|transcode={}|vb={}|gen_feature={}|predict={}".format(time.time() - tot_st, transcode_time,
vb_time, mfcc_time, predict_time))
f_f = ret_map[0][0]
f_m = ret_map[0][1]
f_o = ret_map[0][2]
m_f = ret_map[1][0]
m_m = ret_map[1][1]
m_o = ret_map[1][2]
o_f = ret_map[2][0]
o_m = ret_map[2][1]
o_o = ret_map[2][2]
print("ff:{},fm:{},fo:{}".format(f_f, f_m, f_o))
print("mm:{},mf:{},mo:{}".format(m_m, m_f, m_o))
print("om:{},of:{},oo:{}".format(o_m, o_f, o_o))
# 女性准确率和召回率
f_acc = f_f / (f_f + m_f + o_f)
f_recall = f_f / (f_f + f_m + f_o)
# 男性准确率和召回率
m_acc = m_m / (m_m + f_m + o_m)
m_recall = m_m / (m_m + m_f + m_o)
print("female: acc={}|recall={}".format(f_acc, f_recall))
print("male: acc={}|recall={}".format(m_acc, m_recall))
def test_all():
import glob
base_dir = "/data/datasets/music_voice_dataset_full/online_data_v3_top200"
female = glob.glob(os.path.join(base_dir, "female/*mp4"))
male = glob.glob(os.path.join(base_dir, "male/*mp4"))
other = glob.glob(os.path.join(base_dir, "other/*mp4"))
model_path = "/data/jianli.yang/voice_classification/online/models"
music_voice_pure_model = os.path.join(model_path, "voice_005_rec_v5.pth")
music_voice_no_pure_model = os.path.join(model_path, "voice_10_v5.pth")
gender_pure_model = os.path.join(model_path, "gender_8k_ratev5_v6_adam.pth")
gender_no_pure_model = os.path.join(model_path, "gender_8k_v6_adam.pth")
vc = VoiceClass(music_voice_pure_model, music_voice_no_pure_model, gender_pure_model, gender_no_pure_model)
tot_st = time.time()
ret_map = {
0: {0: 0, 1: 0, 2: 0},
1: {0: 0, 1: 0, 2: 0},
2: {0: 0, 1: 0, 2: 0}
}
for file in female:
st = time.time()
print("------------------------------>>>>>")
gender, female_score = vc.process(file)
ret_map[0][gender] += 1
if gender != 0:
print("err:female->{}|{}|{}".format(gender, file, female_score))
print("process|spend_tm=={}".format(time.time() - st))
for file in male:
st = time.time()
print("------------------------------>>>>>")
gender, female_score = vc.process(file)
ret_map[1][gender] += 1
if gender != 1:
print("err:male->{}|{}|{}".format(gender, file, female_score))
print("process|spend_tm=={}".format(time.time() - st))
for file in other:
st = time.time()
print("------------------------------>>>>>")
gender, female_score = vc.process(file)
ret_map[2][gender] += 1
if gender != 2:
print("err:other->{}|{}|{}".format(gender, file, female_score))
print("process|spend_tm=={}".format(time.time() - st))
global transcode_time, vb_time, mfcc_time, predict_time
print("spend_time:tot={}|transcode={}|vb={}|gen_feature={}|predict={}".format(time.time() - tot_st, transcode_time,
vb_time, mfcc_time, predict_time))
f_f = ret_map[0][0]
f_m = ret_map[0][1]
f_o = ret_map[0][2]
m_f = ret_map[1][0]
m_m = ret_map[1][1]
m_o = ret_map[1][2]
o_f = ret_map[2][0]
o_m = ret_map[2][1]
o_o = ret_map[2][2]
print("ff:{},fm:{},fo:{}".format(f_f, f_m, f_o))
print("mm:{},mf:{},mo:{}".format(m_m, m_f, m_o))
print("om:{},of:{},oo:{}".format(o_m, o_f, o_o))
# 女性准确率和召回率
f_acc = f_f / (f_f + m_f + o_f)
f_recall = f_f / (f_f + f_m + f_o)
# 男性准确率和召回率
m_acc = m_m / (m_m + f_m + o_m)
m_recall = m_m / (m_m + m_f + m_o)
print("female: acc={}|recall={}".format(f_acc, f_recall))
print("male: acc={}|recall={}".format(m_acc, m_recall))
if __name__ == "__main__":
# test_all()
# test_all_feature()
model_path = sys.argv[1]
voice_path = sys.argv[2]
music_voice_pure_model = os.path.join(model_path, "voice_005_rec_v5.pth")
music_voice_no_pure_model = os.path.join(model_path, "voice_10_v5.pth")
gender_pure_model = os.path.join(model_path, "gender_8k_ratev5_v6_adam.pth")
gender_no_pure_model = os.path.join(model_path, "gender_8k_v6_adam.pth")
vc = VoiceClass(music_voice_pure_model, music_voice_no_pure_model, gender_pure_model, gender_no_pure_model)
for i in range(0, 1):
st = time.time()
print("------------------------------>>>>>")
vc.process(voice_path)
print("process|spend_tm=={}".format(time.time() - st))
diff --git a/AutoCoverTool/script/get_song_url.py b/AutoCoverTool/script/get_song_url.py
index 2635982..9b42cdf 100644
--- a/AutoCoverTool/script/get_song_url.py
+++ b/AutoCoverTool/script/get_song_url.py
@@ -1,546 +1,128 @@
"""
获取歌曲的地址
# song_src=2 是来源108和109的歌曲，未被洗过的
# song_src=1 是曲库给的
# song_src=3 # 用于轻变调的
"""
from script.common import *
from copy import deepcopy
from online.common import update_db
def get_url_by_song_id(song_id):
sql = "select task_url,starmaker_songid from silence where starmaker_songid = {} order by task_id desc limit 1".format(
song_id)
ban = deepcopy(banned_user_map)
ban["db"] = "starmaker_musicbook"
data = get_data_by_mysql(sql, ban)
if len(data) > 0:
return data[0][0]
return None
def process():
arr = [
- "611752105030534669",
- "611752105030534671",
- "611752105029291860",
- "611752105030534665",
- "611752105030534667",
- "611752105027734182",
- "611752105030532701",
- "611752105027626957",
- "611752105030532696",
- "611752105030517855",
- "611752105025184103",
- "611752105030517856",
- "611752105024164143",
- "611752105030517852",
- "611752105026681421",
- "611752105024571437",
- "611752105022779865",
- "611752105030517845",
- "611752105030493464",
- "611752105030517843",
- "611752105030517018",
- "611752105030517014",
- "611752105022838003",
- "611752105024118499",
- "611752105030517015",
- "611752105030517012",
- "611752105025587378",
- "611752105023644389",
- "611752105023616289",
- "611752105025502433",
- "611752105030517008",
- "611752105024199100",
- "611752105030517003",
- "611752105026614996",
- "611752105029087255",
- "611752105030517518",
- "611752105030517840",
- "611752105030517844",
- "611752105030517859",
- "611752105022777749",
- "611752105030532705",
- "611752105026265975",
- "611752105030532703",
- "611752105030487368",
- "611752105030483712",
- "611752105030517850",
- "611752105022782535",
- "611752105030517021",
- "611752105030517854",
- "611752105023541359",
- "611752105029792918",
- "611752105024199117",
- "611752105029673594",
- "611752105023674624",
- "611752105028990732",
- "611752105028487811",
- "611752105024194923",
- "611752105023763921",
- "611752105023434554",
- "611752105022838184",
- "611752105030553757",
- "611752105027326100",
- "611752105023977089",
- "611752105023674617",
- "611752105023620980",
- "611752105023536537",
- "611752105023301456",
- "611752105022842491",
- "611752105022841266",
- "611752105030548398",
- "611752105022839768",
- "611752105022838572",
- "611752105022781228",
- "611752105029598755",
- "611752105030517531",
- "611752105023541369",
- "611752105023678576",
- "611752105023346238",
- "611752105027648094",
- "611752105027326090",
- "611752105027832575",
- "611752105030478339",
- "611752105027795229",
- "611752105027734187",
- "611752105022614618",
- "611752105023329571",
- "611752105023234496",
- "611752105026134338",
- "611752105030554109",
- "611752105030517489",
- "611752105029836690",
- "611752105030555726",
- "611752105030555723",
- "611752105030553598",
- "611752105030555722",
- "611752105029656131",
- "611752105030555720",
- "611752105023674607",
- "611752105023478557",
- "611752105030555716",
- "611752105030555717",
- "611752105022784360",
- "611752105022836164",
- "611752105029711726",
- "611752105022783553",
- "611752105030555714",
- "611752105022824550",
- "611752105022838413",
- "611752105022783797",
- "611752105022781620",
- "611752105022775924",
- "611752105022774759",
- "611752105022759877",
- "611752105022785234",
- "611752105030555710",
- "611752105030555712",
- "611752105022768451",
- "611752105022767499",
- "611752105030555707",
- "611752105022755443",
- "611752105022777511",
- "611752105022775921",
- "611752105022841864",
- "611752105022766831",
- "611752105022768795",
- "611752105022779062",
- "611752105030555708",
- "611752105030555700",
- "611752105030517183",
- "611752105030555703",
- "611752105030555697",
- "611752105022768285",
- "611752105020419088",
- "611752105022747795",
- "611752105022774521",
- "611752105030555698",
- "611752105030555694",
- "611752105030555696",
- "611752105030555689",
- "611752105022749162",
- "611752105022785417",
- "611752105022784953",
- "611752105022616389",
- "611752105030555690",
- "611752105030555687",
- "611752105022838918",
- "611752105022775551",
- "611752105022728634",
- "611752105022773679",
- "611752105030555686",
- "611752105022758328",
- "611752105030555684",
- "611752105030555681",
- "611752105030555683",
- "611752105022754068",
- "611752105030516960",
- "611752105022764207",
- "611752105022743905",
- "611752105022757953",
- "611752105030516957",
- "611752105030516959",
- "611752105020332899",
- "611752105022841838",
- "611752105029665047",
- "611752105030516953",
- "611752105030490291",
- "611752105022762732",
- "611752105030516955",
- "611752105022762600",
- "611752105022774589",
- "611752105022754286",
- "611752105030516949",
- "611752105022774560",
- "611752105030516946",
- "611752105030487061",
- "611752105030516947",
- "611752105030516933",
- "611752105030516940",
- "611752105030555748",
- "611752105030555742",
- "611752105030555740",
- "611752105030555741",
- "611752105030555737",
- "611752105030553605",
- "611752105030555739",
- "611752105030555733",
- "611752105030555729",
- "611752105030544017",
- "611752105030555727",
- "611752105030486334",
- "611752105030544028",
- "611752105022647043",
- "611752105024402503",
- "611752105022704184",
- "611752105027532732",
- "611752105028858105",
- "611752105027532750",
- "611752105022778279",
- "611752105030517318",
- "611752105026752362",
- "611752105023636284",
- "611752105030517315",
- "611752105022742205",
- "611752105022754485",
- "611752105030517314",
- "611752105023462684",
- "611752105022728585",
- "611752105023751007",
- "611752105030484779",
- "611752105030550189",
- "611752105022729202",
- "611752105030486059",
- "611752105029292588",
- "611752105022647103",
- "611752105027273004",
- "611752105027460081",
- "611752105026900908",
- "611752105022647046",
- "611752105029648513",
- "611752105028032107",
+ "611752105029372457",
+ "611752105029432778",
+ "611752105029443799",
+ "611752105029460983",
+ "611752105029443805",
+ "611752105029395215",
+ "611752105029290706",
+ "611752105029047776",
+ "611752105028820642",
+ "611752105028820632",
+ "611752105027781516",
+ "611752105022617127",
+ "611752105022613728",
+ "611752105028533644",
+ "611752105020286444",
+ "611752105028960419",
+ "611752105020286452",
+ "611752105020417686",
+ "611752105025184112",
"611752105026452639",
"611752105024996266",
- "611752105024728131",
+ "611752105026536915",
+ "611752105025063752",
+ "611752105024356022",
"611752105027832575",
- "611752105030483919",
- "611752105030484774",
- "611752105027228696",
- "611752105022728126",
- "611752105022840114",
- "611752105026982813",
- "611752105022741687",
- "611752105030486054",
- "611752105020352152",
- "611752105020352156",
- "611752105020336897",
- "611752105020256286",
- "611752105030765622",
+ "611752105028820607",
+ "611752105024766043",
+ "611752105027734198",
+ "611752105024766044",
+ "611752105028922725",
+ "611752105024247386",
+ "611752105028820634",
+ "611752105029461031",
+ "611752105029054060",
+ "611752105024938926",
"611752105028820609",
- "611752105030488595",
- "611752105030517536",
- "611752105030501857",
- "611752105030478339",
- "611752105025957389",
- "611752105024415490",
- "611752105027854244",
- "611752105029527187",
- "611752105028444597",
- "611752105027903168",
- "611752105028778353",
- "611752105028906605",
- "611752105027781526",
- "611752105027877887",
- "611752105027795229",
- "611752105027734187",
- "611752105028820612",
- "611752105027626964",
- "611752105027460080",
- "611752105027507932",
- "611752105027611342",
- "611752105027435127",
- "611752105029648514",
- "611752105026874730",
- "611752105030591117",
"611752105025541483",
"611752105026536913",
- "611752105022647044",
- "611752105023434557",
- "611752105023440333",
- "611752105023460357",
- "611752105023510939",
- "611752105022842387",
- "611752105023674599",
- "611752105023160140",
- "611752105022647074",
- "611752105022615220",
- "611752105028408822",
- "611752105022772279",
- "611752105022614618",
- "611752105020417684",
- "611752105024608150",
- "611752105030499232",
- "611752105030485430",
- "611752105023683357",
- "611752105023301455",
- "611752105023458990",
- "611752105027228689",
- "611752105026437878",
+ "611752105027326105",
+ "611752105024628049",
+ "611752105029461035",
+ "611752105024728134",
"611752105027460089",
- "611752105029570157",
- "611752105022700847",
- "611752105029006303",
- "611752105028820629",
- "611752105023134539",
- "611752105022647087",
- "611752105027326104",
- "611752105022652047",
- "611752105022839468",
- "611752105028944645",
- "611752105022911042",
- "611752105020348944",
- "611752105020348945",
- "611752105020332345",
- "611752105027484925",
- "611752105027484915",
- "611752105028870536",
- "611752105027877846",
- "611752105027307631",
- "611752105026437853",
- "611752105023604729",
- "611752105024230229",
- "611752105022816170",
- "611752105028523417",
- "611752105022652046",
- "611752105022782720",
- "611752105024380150",
- "611752105022839949",
- "611752105022761851",
- "611752105022741054",
- "611752105022756250",
- "611752105030533676",
- "611752105022756563",
- "611752105022728598",
- "611752105022760225",
- "611752105025034424",
- "611752105022776389",
- "611752105022767205",
- "611752105030534929",
- "611752105022775308",
- "611752105030502738",
- "611752105022615626",
- "611752105030502742",
- "611752105022741814",
- "611752105029648552",
- "611752105022742136",
- "611752105030486351",
- "611752105022770959",
- "611752105022765477",
- "611752105022751650",
- "611752105022742577",
- "611752105030534505",
- "611752105022973044",
- "611752105029649153",
- "611752105030549651",
- "611752105025494340",
- "611752105020343697",
- "611752105020283852",
- "611752105020373961",
- "611752105022729294",
- "611752105020409111",
- "611752105024938882",
- "611752105029443802",
- "611752105027903154",
- "611752105022614626",
- "611752105020308424",
- "611752105030629613",
- "611752105030534740",
- "611752105030534855",
- "611752105022782724",
- "611752105027781516",
- "611752105029648513",
- "611752105020417686",
- "611752105024996266",
- "611752105024728131",
- "611752105027832575",
- "611752105029646791",
- "611752105027228696",
- "611752105022614749",
- "611752105020336084",
- "611752105026982813",
- "611752105022783586",
- "611752105022741687",
- "611752105023870020",
- "611752105022729203",
- "611752105028143469",
- "611752105030486054",
- "611752105022729411",
- "611752105020336950",
- "611752105020256284",
- "611752105030548045",
- "611752105030503007",
- "611752105029047774",
- "611752105026792339",
- "611752105026449363",
- "611752105026736869",
- "611752105022614727",
- "611752105022615372",
- "611752105022780524",
- "611752105022769594",
- "611752105022758407",
- "611752105022746664",
- "611752105022763120",
- "611752105022745603",
- "611752105030487351",
- "611752105022747491",
- "611752105022728209",
- "611752105022884087",
- "611752105022890433",
- "611752105022741836",
- "611752105022728574",
- "611752105022728612",
- "611752105022739185",
- "611752105030532703",
- "611752105024118493",
- "611752105030535938",
- "611752105030487366",
- "611752105030487368",
- "611752105030487365",
- "611752105022843075",
- "611752105022790159",
- "611752105022778099",
- "611752105022776703",
- "611752105022776364",
- "611752105022774641",
- "611752105022770768",
- "611752105022770226",
- "611752105022769617",
- "611752105022769056",
- "611752105029780685",
- "611752105030477448",
- "611752105022767219",
- "611752105022754490",
- "611752105022760812",
- "611752105030487360",
- "611752105022749100",
- "611752105022728481",
- "611752105022769181",
- "611752105030487358",
- "611752105022739209",
- "611752105022774610",
- "611752105022728721",
- "611752105022741064",
- "611752105022775968",
- "611752105030487354",
- "611752105022771053",
- "611752105022779825",
- "611752105022744563",
- "611752105022744436",
- "611752105030487355",
- "611752105022771161",
- "611752105022748598",
- "611752105022766486",
- "611752105022814952",
- "611752105022728118",
- "611752105022778616",
- "611752105022778275",
- "611752105022614337",
- "611752105022774253",
- "611752105022762324",
- "611752105026299314",
- "611752105022784079",
- "611752105022774583",
- "611752105022770293",
- "611752105030487359",
- "611752105022765790",
- "611752105030535451",
- "611752105030517752",
- "611752105030487362",
- "611752105022775916",
- "611752105022776159",
- "611752105029292588",
- "611752105022838005",
- "611752105023541359",
- "611752105020336965",
- "611752105020293286"
+ "611752105029432787",
+ "611752105028820629"
]
ban = deepcopy(banned_user_map)
ban["db"] = "av_db"
for sid in arr:
url = get_url_by_song_id(sid)
if url is not None:
print("out,{},{}".format(url, sid))
# 只要没有对外输出过，均可以向其中填充
- sql = "select song_id from svc_queue_table where song_id={} and (song_src in (3, 4, 5) and state=2)".format(sid)
+ sql = "select song_id from svc_queue_table where song_id={} and (song_src in (3, 4, 5) and state=2)".format(
+ sid)
data = get_data_by_mysql(sql, ban)
if len(data) == 0:
tm = int(time.time())
- sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\",{}, {}, 4)" \
+ sql = "replace INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\",{}, {}, 5)" \
.format(sid, url, tm, tm)
update_db(sql, ban)
def get_data_from_song():
sql = """
select tb1.song_id, tb1.recording_count
from (
select song_id,recording_count
from starmaker.song
where song_src in (108,109) and song_status = 2
order by recording_count desc
) as tb1
left join
(
select song_id
from av_db.svc_queue_table
) as tb2
on tb1.song_id = tb2.song_id
where tb2.song_id is null
order by tb1.recording_count desc limit 5000
"""
ban = deepcopy(banned_user_map)
ban_v1 = deepcopy(banned_user_map)
ban["db"] = "starmaker_musicbook"
ban_v1["db"] = "av_db"
data = get_data_by_mysql(sql, ban)
for dt in data:
sid = dt[0]
url = get_url_by_song_id(sid)
if url is not None:
print("out,{},{}".format(url, sid))
tm = int(time.time())
sql = "insert INTO svc_queue_table (song_id, url, create_time, update_time, song_src) VALUES ({}, \"{}\", {}, {}, 3)" \
.format(sid, url, tm, tm)
update_db(sql, ban_v1)
if __name__ == '__main__':
# get_diff_song()
# get_data_from_song()
process()

File Metadata

Mime Type
text/x-diff
Expires
Sun, Jan 12, 08:31 (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1347174
Default Alt Text
(34 KB)

Event Timeline