2019-08-09工作进展

昨天工作：
使用类目过滤信息对dssm网络进行finetune，正样本采用的类目过滤之后剩下的数据，负样本采用的是dssm模型认为是正样本但是被类目过滤掉的数据。finetune之后相关性好了很多，关联到的视频有20w左右，相关数据在表：graph_embedding.hs_query_cluster_finetune。

今天计划：
发现结果中还是有非常多的query中的关键词在关联到的title中没有体现出来，但是query中大多数的词都是非常重要的，现在的想法是通过构造query关键词表并在预处理过程中增加其权重的方式改善dssm网络效果。

1. finetune训练集获得

drop table graph_embedding.hs_query_ugc_co_video_final_result_info_4;
yes
create table graph_embedding.hs_query_ugc_co_video_final_result_info_4 as
select distinct a.* from
(select * from graph_embedding.hs_tmp_170 where score > 0.5 )a join (select * from graph_embedding.jl_top_query_related_cate)b on
a.query == b.se_keyword and a.cate_id == b.cate_id;

得到 hs_tmp_170 - hs_query_ugc_co_video_final_result_info_4 作为负样本

drop table graph_embedding.hs_tmp_196;
yes
create table graph_embedding.hs_tmp_196 as
select a.*, b.video_id as video_idb, b.query_id as query_idb from
(select * from graph_embedding.hs_tmp_170 where score > 0.5)a left join (select * from graph_embedding.hs_query_ugc_co_video_final_result_info_4)b on a.video_id == b.video_id and a.query_id == b.query_id;

drop table graph_embedding.hs_tmp_195;
yes
create table graph_embedding.hs_tmp_195 as select video_id, query_id, 0 as label from graph_embedding.hs_tmp_196 where video_idb is NULL and query_idb is NULL;

drop table graph_embedding.hs_tmp_197;
yes
create table hs_tmp_197 as select video_id, query_id, 1 as label from hs_query_ugc_co_video_final_result_info_4;

create table graph_embedding.hs_tmp_198 as
select distinct c.se_keyword_mainse_ws, d.emb as title_mainse_ws, c.label from
(select a.*, b.emb as se_keyword_mainse_ws from (select * from graph_embedding.hs_tmp_197)a join (select * from graph_embedding.hs_tmp_184)b on a.query_id == b.id)c join (select * from graph_embedding.hs_tmp_185)d on c.video_id == d.id;

insert overwrite table graph_embedding.hs_tmp_198 select * from graph_embedding.hs_tmp_198 DISTRIBUTE by random();

pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="finetune_inference_v7.py" -Dcluster='{"worker":{"count":100, "cpu":200, "memory":4000}, "ps":{"count":10, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_tmp_198,odps://graph_embedding/tables/hs_tmp_193,odps://graph_embedding/tables/hs_tmp_187" -Doutputs="odps://graph_embedding/tables/hs_dssm_result_3" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=1 --ckpt=hs_ugc_video_4e_1.ckpt" -DuseSparseClusterSchema=True;

create table hs_tmp_200
as select c.query, d.title, c.score, c.query_id, c.video_id from
(select a.*, b.query from (select * from hs_dssm_result_3)a left join (select * from hs_dssm_dic_query_inf_1)b on a.query_id == b.id)c left join (select * from hs_dssm_dic_title_inf_1)d on c.video_id == d.id;