首先拿到的数据为样本id 特征数据 以及 样本id 和对应的label两个文档
先把libsvm的特征处理成xgboost能够使用的格式,并按8:1:1的比例将数据分成训练集、验证集以及测试集。
#train id:0-42622
#valid id:42623-47957
#test id:47958-53285
def transdata(feature_file_path,group_file_path,out_feature_path_train,out_feature_path_vaild,out_feature_path_test):
output_feature_train = open(out_feature_path_train,"w")
output_feature_valid = open(out_feature_path_vaild, "w")
output_feature_test = open(out_feature_path_test, "w")
with open (feature_file_path) as features, open(group_file_path) as groups:
for x,y in zip(features,groups):
splits_x = x.strip().split(" ")
splits_y = y.strip().split(" ")
if int(splits_x[0])<42623:
output_feature_train.write(splits_y[0]+" "+splits_x[1]+"\n")
if int(splits_x[0])>42622 and int(splits_x[0])<47958:
output_feature_valid.write(splits_y[0]+" "+splits_x[1]+"\n")
if int(splits_x[0])>47957:
output_feature_test.write(splits_y[0]+" "+splits_x[1]+"\n")
output_feature_train.close()
output_feature_valid.close()
output_feature_test.close()
if __name__ =="__main__":
transdata("topic_features.txt","gt.txt","libsvm_format.train.txt","libsvm_format.valid.txt","libsvm_format.test.txt")
对label进行分组
#train id:0-42622
#valid id:42623-47957
#test id:47958-53285
def transdata(group_file_path,out_feature_path_train,out_feature_path_vaild,out_feature_path_test):
output_feature_train = open(out_feature_path_train,"w")
output_feature_valid = open(out_feature_path_vaild, "w")
output_feature_test = open(out_feature_path_test, "w")
groups = open(group_file_path)
for line in groups:
if not line:
break
splits_x = line.strip().split(" ")
if int(splits_x[1]) < 42623:
output_feature_train.write(line)
if int(splits_x[1]) > 42622 and int(splits_x[0]) < 47958:
output_feature_valid.write(line)
if int(splits_x[1]) > 47957:
output_feature_test.write(line)
output_feature_train.close()
output_feature_valid.close()
output_feature_test.close()
if __name__ =="__main__":
transdata("gt.txt","gt.train.txt","gt.valid.txt","gt.test.txt")
随即根据qid对数据进行分组,生成.group文件
#train id:0-42622
#valid id:42623-47957
#test id:47958-53285
def transgroup(group_file_path,save_path):
group_output = open(save_path,"w")
group_file = open(group_file_path)
group = ""
group_data = []
for line in group_file:
if not line:
break
splits = line.strip().split(" ")
if splits[2]!=group:
group_output.write(str(len(group_data))+"\n")
group_data = []
group = splits[2]
group_data.append(splits[0])
group_output.write(str(len(group_data)) + "\n")
group_output.close()
group_file.close()
if __name__ =="__main__":
transgroup("gt.test.txt","group.test.txt")
objective参数的解释
`rank:pairwise`: Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized
`rank:ndcg`: Use LambdaMART to perform list-wise ranking where [Normalized Discounted Cumulative Gain (NDCG)](http://en.wikipedia.org/wiki/NDCG) is maximized
数据处理完成后,送入xgboost进行训练
#!/usr/bin/python
import xgboost as xgb
from xgboost import DMatrix
from sklearn.datasets import load_svmlight_file
# This script demonstrate how to do ranking with xgboost.train
x_train, y_train = load_svmlight_file("libsvm_format.train.txt")
x_valid, y_valid = load_svmlight_file("libsvm_format.valid.txt")
x_test, y_test = load_svmlight_file("libsvm_format.test.txt")
group_train = []
with open("group.train.txt", "r") as f:
data = f.readlines()
for line in data:
group_train.append(int(line.split("\n")[0]))
group_valid = []
with open("group.valid.txt", "r") as f:
data = f.readlines()
for line in data:
group_valid.append(int(line.split("\n")[0]))
group_test = []
with open("group.test.txt", "r") as f:
data = f.readlines()
for line in data:
group_test.append(int(line.split("\n")[0]))
train_dmatrix = DMatrix(x_train, y_train)
valid_dmatrix = DMatrix(x_valid, y_valid)
test_dmatrix = DMatrix(x_test)
train_dmatrix.set_group(group_train)
valid_dmatrix.set_group(group_valid)
params = {'objective': 'rank:pairwise', 'eta': 0.1, 'gamma': 1.0,
'min_child_weight': 0.1, 'max_depth': 6}
xgb_model = xgb.train(params, train_dmatrix, num_boost_round=4,
evals=[(valid_dmatrix, 'validation')])
pred = xgb_model.predict(test_dmatrix)
参考代码1:https://www.jianshu.com/p/9caef967ec0a
参考代码2: https://github.com/dmlc/xgboost/blob/master/demo/rank/rank.py
网友评论