依赖
<dependency>
<groupId>com.janeluo</groupId>
<artifactId>ikanalyzer</artifactId>
<version>2012_u6</version>
</dependency>
代码
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;
/**
* @author yuyufeng
* @date 2017/11/3
*/
public class Demo {
public static void main(String[] args) {
List<String> keyWords = new ArrayList<>();
int k = 2; //窗口大小/2
float d = 0.85f;
/**
* 标点符号、常用词、以及“名词、动词、形容词、副词之外的词”
*/
Set<String> stopWordSet = new HashSet<String>();
stopWordSet.add("是");
stopWordSet.add("的");
stopWordSet.add("地");
stopWordSet.add("从");
stopWordSet.add("将");
stopWordSet.add("但");
stopWordSet.add("都");
stopWordSet.add("和");
stopWordSet.add("为");
stopWordSet.add("让");
stopWordSet.add("在");
stopWordSet.add("由");
stopWordSet.add("上");
String field = "PageRank近似于一个用户，是指在Internet上随机地单击链接将会到达特定网页的可能性。通常，能够从更多地方到达的网页更为重要，因此具有更高的PageRank。每个到其他网页的链接，都增加了该网页的PageRank。具有较高PageRank的网页一般都是通过更多其他网页的链接而提高的。";
Analyzer analyzer = new IKAnalyzer(true);
TokenStream ts = null;
//分词
try {
ts = analyzer.tokenStream("myfield", new StringReader(field));
OffsetAttribute offset = (OffsetAttribute) ts.addAttribute(OffsetAttribute.class);
CharTermAttribute term = (CharTermAttribute) ts.addAttribute(CharTermAttribute.class);
TypeAttribute type = (TypeAttribute) ts.addAttribute(TypeAttribute.class);
ts.reset();
while (ts.incrementToken()) {
if (!stopWordSet.contains(term.toString())) {
keyWords.add(term.toString());
}
}
ts.end();
} catch (IOException var14) {
var14.printStackTrace();
} finally {
if (ts != null) {
try {
ts.close();
} catch (IOException var13) {
var13.printStackTrace();
}
}
}
Map<String, Set<String>> relationWords = new HashMap<>();
//获取每个关键词 前后k个的组合
for (int i = 0; i < keyWords.size(); i++) {
String keyword = keyWords.get(i);
Set<String> keySets = relationWords.get(keyword);
if (keySets == null) {
keySets = new HashSet<>();
relationWords.put(keyword, keySets);
}
for (int j = i - k; j <= i + k; j++) {
if (j < 0 || j >= keyWords.size() || j == i) {
continue;
} else {
keySets.add(keyWords.get(j));
}
}
}
/* for (String s : relationWords.keySet()) {
System.out.print(s+" ");
for (String s1 : relationWords.get(s)) {
System.out.print(s1+" ");
}
System.out.println();
}*/
Map<String, Float> score = new HashMap<>();
float min_diff = 0.1f; //差值最小
int max_iter = 100;//最大迭代次数
//迭代
for (int i = 0; i < max_iter; i++) {
Map<String, Float> m = new HashMap<>();
float max_diff = 0;
for (String key : relationWords.keySet()) {
Set<String> value = relationWords.get(key);
//先给每个关键词一个默认rank值
m.put(key, 1 - d);
//一个关键词的TextRank由其它成员投票出来
for (String other : value) {
int size = relationWords.get(other).size();
if (key.equals(other) || size == 0) {
continue;
} else {
m.put(key, m.get(key) + d / size * (score.get(other) == null ? 0 : score.get(other)));
}
}
// System.out.println("m.get(key):"+m.get(key)+" score:"+(score.get(key) == null ? 0 : score.get(key)));
max_diff = Math.max(max_diff, Math.abs(m.get(key) - (score.get(key) == null ? 0 : score.get(key))));
}
score = m;
if (max_diff <= min_diff) {
System.out.println("迭代次数：" + i);
break;
}
}
List<Score> scores = new ArrayList<>();
for (String s : score.keySet()) {
Score score1 = new Score();
score1.key = s;
score1.significance = score.get(s);
scores.add(score1);
}
scores.sort(new Comparator<Score>() {
@Override
public int compare(Score o1, Score o2) {
if (o2.significance - o1.significance > 0) {
return 1;
} else {
return -1;
}
}
});
for (Score score1 : scores) {
System.out.println(score1);
}
}
}
class Score {
String key;
float significance;
@Override
public String toString() {
return "关键词=" + key +
", 重要程度=" + significance;
}
}
运行结果
迭代次数：11
关键词=网页, 重要程度=2.8311346
关键词=链接, 重要程度=1.646728
关键词=pagerank, 重要程度=1.6038197
关键词=更多, 重要程度=1.2489531
关键词=到达, 重要程度=1.1083827
关键词=具有, 重要程度=0.98187566
关键词=其他, 重要程度=0.9651773
关键词=用户, 重要程度=0.81595975
关键词=指在, 重要程度=0.8086006
关键词=internet, 重要程度=0.80388165
关键词=一个, 重要程度=0.787644
关键词=随机, 重要程度=0.7764552
关键词=单击, 重要程度=0.76052386
关键词=将会, 重要程度=0.71690917
关键词=能够, 重要程度=0.7066941
关键词=可能性, 重要程度=0.70503104
关键词=更高, 重要程度=0.7045265
关键词=每个, 重要程度=0.7005399
关键词=特定, 重要程度=0.6963727
关键词=通过, 重要程度=0.69495517
关键词=因此, 重要程度=0.69311315
关键词=通常, 重要程度=0.69245243
关键词=该, 重要程度=0.6918771
关键词=一般, 重要程度=0.6895788
关键词=都是, 重要程度=0.686642
关键词=到, 重要程度=0.68152785
关键词=更为重要, 重要程度=0.68064004
关键词=地方, 重要程度=0.6771895
关键词=近似于, 重要程度=0.6137907
关键词=而, 重要程度=0.594995
关键词=增加了, 重要程度=0.5508093
关键词=较高, 重要程度=0.5392841
关键词=提高, 重要程度=0.44995427
网友评论