这几天在看输入法的东西，资料比较少，特此记录一下，这里使用到了前向概率Viterbi算法，公式是
P = 初始状态概率*转移概率*发射概率
这里会得到很多概率值，我们只获取最大概率的中文汉字链。
定义一个汉字节点
type Node struct {
Word string //汉字
Py string //拼音
Emission float64 //汉字出现的概率
MaxScore float64 //最大分数
PreNode *Node //下一个汉字节点
}
定义需要的数据
/**
读取整理好的汉字拼音数据，获取的map数据，数据格式类型为
{
"ni":{"你"：0.91，"尼"：0.789,...},
"wo":{"我"：0.91，"窝"：0.189,...},
.....
}
**/
var emissionMap = make(map[string]map[string]float64)
/**
读取大量词语数据获取的词语数组
**/
var wordsArray = make([]string, 1)
/**
分析汉字拼音数据，获取的数组数据。格式如下：
[
{
"你"：Node ，
"好"：Node，
},
...
]
**/
var inputSequence = make([]map[string]*Node, 1)
/**
词语出现的概率值，这里会将词语位置颠倒
｛
"好你"：0.789，
"兴高":0.567,
....
｝
**/
var freqMap = make(map[string]float64)
/**
viterbi算法的数据缓存
**/
var viterbi_cache = make(map[string]float64)
/**
输入的已经切分的拼音数组
**/
var pinyins []string
读取拼音汉字数据
func readPinyinData() {
f, err := os.Open("googlepinyin.txt")
if err != nil {
fmt.Println(err.Error())
return
}
buf := bufio.NewReader(f)
for {
line, err := buf.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
fmt.Println(err.Error())
break
}
line = strings.TrimSpace(line)
if len(line) <= 1 {
continue
}
PinyinArray := strings.Split(line, " ")
word := PinyinArray[0]
f, _ := strconv.ParseFloat(PinyinArray[1], 64)
//因为这里获取是个大于1的值，所以求正切值
em := math.Atan(f) / (math.Pi / 2)
py := PinyinArray[len(PinyinArray)-1]
if len(strings.Split(word, "")) > 1 {
continue
}
if emissionMap[py] == nil {
emissionMap[py] = make(map[string]float64)
}
emissionMap[py][word] = em
pyArray := strings.Split(py, "")
py = pyArray[0]
if len(strings.Split(word, "")) > 1 {
break
}
if emissionMap[py] == nil {
emissionMap[py] = make(map[string]float64)
}
emissionMap[py][word] = em
}
读取词语数据
func readWords() {
f, err := os.Open("RenMinData.txt")
if err != nil {
fmt.Println(err.Error())
return
}
buf := bufio.NewReader(f)
for {
line, err := buf.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
fmt.Println(err.Error())
break
}
line = strings.TrimSpace(line)
if len(line) <= 1 {
continue
}
wordsArray = append(wordsArray, "<s>")
words := strings.Split(line, "")
for j := 0; j < len(words); j++ {
wordsArray = append(wordsArray, strings.TrimSpace(words[j]))
}
wordsArray = append(wordsArray, "</s>")
}
wordsArray = wordsArray[1:]
for k, _ := range wordsArray {
i := k
key := ""
for j := i; (i-j < 6) && (j >= 0); j-- {
key += wordsArray[j]
freqMap[key] = freqMap[key] + 1
}
}
}
获取转移概率
func get_trans_prop(args ...string) float64 {
key := ""
for _, v := range args {
key += v
}
C_2 := float64(freqMap[key] + 1.0)
C_1 := float64(len(wordsArray))
return C_2 / C_1
}
获取初始概率
func get_init_prop(word string) float64 {
return get_trans_prop(word, "<s>")
}
获取inputSequence 数据
func get_inputSequence(pys []string) {
pinyins = pys
for _, v := range pinyins {
mymap := make(map[string]*Node)
if emissionMap[v] == nil {
continue
}
for w, r := range emissionMap[v] {
mymap[w] = &Node{Word: w, Emission: r, Py: v}
}
inputSequence = append(inputSequence, mymap)
}
inputSequence = inputSequence[1:]
}
���
viterbi算法
func get_key(t int, k string) string {
return strings.Join([]string{strconv.Itoa(t), k}, "_")
}
func viterbi(t int, k string) float64 {
//如果有缓存数据，直接返回数据值
if viterbi_cache[get_key(t, k)] != 0 {
return viterbi_cache[get_key(t, k)]
}
node := inputSequence[t][k]
if t == 0 {
state_transfer := get_init_prop(k)
emission_prop := emissionMap[pinyins[t]][k]
node.MaxScore = 1.0 * state_transfer * emission_prop
viterbi_cache[get_key(t, k)] = node.MaxScore
return node.MaxScore
}
n := t - 1
for i, v := range inputSequence[n] {
state_transfer := get_trans_prop(k, i)
emission_prop := emissionMap[pinyins[n]][i]
if len(pinyins)-1 == t {
emission_prop *= emissionMap[pinyins[t]][k]
}
score := viterbi(n, i) * state_transfer * emission_prop
if score > node.MaxScore {
node.MaxScore = score
node.PreNode = v
}
}
viterbi_cache[get_key(t, k)] = node.MaxScore
return node.MaxScore
}
获取最大概率的数据
func translate(pys []string) {
get_inputSequence(pys)
// 使用viterbi算法求解最大路径
var max_node *Node
max_score := 0.0
for k, node := range inputSequence[len(inputSequence)-1] {
score := viterbi(len(pys)-1, k)
if score > max_score {
max_score = score
max_node = node
}
}
// 回溯输出最大路径
results := make([]string, 1)
for {
results = append(results, max_node.Word)
if max_node.PreNode != nil {
max_node = max_node.PreNode
} else {
break
}
}
results = results[1:]
for i := len(results) - 1; i >= 0; i-- {
fmt.Print(results[i])
}
}
使用
func main() {
readPinyinData()
readWords()
translate([]string{"ni", "zai", "na", "li"})
}
网友评论