scratch/SriLanguageModel.java at master · debanjanghosh/scratch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
package nlp.assignments.lm;


import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;

import nlp.langmodel.LanguageModel;
import nlp.util.Counter;
import nlp.util.CounterMap;

/**
 * A dummy language model -- uses empirical unigram counts, plus a single
 * ficticious count for unknown words.
 */
public class SriLanguageModel implements LanguageModel {

  static final String START = "<s>";
  static final String STOP = "</s>";
  static final String UNKNOWN = "<unk>";

  Counter<String> probabilities = new Counter<String>();
  Counter<String> backoffs = new Counter<String>();

  public double getTrigramProbability(String prePreviousWord, String previousWord, String word) {
    double trigramProbability = probabilities.getCount(prePreviousWord + " " + previousWord + " " + word);
    if (trigramProbability != 0) return Math.exp(trigramProbability);

    double bigramProbability = probabilities.getCount(previousWord + " " + word);
    if (bigramProbability != 0) return Math.exp(bigramProbability + backoffs.getCount(prePreviousWord + " " + previousWord));

    double unigramProbability = probabilities.getCount(word);
    if (unigramProbability == 0) {
    	System.out.println("UNKNOWN Word: " + word);
    	unigramProbability = probabilities.getCount(UNKNOWN);
    }
    return Math.exp(unigramProbability + backoffs.getCount(previousWord));
  }

  public double getSentenceProbability(List<String> sentence) {
    List<String> stoppedSentence = new ArrayList<String>(sentence);
    stoppedSentence.add(0, START);
    stoppedSentence.add(0, START);
    stoppedSentence.add(STOP);
    double probability = 1.0;
    String prePreviousWord = stoppedSentence.get(0);
    String previousWord = stoppedSentence.get(1);
    for (int i = 2; i < stoppedSentence.size(); i++) {
      String word = stoppedSentence.get(i);
      probability *= getTrigramProbability(prePreviousWord, previousWord, word);
      prePreviousWord = previousWord;
      previousWord = word;
    }
    if (probability == 0) System.err.println("Underflow");
    return probability;
  }

  String generateWord() {
    return UNKNOWN;
  }

  public List<String> generateSentence() {
    List<String> sentence = new ArrayList<String>();
    String word = generateWord();
    while (!word.equals(STOP)) {
      sentence.add(word);
      word = generateWord();
    }
    return sentence;
  }

  public SriLanguageModel(String fileName) {
	BufferedReader reader;
	try {
		reader = new BufferedReader(new FileReader(fileName));
		String line = reader.readLine();
		while (line != null) {
			if (!line.isEmpty() && line.charAt(0) == '-') {
				String[] parts = line.split("\t");
				if (parts.length != 2 && parts.length != 3) {
					System.err.println("BUG: " + Arrays.toString(parts));
				}
				probabilities.setCount(parts[1], Double.parseDouble(parts[0])/Math.log10(Math.E));
				if (parts.length == 3) {
					backoffs.setCount(parts[1], Double.parseDouble(parts[2])/Math.log10(Math.E));
				}
			}
			line = reader.readLine();
		}
	} catch (Exception e) {
		e.printStackTrace();
	}

  }

}