CreateTopicsForDocument.java

package com.research.course.debate.LDA;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.List;

import com.aliasi.stats.Statistics;
import com.aliasi.util.AbstractExternalizable;
import com.research.course.debate.util.DocumentProperty;

public class CreateTopicsForDocument {

	/**
	 * @param args
	 */
	
	private LDAGenerator ldaGenrObj ; 
	
	public CreateTopicsForDocument()
	{
		ldaGenrObj = new LDAGenerator();
	}
/*	
	public void createTopics (String output) throws Exception
	{
		String text = "mr. speaker , for the purpose of debate only , i yield the customary" +
				" 30 minutes to the gentleman from massachusetts ( mr. mcgovern ) xz4002630 , " +
				"pending which i yield myself such time as i may consume . during consideration of " +
				"this resolution , all time yielded is for the purpose of debate only . mr. speaker , " +
				"on tuesday the committee on rules met and granted a rule for house concurrent resolution 36 , " +
				"expressing congressional support for equal access of military recruiters to institutions of " +
				"higher education . the rule provides for 1 hour of general debate equally divided and controlled " +
				"by the chairman and ranking minority member of the committee on armed services . the " +
				"rule also provides for one motion to recommit . mr. speaker , this concurrent resolution is an " +
				"important first step in addressing a misguided ruling by the third circuit court of appeals " +
				"regarding access of military recruiters to institutions of higher education . during this " +
				"time of conflict and the global war on terror , it is more important than ever to maintain the" +
				" ability to recruit quality men and women for service in our military . the primary way that recruiters " +
				"are able to do this is to work through those institutions which work closely with our young men and women , " +
				"schools and universities . military recruiters need the same access to college campuses provided to other potential " +
				"employers , and students deserve the right to discuss the option of a career in the united states military with" +
				" the representatives of the armed forces . mr. speaker , some ask , why the need for this concurrent resolution ? " +
				"well , the answer is succinct . this concurrent resolution grows out of an egregious decision by the third circuit " +
				"court of appeals overturning the power of congress to control the purse . this decision simply states that congress and the " +
				"government may not as a matter of law deny funds to universities on the basis of their denial of access to " +
				"recruiters and rotc units . this decision , couched in the language of civil rights , fails to recognize the " +
				"underlying inequity behind these university policies . this decision asserts the congress has compelled speech " +
				"by these universities to the effect that they `` agree '' with the military 's `` do n't ask , do n't tell '' policy " +
				"with respect to homosexuals in the service . mr. speaker , nothing could be further than the truth . the solomon " +
				"amendment compelled no such thing . it simply proposed standards for the receipt of federal funds . setting such " +
				"standards is a normal and legitimate function of the legislative branch . it is what defines the power of the " +
				"purse . this is an issue that the house and senate have revisited and affirmed in bipartisan votes in 1995 , " +
				"1996 , 1999 , and 2002 after the enactment of the original solomon amendment . mr. speaker , it is disappointing " +
				"to note that the reserve officers training corps , or popularly known as the rotc , has been embattled on some " +
				"university and college campuses since the 1960s . this stems from what only can be described as a consistently " +
				"anti-military philosophy advocated by some , and i want to say only some , college and university professors " +
				"and administrators . the new purported reasons for not allowing rotc on campus often serves the convenient cover " +
				"for these anti-military sentiments . some educators now believe that they should be allowed to discriminate " +
				"against students who wish to enter the military in order to please another group of students who object to the " +
				"policies and procedures of the armed services , all the while soliciting and accepting federal funds for their " +
				"institutions . this is rank hypocrisy . why would an institution seek and use federal funds , often from the " +
				"department of defense , while denying representatives of the u.s. armed forces access to their campuses ? mr. speaker , the decision " +
				"by the third circuit court of appeals is a classic case of judicial overreach and one that must be " +
				"addressed . as a former university educator and the son of a career air force noncommissioned officer , i " +
				"find this decision disturbing and insulting to those men and women who defend our freedom and to those who wish to join their " +
				"ranks . the very least we can do is put the courts on notice as to exactly where the congress stands on this " +
				"issue . for that reason , this concurrent resolution is necessary and timely . hopefully , it will underscore " +
				"the importance that the congress places on military recruiters having access to the educational institutions that receive federal funds . " +
				"during this time of war , we should insist that institutions who pride themselves on freedom of expression allow the defenders of that freedom , " +
				"the united states military , to freely recruit the soldiers who protect our democracy . to that end , i urge support for the rule and the underlying bill . mr. speaker , " +
				"i reserve the balance of my time . " ;
		
				ldaGenrObj.setOutputPath(output);
				List<DocumentProperty> topics = ldaGenrObj.doGenerateMetadata(text) ;
				
				
				for ( DocumentProperty topic : topics  )
				{
					System.out.println(topic.toString());
					
				//	Statistics.klDivergence(p, q) ;
				}
				
	}
	
	*/

	public void createTopics ( String inputDir, String outputDir, String extn ) throws Exception
	{
		ldaGenrObj.setOutputPath(outputDir);
		ldaGenrObj.setLDaParameters(extn) ;
		
		File file = new File ( inputDir) ;
		File files[] = file.listFiles() ;
		
		BufferedWriter writer = null ;
		
		if (files != null) 
		{
			for (File f : files) 
			{
				
				//discard the Ds_Store and other zip files 
				if (f.isDirectory())
				{
					continue ;
				}
				
			//	if (f.toString().contains("16") || f.toString().contains("282"))
			//	{
			//		continue ;
			//	}
				
				if(f.toString().contains("zip") || f.toString().contains("Store"))
				{
					continue ;
				}
				
				System.out.println(" debate file is " + f ) ;
				
				List<String> debateSpeeches = LDADataUtil.getDebateSpeeched(f);
				
				writer = new BufferedWriter ( new OutputStreamWriter(
						new FileOutputStream(outputDir + "/" + "debate_" + f.getName() + 
								"_511.txt"), "UTF8"));
				
				for ( String speech : debateSpeeches )
				{
					DocumentProperty topic = ldaGenrObj.doGenerateMetadata(speech) ;
					{
						writer.write(speech + "\t" + topic.getValue());
						writer.newLine();
					}
				}
				writer.close();
			}
		}
	}
	public static void main(String[] args) throws Exception 
	{
		// TODO Auto-generated method stub
		String inputDir = args[0] ;
		String outputDir = args[1];
		String extn = args[2] ;
		CreateTopicsForDocument documentLDAObject = new CreateTopicsForDocument();
		documentLDAObject.createTopics(inputDir,outputDir,extn);
	}

}