org.deeplearning4j.spark.text.functions

Class TextPipeline

java.lang.Object

org.deeplearning4j.spark.text.functions.TextPipeline

public class TextPipeline
extends java.lang.Object

A spark based text pipeline with minimum word frequency and stop words

Constructor Summary

Constructors

Constructor and Description
TextPipeline()
TextPipeline(org.apache.spark.api.java.JavaRDD<java.lang.String> corpusRDD, org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.Object>> broadcasTokenizerVarMap)

Method Summary

Modifier and Type	Method and Description
void	buildVocabCache()
void	buildVocabWordListRDD()
void	filterMinWordAddVocab(Counter<java.lang.String> wordFreq)
org.apache.spark.broadcast.Broadcast<VocabCache<VocabWord>>	getBroadCastVocabCache()
org.apache.spark.api.java.JavaRDD<java.util.concurrent.atomic.AtomicLong>	getSentenceCountRDD()
org.apache.spark.api.java.JavaRDD<Pair<java.util.List<java.lang.String>,java.util.concurrent.atomic.AtomicLong>>	getSentenceWordsCountRDD()
java.lang.Long	getTotalWordCount()
VocabCache<VocabWord>	getVocabCache()
org.apache.spark.api.java.JavaRDD<java.util.List<VocabWord>>	getVocabWordListRDD()
org.apache.spark.Accumulator<Counter<java.lang.String>>	getWordFreqAcc()
void	setRDDVarMap(org.apache.spark.api.java.JavaRDD<java.lang.String> corpusRDD, org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.Object>> broadcasTokenizerVarMap)
org.apache.spark.api.java.JavaRDD<java.util.List<java.lang.String>>	tokenize()
org.apache.spark.api.java.JavaRDD<Pair<java.util.List<java.lang.String>,java.util.concurrent.atomic.AtomicLong>>	updateAndReturnAccumulatorVal(org.apache.spark.api.java.JavaRDD<java.util.List<java.lang.String>> tokenizedRDD)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TextPipeline

public TextPipeline()

TextPipeline

public TextPipeline(org.apache.spark.api.java.JavaRDD<java.lang.String> corpusRDD,
                    org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.Object>> broadcasTokenizerVarMap)
             throws java.lang.Exception

Throws:

java.lang.Exception

Method Detail

setRDDVarMap

public void setRDDVarMap(org.apache.spark.api.java.JavaRDD<java.lang.String> corpusRDD,
                         org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.Object>> broadcasTokenizerVarMap)

tokenize

public org.apache.spark.api.java.JavaRDD<java.util.List<java.lang.String>> tokenize()

updateAndReturnAccumulatorVal

public org.apache.spark.api.java.JavaRDD<Pair<java.util.List<java.lang.String>,java.util.concurrent.atomic.AtomicLong>> updateAndReturnAccumulatorVal(org.apache.spark.api.java.JavaRDD<java.util.List<java.lang.String>> tokenizedRDD)

filterMinWordAddVocab

public void filterMinWordAddVocab(Counter<java.lang.String> wordFreq)

buildVocabCache

public void buildVocabCache()

buildVocabWordListRDD

public void buildVocabWordListRDD()

getWordFreqAcc

public org.apache.spark.Accumulator<Counter<java.lang.String>> getWordFreqAcc()

getBroadCastVocabCache

public org.apache.spark.broadcast.Broadcast<VocabCache<VocabWord>> getBroadCastVocabCache()
                                                                                   throws java.lang.IllegalStateException

Throws:

java.lang.IllegalStateException

getVocabCache

public VocabCache<VocabWord> getVocabCache()
                                    throws java.lang.IllegalStateException

Throws:

java.lang.IllegalStateException

getSentenceWordsCountRDD

public org.apache.spark.api.java.JavaRDD<Pair<java.util.List<java.lang.String>,java.util.concurrent.atomic.AtomicLong>> getSentenceWordsCountRDD()

getVocabWordListRDD

public org.apache.spark.api.java.JavaRDD<java.util.List<VocabWord>> getVocabWordListRDD()
                                                                                 throws java.lang.IllegalStateException

Throws:

java.lang.IllegalStateException

getSentenceCountRDD

public org.apache.spark.api.java.JavaRDD<java.util.concurrent.atomic.AtomicLong> getSentenceCountRDD()
                                                                                              throws java.lang.IllegalStateException

Throws:

java.lang.IllegalStateException

getTotalWordCount

public java.lang.Long getTotalWordCount()