diff options
| author | prasant.kolachina <prasant.kolachina@cse.gu.se> | 2015-04-27 13:05:14 +0000 |
|---|---|---|
| committer | prasant.kolachina <prasant.kolachina@cse.gu.se> | 2015-04-27 13:05:14 +0000 |
| commit | 9b723af9955e0e244a3dfda37aa32d41fc7963e3 (patch) | |
| tree | e1d6c9164a9e1abfa489600075c1c381b9da0e42 /src | |
| parent | 25558916546e51516ed6791971a31d957d0e9292 (diff) | |
remove parsing for long sentences, to stop pipeline from crashing/hanging
Diffstat (limited to 'src')
| -rw-r--r-- | src/runtime/python/examples/gf_utils.py | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/src/runtime/python/examples/gf_utils.py b/src/runtime/python/examples/gf_utils.py index b097cb118..edc36d1fa 100644 --- a/src/runtime/python/examples/gf_utils.py +++ b/src/runtime/python/examples/gf_utils.py @@ -122,7 +122,7 @@ def getKLinearizations(grammar, tgtlanguage, abstractParsesList): kBestTrans.append( ((parseprob,), postprocessor( generator(parse) )) ); yield kBestTrans; -def getKBestParses(grammar, language, K, callbacks=[], serializable=False, sentid=count(1)): +def getKBestParses(grammar, language, K, callbacks=[], serializable=False, sentid=count(1), max_length=50): parser = grammar.languages[language].parse; def worker(sentence): sentence = sentence.strip(); @@ -130,6 +130,10 @@ def getKBestParses(grammar, language, K, callbacks=[], serializable=False, senti tstart = time.time(); kBestParses = []; parseScores = {}; + if len(sentence.split()) > max_length: + tend, err = time.time(), "Sentence too long (%d tokens). Might potentially run out of memory" %(len(sentence.split())); + print >>sys.stderr, '%d\t%.4f\t%s' %(curid, tend-tstart, err); + return tend-tstart, kBestParses; # temporary hack to make sure parser does not get killed for very long sentences; try: for parseidx, parse in enumerate( parser(sentence, heuristics=0, callbacks=callbacks) ): parseScores[parse[0]] = True; |