Examples for using Python bindings

The examples directory contains two modules for using Python bindings in GF.
The gf_utils module provides various Python functions to perform parsing
and linearization using the C runtime. Additionally, the translation_pipeline
module is an replica of the current pipeline used in the Wide-coverage
Web translation pipeline.

2 files changed, 698 insertions, 0 deletions

diff --git a/src/runtime/python/examples/gf_utils.py b/src/runtime/python/examples/gf_utils.py
new file mode 100644
index 000000000..bbb8d1766
--- /dev/null
+++ b/src/runtime/python/examples/gf_utils.py
@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+
+"""
+"""
+
+import argparse, re, string, sys, time;
+from itertools import imap, count;
+from operator import itemgetter;
+
+import pgf;
+
+def lexerI(sentence):
+    return sentence.rstrip(string.whitespace+string.punctuation);
+
+def lexerChi(sentence):
+    sentence = sentence.decode('utf-8');
+    tokens, idx, n = [], 0, len(sentence);
+    prev = True;
+    while idx < n:
+	if sentence[idx] in string.whitespace:
+	    prev = True;
+	    idx += 1;
+	    continue;
+	if 0 < ord(sentence[idx]) < 128:
+	    if sentence[idx] in string.punctuation:
+		prev = True;
+	    if prev:
+		tokens.append( sentence[idx] );
+		prev = False;
+	    else:
+		tokens[-1] = tokens[-1]+sentence[idx];
+	else:
+	    prev = True;
+	    tokens.append( sentence[idx] );
+	idx += 1;
+    return ' '.join(tokens).encode('utf-8');
+
+def lexer(lang='Eng'):
+    if lang[-3:] == 'Eng':
+	return lexerI;
+    elif lang[-3:] == 'Chi':
+	return lexerChi;
+    elif lang == 'translator':
+	import translation_pipeline;
+	return translation_pipeline.pipeline_lexer;
+    else:
+	return lexerI;
+
+def postprocessor(sentence):
+    if sentence == None:
+	return '';
+    if sentence.startswith('* ') or sentence.startswith('% '):
+	sentence = sentence[2:];
+    sentence = sentence.replace(' &+ ', '');
+    sentence = sentence.replace('<+>', ' ');
+    return sentence;
+
+def readJohnsonRerankerTrees(inputStream):
+    endOfParse = False;
+    while True:
+	sentheader = inputStream.next();
+	if sentheader == '':
+	    break;
+	parsescount, sentidx = map(int, sentheader.strip().split());
+	parsesBlock = [];
+	for i in xrange(parsescount):
+	    parseprob = inputStream.next();
+	    if parseprob.strip() == '':
+		endOfParse = True;
+		break;
+	    parse = inputStream.next();
+	    parsesBlock.append( (float(parseprob.strip()), pgf.readExpr(parse.strip())) );
+	yield sentidx, parsesBlock;
+	if not endOfParse:
+	    _ = inputStream.next();
+	endOfParse = False;
+
+def readMosesNbestFormat(inputStream):
+    transBlock = [];
+    currentHypothesisId = 0;
+    while True:
+	line = inputStream.next();
+	if line == '':
+	    break;
+	fields = line.strip().split('|||');
+	if str(fields[0].strip()) != str(currentHypothesisId):
+	    yield currentHypothesisId, transBlock;
+	    transBlock = [];
+	    currentHypothesisId = int(fields[0]);
+	transBlock.append( (map(float, tuple([val.strip() for val in fields[3].split()])), fields[1].strip()) );
+
+def printJohnsonRerankerFormat(gfparsesList, sentid=count(1)):
+    johnsonRepr = [];
+    parseHash = {};
+    for parse in sorted(gfparsesList, key=itemgetter(0)):
+	if not parseHash.has_key(parse[1]):
+	    johnsonRepr.append( str(-1*parse[0]) );
+	    johnsonRepr.append( str(parse[1]) );
+	parseHash.setdefault(parse[1], []).append(parse[0]);
+    curid = sentid.next();
+    if len(gfparsesList):
+	johnsonRepr.insert(0, '%d %d' %(len(parseHash.values()), curid));
+    duplicateInstances = len(filter(lambda X: len(parseHash[X]) > 1, parseHash.keys()));
+    #if duplicateInstances: print >>sys.stderr, "%d duplicate parses found in K-best parsing" %(duplicateInstances);
+    return '\n'.join(johnsonRepr)+'\n';
+
+def printMosesNbestFormat(hypothesisList, sentid=count(1)):
+    mosesRepr = [];
+    sid = sentid.next();
+    for hypScores, hypStr in hypothesisList:
+	if not hasattr(hypScores, '__iter__'):
+	    hypScores = (hypScores, );
+	mosesRepr.append("%d ||| %s ||| NULL ||| %s" %(sid, hypStr, ' '.join(['%.6f'%score for score in hypScores])));
+    return '\n'.join(mosesRepr);
+
+def getKLinearizations(grammar, tgtlanguage, abstractParsesList):
+    generator = grammar.languages[tgtlanguage].linearize;
+    for parsesBlock in abstractParsesList:
+	kBestTrans = [];
+	for parseprob, parse in parsesBlock:
+	    #print str(parse);
+	    kBestTrans.append( ((parseprob,), postprocessor( generator(parse) )) );
+	yield kBestTrans;
+
+def getKBestParses(grammar, language, K, callbacks=[], serializable=False, sentid=count(1)):
+    parser = grammar.languages[language].parse;
+    def worker(sentence):
+	sentence = sentence.strip();
+	curid = sentid.next();
+	tstart = time.time();
+	kBestParses = [];
+	parseScores = {};
+	try:
+	    for parseidx, parse in enumerate( parser(sentence, heuristics=0, callbacks=callbacks) ):
+		parseScores[parse[0]] = True;
+		kBestParses.append( (parse[0], str(parse[1]) if serializable else parse[1]) );
+		if parseidx == K-1: break;
+		#if len(parseScores) >= K: break;
+	    tend = time.time();
+	    print >>sys.stderr, '%d\t%.4f' %(curid, tend-tstart);
+	    return tend-tstart, kBestParses;
+	except pgf.ParseError, err:
+	    tend = time.time();
+	    print >>sys.stderr, '%d\t%.4f\t%s' %(curid, tend-tstart, err);
+	    return tend-tstart, kBestParses;
+	except UnicodeEncodeError, err:
+	    tend = time.time();
+	    print >>sys.stderr, '%d\t%.4f\t%s' %(curid, tend-tstart, err);
+	    return tend-tstart, kBestParses;
+    return worker;
+
+def pgf_parse(args):
+    grammar  = pgf.readPGF(args.pgfgrammar);
+    inputSet = web_lexer(grammar, args.srclang, args.inputstream);
+    outputPrinter = lambda X: "%f\t%s" %(X[0], str(X[1])); #operator.itemgetter(1);
+    
+    import translation_pipeline;
+    callbacks = [('PN', translation_pipeline.parseNames(grammar, args.srclang)), ('Symb', translation_pipeline.parseUnknown(grammar, args.srclang))];
+    parser = getKBestParses(grammar, args.srclang, 1, callbacks);
+    
+    sentidx = 0;
+    for time, parsesBlock in imap(parser, inputSet):
+	sentidx += 1;
+	print >>args.outputstream, "%d\t%f\t%s" %(sentidx, time, str(outputPrinter(parsesBlock[0])) if len(parsesBlock) else '');
+    return;
+
+def pgf_kparse(args):
+    grammar = pgf.readPGF(args.pgfgrammar);
+    inputSet = web_lexer(grammar, args.srclang, args.inputstream);
+    outputPrinter = printJohnsonRerankerFormat;
+    
+    import translation_pipeline;
+    callbacks = [('PN', translation_pipeline.parseNames(grammar, args.srclang)), ('Symb', translation_pipeline.parseUnknown(grammar, args.srclang))];
+    parser = getKBestParses(grammar, args.srclang, args.K, callbacks=callbacks);
+
+    sentidx = 0;
+    for time, parsesBlock in imap(parser, inputSet):
+	sentidx += 1;
+	strParses = str(outputPrinter(parsesBlock));
+	if not (strParses == '\n'):
+	    print >>args.outputstream, strParses;
+    return;
+
+def pgf_linearize(args):
+    grammar = pgf.readPGF(args.pgfgrammar);
+    outputPrinter = postprocessor;
+    inputSet = [];
+    for line in args.inputstream:
+	try:
+	    sentid, parsetime, parserepr = line.strip('\n').split('\t', 2);
+	except ValueError:
+	    print line.strip();
+	parseprob, abstree = parserepr.split('\t') if parserepr.strip() else (0, '');
+	inputSet.append( (int(sentid), float(parsetime), float(parseprob), pgf.readExpr(abstree) if abstree else None) );
+    linearizer = grammar.languages[args.tgtlang].linearize;
+    for sentid, _, _, abstree in inputSet:
+	if abstree:
+	    print >>args.outputstream, str(outputPrinter(linearizer(abstree)));
+	else:
+	    print >>args.outputstream, "";
+    return;
+
+def pgf_klinearize(args):
+    grammar = pgf.readPGF(args.pgfgrammar);
+    outputPrinter = printMosesNbestFormat;
+    inputSet = [(sentid, parsesBlock) for sentid, parsesBlock in readJohnsonRerankerTrees(args.inputstream)];
+    sentIdsList = imap(itemgetter(0), inputSet);
+    parsesBlocks = map(itemgetter(1), inputSet);
+
+    for transBlock in getKLinearizations(grammar, args.tgtlang, parsesBlocks):
+	strTrans = str(outputPrinter(transBlock, sentIdsList));
+	if strTrans:
+	    print >>args.outputstream, strTrans;
+    return;
+
+def cmdLineParser():
+    argparser = argparse.ArgumentParser(prog='gf_utils.py',  description='Examples for carrying out (K-best) parsing, translation and linearization using GF C runtime.');
+    
+    subparsers  = argparser.add_subparsers();
+    parser      = subparsers.add_parser('parse',      help='GF parsing of sentences');
+    kparser     = subparsers.add_parser('kparse',     help='K-best GF parsing of sentences');
+    linearizer  = subparsers.add_parser('linearize',  help='Linearize GF abstract syntax treess');
+    klinearizer = subparsers.add_parser('klinearize', help='Linearize K-variants of GF abstract syntax trees');
+   
+    parser.set_defaults(func=pgf_parse);
+    parser.add_argument('-g', '--pgf',       dest='pgfgrammar',   required=True, \
+	    help='PGF Grammar file');
+    parser.add_argument('-p', '--start-sym', dest='startcat',     required=False, \
+	    help='Start symbol in the grammar');
+    parser.add_argument('-s', '--src-lang',  dest='srclang',      required=True, \
+	    help='Source language');
+    parser.add_argument('-i', '--input',     dest='inputstream',  nargs='?', type=argparse.FileType(mode='r'), default=sys.stdin,  \
+	    help='Input file') ;
+    parser.add_argument('-o', '--output',    dest='outputstream', nargs='?', type=argparse.FileType(mode='w'), default=sys.stdout, \
+	    help='Output file');
+
+    kparser.set_defaults(func=pgf_kparse);
+    kparser.add_argument('-g', '--pgf',       dest='pgfgrammar',   required=True, \
+	    help='PGF Grammar file');
+    kparser.add_argument('-p', '--start-sym', dest='startcat',     required=False, \
+	    help='Start symbol in the grammar');
+    kparser.add_argument('-s', '--src-lang',  dest='srclang',      required=True, \
+	    help='Source language');
+    kparser.add_argument('-K',                dest='K',            required=True, type=int, \
+	    help='K value for multiple parses');
+    kparser.add_argument('-i', '--input',     dest='inputstream',  nargs='?', type=argparse.FileType(mode='r'), default=sys.stdin, \
+	    help='Input file');
+    kparser.add_argument('-o', '--output',    dest='outputstream', nargs='?', type=argparse.FileType(mode='w'), default=sys.stdout, \
+	    help='Output file');
+    
+    linearizer.set_defaults(func=pgf_linearize);
+    linearizer.add_argument('-g', '--pgf',      dest='pgfgrammar',   required=True, \
+	    help='PGF Grammar file');
+    linearizer.add_argument('-t', '--tgt-lang', dest='tgtlang',      required=True, \
+	    help='Target language');
+    linearizer.add_argument('-i', '--input',    dest='inputstream',  nargs='?', type=argparse.FileType(mode='r'), default=sys.stdin,  \
+	    help='Input file');
+    linearizer.add_argument('-o', '--output',   dest='outputstream', nargs='?', type=argparse.FileType(mode='w'), default=sys.stdout, \
+	    help='Output file'); 
+
+    klinearizer.set_defaults(func=pgf_klinearize);
+    klinearizer.add_argument('-g', '--pgf',      dest='pgfgrammar',   required=True, \
+	    help='PGF Grammar file');
+    klinearizer.add_argument('-t', '--tgt-lang', dest='tgtlang',      required=True, \
+	    help='Target language');
+    klinearizer.add_argument('-K',               dest='K',            required=True, type=int, \
+	    help='K value for multiple linearizations');
+    klinearizer.add_argument('-i', '--input',    dest='inputstream',  nargs='?', type=argparse.FileType(mode='r'), default=sys.stdin,  \
+	    help='Input file');
+    klinearizer.add_argument('-o', '--output',   dest='outputstream', nargs='?', type=argparse.FileType(mode='w'), default=sys.stdout, \
+	    help='Output file');
+
+    return argparser;
+
+if __name__ == '__main__':
+    args = cmdLineParser().parse_args(sys.argv[1:]);
+    args.func(args);
diff --git a/src/runtime/python/examples/translation_pipeline.py b/src/runtime/python/examples/translation_pipeline.py
new file mode 100644
index 000000000..22c33c97f
--- /dev/null
+++ b/src/runtime/python/examples/translation_pipeline.py
@@ -0,0 +1,421 @@
+#!/usr/bin/env python
+
+import argparse, codecs, copy, itertools, logging, math, operator, os, os.path, re, string, sys, time;
+import xml.etree.ElementTree as etree;
+
+import pgf;
+import gf_utils;
+
+# http://snipplr.com/view/25657/indent-xml-using-elementtree/
+def indentXMLNodes(elem, level=0):
+    i = "\n" + level*"  "
+    if len(elem):
+	if not elem.text or not elem.text.strip():
+	    elem.text = i + "  "
+	if not elem.tail or not elem.tail.strip():
+	    elem.tail = i
+	for elem in elem:
+	    indentXMLNodes(elem, level+1)
+	if not elem.tail or not elem.tail.strip():
+	    elem.tail = i
+    else:
+	if level and (not elem.tail or not elem.tail.strip()):
+	    elem.tail = i
+
+def readTranslationPipelineOptions(propsfile, default_namespace):
+    with codecs.open(propsfile, 'r', 'utf-8') as infile:
+	for line in infile:
+	    if not line.strip():
+		continue;
+	    key, value = line.strip().split('=', 1);
+	    key, value = key.strip(), value.strip();
+	    if   key == 'srclang':  default_namespace.srclang = value;
+	    elif key == 'tgtlangs': default_namespace.tgtlangs = [val.strip() for val in ','.split(value)];
+	    elif key == 'input':    default_namespace.input = value;
+	    elif key == 'format':   default_namespace.format = value;
+	    elif key == 'exp_directory': default_namespace.exp_directory = value;
+	    else:
+		#print >>sys.stderr, "Unknown option-%s found in props file. Ignoring and proceeding." %(key);
+		logging.warning("Unknown option-%s found in props file. Ignoring and proceeding." %(key));
+		continue;
+    return default_namespace;
+
+def sgmReader(sgmDoc):
+    root = sgmDoc.getroot();
+    for element in root.iter():
+	if element.text is not None and element.text.strip():
+	    yield element.text.strip().encode('utf-8');
+
+def addToSgm(sgmDoc, strItem):
+    for node in sgmDoc.findall('.//seg'):
+	if not node.text.strip():
+	    strItem = strItem.decode('utf-8');
+	    node.text = ' %s ' %(strItem if strItem.strip() else 'EMPTY');
+	    return;
+    logging.error("No more nodes available for adding content");
+    return;
+
+def sgmWriter(sgmDoc):
+    indentXMLNodes( sgmDoc.getroot() );
+    return etree.tostring(sgmDoc.getroot(), encoding='utf-8', method='xml');
+
+def getXMLSkeleton(sgmDoc, tgtlang):
+    skeletonDoc = copy.deepcopy(sgmDoc);
+    root = skeletonDoc.getroot();
+    root.tag = 'tstset';
+    root.attrib['trlang'] = tgtlang[-3:];
+    root.find('doc').attrib['sysid'] = tgtlang[:-3];
+    for node in root.findall('.//seg'):
+	node.text = '';
+    return skeletonDoc;
+
+def pipeline_lexer(sentence):
+    tokens = sentence.strip().split();
+    #tokens = filter(None, re.split('(\W+)', sentence.strip()));
+    n = len(tokens);
+    idx = len(tokens)-1;
+    while idx >= 0:
+	if tokens[idx] in ".?!)":
+	    idx -= 1;
+	else:
+	    break;
+    tokens = tokens[:idx+1];
+    idx = 0;
+    while idx < len(tokens):
+	if tokens[idx] in "'\"(":
+	    idx += 1;
+	else:
+	    break;
+    tokens = tokens[idx:];
+    return ' '.join(tokens);
+
+def web_lexer(grammar, lang, sentences):
+    for instance in sentences:
+	tokensList = re.split('\s+?', instance);
+	for idx, token in enumerate(tokensList):
+	    if not token[0].isupper():
+		continue;
+	    lowertoken = tokensList[idx].lower();
+	    count = 0;
+	    for analysis in grammar.languages[lang].lookupMorpho(lowertoken):
+		count += 1;
+	    if count:
+		print "replacing %s with %s" %(token, lowertoken);
+	    tokensList[idx] = lowertoken if count else token;
+	for idx, token in enumerate(tokensList):
+	    if token.find('-') == -1:
+		continue;
+	    count = 0;
+	    for analysis in grammar.languages[lang].lookupMorpho(token):
+		count += 1;
+	    if count: 
+		continue;
+	    token = tokensList[idx].replace('-', '');
+	    for analysis in grammar.languages[lang].lookupMorpho(token):
+		count += 1;
+	    if count:
+		tokensList[idx] = token;
+		continue;
+	    token = tokensList[idx].replace('-', ' ');
+	yield ' '.join(tokensList);
+
+def clean_gfstrings(sentence):
+    absFuncName = re.compile('\[[^]]+?\]');
+    untranslatedEntries = {};
+    for entry in re.findall(absFuncName, sentence):
+	untranslatedEntries[entry] = untranslatedEntries.setdefault(entry, 0)+1;
+    for entry in untranslatedEntries:
+	while untranslatedEntries[entry] > 1:
+	    sentence = sentence.replace(entry, '', 1);
+	    untranslatedEntries[entry] -= 1;
+	sentence = sentence.replace(entry, ' '.join(entry[1:-1].split('_')[:-1]) if entry.find('_') != -1 else '');
+    return ' '.join( sentence.split() );
+
+def parseNames(grammar, language):
+    def callback(lin_idx, sentence, start):
+	moving_start, end, eot = start, len(sentence), True;
+	if moving_start < end and (not sentence[moving_start].isupper()):
+	    return None;
+	while moving_start < end:
+	    if sentence[moving_start] in string.whitespace:
+		eot = True;
+	    elif eot and sentence[moving_start].isupper():
+		eot = False;
+	    elif eot and (not sentence[moving_start].isupper()):
+		end = moving_start-1;
+		break;
+	    moving_start += 1;
+	possible_name = sentence[start:end].strip();
+	if possible_name:
+	    if language.endswith('Eng') and (possible_name == "I" or possible_name == "I'm"):
+		return None;
+	    elif language.endswith('Eng') and possible_name.endswith("'s"):
+		end_idx = possible_name.rfind("'s");
+		if end_idx != -1:
+		    possible_name = possible_name[:end_idx].strip();
+		    end -= 2;
+		    if not possible_name:
+			return None;
+	    expr, prob = None, None;
+	    for analysis in grammar.languages[language].lookupMorpho(possible_name):
+		category = grammar.functionType(analysis[0]).cat;
+		if prob < analysis[-1]:
+		    if category == "PN":
+			expr, prob = pgf.Expr(analysis[0], []), analysis[-1];
+		    elif category == "Weekday":
+			expr, prob = pgf.Expr("weekdayPN", [pgf.Expr(analysis[0], [])]), analysis[-1];
+		    elif category == "Month":
+			expr, prob = pgf.Expr("monthPN", [pgf.Expr(analysis[0], [])]), analysis[-1];
+		    elif category == "Language":
+			return None;
+	    # generic named entity
+	    if expr == None:
+		expr = pgf.Expr(possible_name);
+		expr = pgf.Expr("MkSymb", [expr]);
+		expr = pgf.Expr("SymbPN", [expr]);
+	    return (expr, 0, end);
+	return None;
+    return callback;
+
+def parseUnknown(grammar, language):
+    def callback(lin_idx, sentence, start):
+	moving_start, end, eot = start, len(sentence), True;
+	isNewToken = (moving_start == 0) or (moving_start > 1 and sentence[moving_start-1].isspace()) # -- added to deal with segmentation errors like may => ma_N + Symb y
+	if moving_start < end and (not sentence[moving_start].isupper()):
+	    while moving_start < end:
+		if sentence[moving_start] in string.whitespace:
+		    end = moving_start;
+		    break;
+		moving_start += 1;
+	    unknown_word = sentence[start:end].strip();
+	    if unknown_word and isNewToken:
+		count = 0;
+		for analysis in grammar.languages[language].lookupMorpho(unknown_word):
+		    count += 1;
+		if not count:
+		    expr = pgf.Expr("MkSymb", [pgf.Expr(unknown_word)]);
+		    return (expr, 0, end);
+	return None;
+    return callback;
+
+def parseTester(grammar, language):
+    def callback(lin_idx, sentence, start):
+	if start < len(sentence):
+	    return (pgf.Expr(sentence[start]), 0, start+1);
+	return None;
+    return callback;
+
+def translateWord(grammar, language, tgtlanguage, word):
+    lowerword = word.lower();
+    try:
+	partialExprList = grammar.languages[language].parse(word, cat='Chunk');
+	for expr in partialExprList:
+	    trans = grammar.languages[tgtlanguage].linearize(expr[1]);
+	    if not trans:
+		print expr[1], tgtlanguage;
+	    return gf_utils.gf_postprocessor( trans if trans else ' ' );
+    except pgf.ParseError:
+	morphAnalysis = grammar.languages[language].lookupMorpho(word) + grammar.languages[language].lookupMorpho(lowerword);
+	for morph in morphAnalysis:
+	    if grammar.languages[tgtlanguage].hasLinearization(morph[0]):
+		return gf_utils.gf_postprocessor( grammar.languages[tgtlanguage].linearize( pgf.readExpr(morph[0]) ) );
+    return word;
+
+def translationByLookup(grammar, language, tgtlanguages, sentence):
+    return [(lang, gf_utils.gf_postprocessor("% " + " ".join([translateWord(grammar, language, lang, word) for word in sentence.split()]))) \
+	    for lang in tgtlanguages];
+
+def translateWordsAsChunks(grammar, language, tgtlanguages, word):
+    parser = grammar.languages[language].parse;
+    linearizersList = dict((lang, grammar.languages[lang].linearize) for lang in tgtlanguages);
+    translations = [];
+    try:
+	for parseidx, parse in enumerate( parser(word) ):
+	    for lang in tgtlanguages:
+		trans = linearizersList[lang](parse[1]);
+		translations.append(( lang, gf_utils.postprocessor(trans.strip() if trans else '') ) );
+	    break;
+    except pgf.ParseError, err:
+	return [];
+    return translations;
+
+def translateWord_(grammar, language, tgtlanguages, word):
+    possible_translations = translateWordsAsChunks(grammar, language, tgtlanguages, word);
+    if len(possible_translations):
+	return possible_translations;
+
+    lowerword = word.lower();
+    try:
+	partialExprList = grammar.languages[language].parse(word, cat='Chunk');
+	for expr in partialExprList:
+	    return [(lang, gf_utils.gf_postprocessor( grammar.languages[lang].linearize(expr[1]) )) for lang in tgtlanguages];
+    except pgf.ParseError:
+	morphAnalysis = grammar.languages[language].lookupMorpho(word) + grammar.languages[language].lookupMorpho(lowerword);
+	for morph in morphAnalysis:
+	    countPositiveLanguages = filter(None, [grammar.languages[lang].hasLinearization(morph[0]) for lang in tgtlanguages]);
+	    if len(countPositiveLanguages) > 0.5*len(tgtlanguages):
+		return [(lang, gf_utils.gf_postprocessor( grammar.languages[lang].linearize( pgf.readExpr(morph[0]) ) )) for lang in tgtlanguages];
+    return [(lang, word) for lang in tgtlanguages];
+
+def translationByLookup_(grammar, language, tgtlanguages, sentence):
+    parser = grammar.languages[language].parse;
+    linearizersList = dict([(lang, grammar.languages[lang].linearize) for lang in tgtlanguages]);
+    queue = [sentence.strip().split()];
+    transChunks = {};
+    while len(queue):
+	head = queue[0];
+	if not len(head):
+	    pass;
+	elif len(head) == 1 and head[0].strip():
+	    for lang, wordchoice in translateWord_(grammar, language, tgtlanguages, head[0]):
+		transChunks.setdefault(lang, []).append( gf_utils.postprocessor(wordchoice) );
+	else:
+	    try:
+		for parseidx, parse in enumerate( parser(' '.join(head)) ):
+		    for lang in tgtlanguages:
+			if linearizersList[lang](parse[1]) == None:
+			    transChunks.setdefault(lang, []).append( ' ' );
+			else:
+			    transChunks.setdefault(lang, []).append( gf_utils.postprocessor( linearizersList[lang](parse[1]).strip() ) );
+		    break;
+	    except pgf.ParseError, err:
+		#unseenToken = re.findall('"[^"]+?"', err.message)[0][1:-1];
+		unseenToken = err.message.strip().split()[-1][1:-1];
+		idx = head.index(unseenToken);
+		queue.insert(1, head[:idx] );
+		queue.insert(2, [head[idx]] );
+		queue.insert(3, head[idx+1:] );
+	del queue[0];
+    for lang in tgtlanguages:
+	yield (lang, ' '.join(transChunks[lang]));
+
+def pipelineParsing(grammar, language, sentences, K=20):
+    #buf = [sent for sent in sentences];
+    buf, sentences = itertools.tee(sentences, 2);
+    sentences = itertools.imap(gf_utils.lexer(lang=language), sentences);
+    parser = gf_utils.getKBestParses(grammar, language, K, callbacks=[("PN", parseNames(grammar, language)), ("Symb", parseUnknown(grammar, language))]);
+    for sent, (time, parsesBlock) in itertools.izip(buf, itertools.imap(parser, sentences)):
+	yield (sent, parsesBlock);
+
+def translation_pipeline(props):
+    if props.propsfile:
+	props = readTranslationPipelineOptions(props.propsfile, props);
+
+    # UGLY HACK FOR K-best translation: if K-best translation output format is only txt
+    if props.bestK != 1:
+	props.format = 'txt';
+
+    if not os.path.isdir( props.exp_directory ):
+	logging.info("Creating output directory: %s" %(props.exp_directory));
+	os.makedirs(props.exp_directory);
+    
+    if not props.srclang:
+	logging.critical("Mandatory option source-lang missing. Can not determine source language.");
+	sys.exit(1);
+    
+    grammar = pgf.readPGF(props.pgffile);
+    
+    sourceLanguage = filter(None, [lang if lang[-3:] == props.srclang else '' for lang in grammar.languages.keys()])[0];
+    logging.info("Translating from %s" %(sourceLanguage));
+    
+    if len(props.tgtlangs):
+	target_langs = props.tgtlangs;
+    else:
+	target_langs = filter(None, [lang[-3:] if lang != sourceLanguage else '' for lang in grammar.languages.keys()]);
+    targetLanguages = filter(None, [lang if lang[-3:] in target_langs else '' for lang in grammar.languages.keys()]);
+    logging.info("Translating into the following languages: %s" %(','.join(targetLanguages)));
+    
+    K = props.bestK if props.bestK != 1 else 20; # by default we look for 20 best parses
+    bestK = props.bestK;
+
+    if not props.input:
+	logging.info( "Input file name missing. Reading input from stdin." );
+	inputStream = sys.stdin;
+	outputPrefix = os.getpid();
+	
+    else:
+	inputStream = codecs.open(props.input, 'r');
+	outputPrefix = os.path.splitext( os.path.split(props.input)[1] )[0];
+    
+    if props.format == 'sgm':
+	inputDoc    = etree.parse(inputStream);
+	reader      = sgmReader;
+	skeletonDoc = getXMLSkeleton;
+	addItem     = addToSgm;
+	writer      = sgmWriter;
+    elif props.format == 'txt':
+	logging.info("Input format is txt. Assuming one-sentence-per-line format.");
+	inputDoc    = inputStream;
+	reader      = lambda X: X;
+	skeletonDoc = lambda X, lang: list();
+	addItem     = lambda X, y: list.append(X, y); 
+	writer      = lambda X: ('\n'.join(X) if bestK == 1 else '\n'.join(map(gf_utils.printMosesNbestFormat, X)));
+    
+    translationBlocks = {};
+    for tgtlang in targetLanguages+['abstract']:
+	translationBlocks[tgtlang] = skeletonDoc(inputDoc, tgtlang);
+
+    preprocessor  = pipeline_lexer;
+    postprocessor = clean_gfstrings;
+
+    logging.info( "Parsing text in %s" %(sourceLanguage) );
+    # 1. Get Abstract Trees for sentences in source language.
+    tokenized_sentences = itertools.imap(preprocessor, reader(inputDoc));
+    absParses  = [parsesBlock for parsesBlock in pipelineParsing(grammar, sourceLanguage, web_lexer(grammar, sourceLanguage, tokenized_sentences), K)];
+
+    logging.info( "Linearizing into %s" %(','.join(targetLanguages)) );
+    # 2. Linearize in all target Languages
+    for idx, parsesBlock in enumerate( itertools.imap(operator.itemgetter(1), absParses) ):
+	translationBuffer = {};
+	if not len(parsesBlock):
+	    # failed to parse;
+	    # translate using lookup
+	    for tgtlang, translation in translationByLookup_(grammar, sourceLanguage, targetLanguages, absParses[idx][0]):
+		if bestK == 1:
+		    addItem(translationBlocks[tgtlang], postprocessor(translation));
+		else:
+		    addItem(translationBlocks[tgtlang], [((0,), postprocessor(translation))]);
+	    addItem(translationBlocks['abstract'], '');
+	else:
+	    bestTranslationIdx = 0;
+	    for tgtlang in targetLanguages:
+		translationBuffer[tgtlang] = gf_utils.getKLinearizations(grammar, tgtlang, [parsesBlock]).next();
+		if bestK == 1:
+		    for tidx, translation in enumerate(translationBuffer[tgtlang]):
+			if postprocessor(translation[1]).strip():
+			    if tidx > bestTranslationIdx:
+				bestTranslationIdx = tidx;
+				break;
+	    for tgtlang in targetLanguages:
+		if bestK == 1:
+		    translation = postprocessor(translationBuffer[tgtlang][bestTranslationIdx][1]) if len(translationBuffer[tgtlang]) > bestTranslationIdx else ((None,), '');
+		    abstract = str(parsesBlock[bestTranslationIdx][1]);
+		else:
+		    translation = translationBuffer[tgtlang] if len(translationBuffer[tgtlang]) else [];
+		    abstract = parsesBlock;
+		addItem(translationBlocks[tgtlang], translation);
+	    addItem(translationBlocks['abstract'], abstract);
+
+    for tgtlang in targetLanguages+['abstract']:
+	outputFile = os.path.join( props.exp_directory, '%s-%s.%s' %(outputPrefix, tgtlang[-3:] if tgtlang!='abstract' else 'abstract', props.format) );
+	logging.info( "Writing translations for %s to %s" %(tgtlang, outputFile) );
+	with codecs.open(outputFile, 'w') as outputStream:
+	    print >>outputStream, writer(translationBlocks[tgtlang]);
+    return;
+
+def cmdLineParser():
+    argparser = argparse.ArgumentParser(prog='translation_pipeline.py', description='Run the GF translation pipeline on standard test-sets');
+    argparser.add_argument('-g', '--pgf', dest='pgffile', required=True, help='PGF grammar file to run the pipeline');
+    argparser.add_argument('-s', '--source', dest='srclang', default='', help='Source language of input sentences');
+    argparser.add_argument('-t', '--target', dest='tgtlangs', nargs='*', default=[], help='Target languages to linearize (default is all other languages)');
+    argparser.add_argument('-i', '--input', dest='input', default='', help='input file (default will accept STDIN)');
+    argparser.add_argument('-e', '--exp', dest='exp_directory', default=os.getcwd(), help='experiement directory to write translation files');
+    argparser.add_argument('-f', '--format', dest='format', default='txt', choices=['txt', 'sgm'], help='input file format (output files will be written in the same format)');
+    argparser.add_argument('-p', '--props', dest='propsfile', default='', help='properties file for the translation pipeline (specify the above arguments in a file)');
+    argparser.add_argument('-K', dest='bestK', type=int, default=1, help='K value for K-best translation');
+    return argparser;
+
+if __name__ == '__main__':
+    logging.basicConfig(level='INFO');
+    pipelineEnv = cmdLineParser().parse_args(sys.argv[1:]);
+    translation_pipeline(pipelineEnv);