grammar com.hpe.nsdee.lexer.LexerDsl with org.eclipse.xtext.common.Terminals hidden ( COPY_STMT, WS )

generate lexerDsl "http://www.hpe.com/nsdee/lexer/LexerDsl"

Model:
	greetings+=Greeting*;
	
Greeting:
	'Hello' name=ID '!';


terminal COPY_STMT	: 'Copy' -> ';';

Hello virag! 
Copy c:\temp\test2.lex;
Purnam!         
Hello newName! 

Hello anotherName!
Hello 

Hello virag!
Hello anotherName!
Hello 
Purnam!         
Hello newName! 

@Override
	public Class<? extends org.eclipse.xtext.parser.IParser> bindIParser() {
		return RetokenizingCompleteOCLParser.class;
	}

	public static class RetokenizingCompleteOCLParser extends CompleteOCLParser
	{
		@Override
		protected XtextTokenStream createTokenStream(TokenSource tokenSource) {
			return super.createTokenStream(new RetokenizingTokenSource(tokenSource, getTokenDefProvider().getTokenDefMap()));
		}
	}

@Override
	public Class<? extends org.eclipse.xtext.parser.IParser> bindIParser() {
		return RetokenizingParser.class;
	}

	public static class RetokenizingParser extends LexerDslParser
	{
		@Override
		protected XtextTokenStream createTokenStream(TokenSource tokenSource) {
			return super.createTokenStream(new CustomTokenStream(tokenSource, getTokenDefProvider().getTokenDefMap()));
		}
	}

package com.hpe.nsdee.lexer;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.antlr.runtime.ANTLRReaderStream;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenSource;
import org.eclipse.xtext.parser.IParseResult;
import org.eclipse.xtext.parser.antlr.Lexer;
import org.eclipse.xtext.parser.antlr.LexerBindings;
import org.eclipse.xtext.parser.antlr.XtextTokenStream;
import org.eclipse.xtext.resource.XtextResource;
import org.eclipse.xtext.ui.editor.model.XtextDocumentProvider;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.name.Named;

public class CustomTokenStream implements TokenSource {

	protected final TokenSource tokenSource;
	protected final Map<Integer, String> tokenDefMap;
	protected final LinkedList<Token> queue = new LinkedList<Token>();
	public static Integer value = 0;
	public static List<String> words = new ArrayList<>();
	public static Integer lineNumber = 0;
	public static Integer charPositionInLine = 0;
	public static Integer start = 0;
	public static Integer stop = 0;
	
	@Inject
	@Named(LexerBindings.RUNTIME)
	private Provider<Lexer> lexerProvider;

	public CustomTokenStream(TokenSource tokenSource, Map<Integer, String> tokenDefMap) {
		this.tokenSource = tokenSource;
		this.tokenDefMap = tokenDefMap;
		this.words = new ArrayList<>();
		lineNumber = 0;
		charPositionInLine = 0;
		start = 0;
		stop = 0;
	}

	@Override
	public String getSourceName() {
		return tokenSource.getSourceName();
	}

	@Override
	public Token nextToken() {
		while (words.size() > 0) {
			String word = words.get(0);
			CommonToken commonToken = getToken(word);
			words.remove(words.get(0));
			System.out.println(commonToken.toString());
			return commonToken;
		}
		Token nextToken = tokenSource.nextToken();
		if (nextToken.getText() != null && !nextToken.getText().isEmpty() && nextToken.getText().startsWith("Copy")) {			
			List<String> wordsFromFile = readFileAndReturnWords("c:\\temp\\test1.lex");
			words.addAll(wordsFromFile);
			lineNumber = nextToken.getLine();
			start = ((CommonToken) nextToken).getStartIndex();
			stop = ((CommonToken) nextToken).getStopIndex();
			String word = words.get(0);
			charPositionInLine = nextToken.getCharPositionInLine();
			CommonToken commonToken = getToken(word);
			words.remove(words.get(0));
			System.out.println(commonToken.toString());
			return commonToken;
		}
		if(nextToken.getText() == null) {
			return Token.EOF_TOKEN;
		}
		System.out.println(nextToken.toString());
		return nextToken;
	}

	private CommonToken getToken(String word) {
		CommonToken commonToken = null;
		switch (word) {
		case "Hello":
			commonToken = new CommonToken(12, "Hello");
			break;
		case "!":
			commonToken = new CommonToken(13, "!");
			break;
		case "\r\n":
			commonToken = new CommonToken(10, " ");
			break;
		default:
			commonToken = new CommonToken(4, word);
			break;
		}
		commonToken.setLine(lineNumber);		
//		charPositionInLine = charPositionInLine + commonToken.getText().length();
		commonToken.setCharPositionInLine(0);
//		commonToken.setStartIndex(start+1);
//		start = start + commonToken.getText().length();
//		commonToken.setStopIndex(start);
		return commonToken;
	}

	private List<String> readFileAndReturnWords(String filePath) {
		List<String> words = new ArrayList<>();
		try (InputStream fis = new FileInputStream(filePath);
				InputStreamReader isr = new InputStreamReader(fis, Charset.forName("UTF-8"));
				BufferedReader br = new BufferedReader(isr);) {
			String line;
			while ((line = br.readLine()) != null) {
				String[] wordArray = line.split(" ");
				for (String word : wordArray) {
					if (!word.trim().isEmpty()) {
						words.add(word);
						words.add("\r\n");
					}
				}
			}
			return words;

		} catch (Exception e) {
			// TODO: handle exception
		}
		return new ArrayList<>();

	}

}

Hello Aneesh!
Copy C:\\temp\\test1.lex; 
Hello arun!
Hello Anitha!
Hello Senthil!

Hello name1 !
Hello name2 !
Hello name3 !
Hello name4 !
Hello name5 !
Hello name6 !

Hello Aneesh!
Hello name1 !
Hello name2 !
Hello name3 !
Hello name4 !
Hello name5 !
Hello name6 !
Hello arun!
Hello Anitha!
Hello Senthil!