I was writing a keyword scanner for my eclipse editor today. The beginning was easy (as usual) but in the end I had a highlighting issue with my org.eclipse.jface.text.rules.RuleBasedScanner:

The list of language keywords was detected correctly – too correctly for my taste. The editor did not only highlight keywords like “case“, “new” and “import“, it also detected these keywords if they were parts of other tokens like “doImport()” or “renewSomething()“.

After spending several hours only to find out, that there are more of you with exactly the same problem but no (or very complicated) solutions, I want to share a very simple solution I found out after some try and error.

You have to add a predefined default Token to your org.eclipse.jface.text.rules.WordRule. There is a special constructor for this purpose.

Look into line 54 for the solution.

package demoPlugin.editors;

import java.util.ArrayList;
import java.util.List;

import org.eclipse.jface.text.rules.EndOfLineRule;
import org.eclipse.jface.text.rules.IRule;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.IWordDetector;
import org.eclipse.jface.text.rules.MultiLineRule;
import org.eclipse.jface.text.rules.RuleBasedScanner;
import org.eclipse.jface.text.rules.SingleLineRule;
import org.eclipse.jface.text.rules.Token;
import org.eclipse.jface.text.rules.WordRule;

/**
 * @author Lars
 *
 */
public class MyScanner extends RuleBasedScanner {

	private static final String[] dKeywords = {
		"abstract", "alias", "align", "asm", "assert", "auto", "body", "bool",
		"break", "byte", "case", "cast", "catch", "cdouble", "cent", "cfloat",
		"char", "class", "const", "continue", "creal", "dchar", "debug",
		"default", "delegate", "delete", "deprecated", "do", "double", "else",
		"enum", "export", "extern", "false", "final", "finally", "float", "for",
		"foreach", "foreach_reverse", "function", "goto", "idouble", "if",
		"ifloat", "import", "in", "inout", "int", "interface", "invariant",
		"ireal", "is", "lazy", "long", "macro", "mixin", "module", "new",
		"null", "out", "override", "package", "pragma", "private", "protected",
		"public", "real", "ref", "return", "scope", "short", "static", "struct",
		"super", "switch", "synchronized", "template", "this", "throw",
		"__traits", "true", "try", "typedef", "typeid", "typeof", "ubyte",
		"ucent", "uint", "ulong", "union", "unittest", "ushort", "version",
		"void", "volatile", "wchar", "while", "with"
	};

	public DScanner(ColorManager manager) {
		IToken dDefault = new Token(DTextAttribute.DEFAULT.getTextAttribute(manager));
		IToken dKeyword = new Token(DTextAttribute.KEYWORD.getTextAttribute(manager));

		List rules = new ArrayList();
		WordRule wordRule  = new WordRule(new IWordDetector(){
			@Override
			public boolean isWordPart(char c) {
				return Character.isJavaIdentifierPart(c);
			}

			@Override
			public boolean isWordStart(char c) {
				return Character.isJavaIdentifierStart(c);
			}
		}, dDefault);  //HERE IS THE MAGIC!! Add the default token and you're finished.

		for(String operatorString:dKeywords){
			wordRule.addWord(operatorString, dKeyword);
		}
		rules.add(wordRule);
		IRule[] aRule = new IRule[rules.size()];
		rules.toArray(aRule);
		setRules(aRule);
	}
}