AbstractTokenMaker), so this class has been modified so that
+ * Strings are never allocated (via yytext()), and the scanner never has to
+ * worry about refilling its buffer (needlessly copying chars around).
+ * We can achieve this because RText always scans exactly 1 line of tokens at a
+ * time, and hands the scanner this line as an array of characters (a Segment
+ * really). Since tokens contain pointers to char arrays instead of Strings
+ * holding their contents, there is no need for allocating new memory for
+ * Strings.+ * + * The actual algorithm generated for scanning has, of course, not been + * modified.
+ * + * If you wish to regenerate this file yourself, keep in mind the following: + *
PlainTextTokenMaker.java file will contain
+ * two definitions of both zzRefill and yyreset.
+ * You should hand-delete the second of each definition (the ones
+ * generated by the lexer), as these generated methods modify the input
+ * buffer, which we'll never have to do.yylex() on the generated scanner
+ * directly; rather, you should use getTokenList as you would
+ * with any other TokenMaker instance.When a lead/high surrogate has been read from the input stream into the final
+ * {@link #zzBuffer} position, this will have a value of 1; otherwise, it will have a value of 0.
+ */
+ private int zzFinalHighSurrogate = 0;
+
+ /** Number of newlines encountered up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private int yyline;
+
+ /** Number of characters from the last newline up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private int yycolumn;
+
+ /** Number of characters up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private long yychar;
+
+ /** Whether the scanner is currently at the beginning of a line. */
+ @SuppressWarnings("unused")
+ private boolean zzAtBOL = true;
+
+ /** Whether the user-EOF-code has already been executed. */
+ @SuppressWarnings("unused")
+ private boolean zzEOFDone;
+
+ /* user code: */
+
+
+ /**
+ * Constructor. This must be here because JFlex does not generate a
+ * no-parameter constructor.
+ */
+ public PlainTextTokenMaker() {
+ }
+
+
+
+
+ /**
+ * Always returns TokenTypes.NULL, as there are no multiline
+ * tokens in properties files.
+ *
+ * @param text The line of tokens to examine.
+ * @param initialTokenType The token type to start with (i.e., the value
+ * of getLastTokenTypeOnLine for the line before
+ * text).
+ * @return TokenTypes.NULL.
+ */
+ public int getLastTokenTypeOnLine(Segment text, int initialTokenType) {
+ return TokenTypes.NULL;
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @see #addToken(int, int, int)
+ */
+ private void addHyperlinkToken(int start, int end, int tokenType) {
+ int so = start + offsetShift;
+ addToken(zzBuffer, start,end, tokenType, so, true);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ */
+ private void addToken(int tokenType) {
+ addToken(zzStartRead, zzMarkedPos-1, tokenType);
+ }
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @param link Whether this token is a hyperlink.
+ */
+ private void addToken(int tokenType, boolean link) {
+ int so = zzStartRead + offsetShift;
+ super.addToken(zzBuffer, zzStartRead,zzMarkedPos-1, tokenType, so, link);
+ zzStartRead = zzMarkedPos;
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @see #addHyperlinkToken(int, int, int)
+ */
+ private void addToken(int start, int end, int tokenType) {
+ int so = start + offsetShift;
+ addToken(zzBuffer, start,end, tokenType, so, false);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param array The character array.
+ * @param start The starting offset in the array.
+ * @param end The ending offset in the array.
+ * @param tokenType The token's type.
+ * @param startOffset The offset in the document at which this token
+ * occurs.
+ * @param hyperlink Whether this token is a hyperlink.
+ */
+ @Override
+ public void addToken(char[] array, int start, int end, int tokenType, int startOffset, boolean hyperlink) {
+ super.addToken(array, start,end, tokenType, startOffset, hyperlink);
+ zzStartRead = zzMarkedPos;
+ }
+
+
+ /**
+ * Returns the text to place at the beginning and end of a
+ * line to "comment" it in a this programming language.
+ *
+ * @return null, as there are no comments in plain text.
+ */
+ @Override
+ public String[] getLineCommentStartAndEnd(int languageIndex) {
+ return new String[] { "//", null };
+ }
+
+
+ /**
+ * Always returns false, as you never want "mark occurrences"
+ * working in plain text files.
+ *
+ * @param type The token type.
+ * @return Whether tokens of this type should have "mark occurrences"
+ * enabled.
+ */
+ @Override
+ public boolean getMarkOccurrencesOfTokenType(int type) {
+ return false;
+ }
+
+
+ /**
+ * Returns the first token in the linked list of tokens generated
+ * from text. This method must be implemented by
+ * subclasses so they can correctly implement syntax highlighting.
+ *
+ * @param text The text from which to get tokens.
+ * @param initialTokenType The token type we should start with.
+ * @param startOffset The offset into the document at which
+ * text starts.
+ * @return The first Token in a linked list representing
+ * the syntax highlighted text.
+ */
+ public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
+
+ resetTokenList();
+ this.offsetShift = -text.offset + startOffset;
+
+ // Start off in the proper state.
+ s = text;
+ try {
+ yyreset(zzReader);
+ yybegin(YYINITIAL);
+ return yylex();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return new TokenImpl();
+ }
+
+ }
+
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return true if EOF was reached, otherwise
+ * false.
+ */
+ private boolean zzRefill() {
+ return zzCurrentPos>=s.offset+s.count;
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * cannot be reused (internal buffer is discarded and lost).
+ * Lexical state is set to YY_INITIAL.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(java.io.Reader reader) {
+ // 's' has been updated.
+ zzBuffer = s.array;
+ /*
+ * We replaced the line below with the two below it because zzRefill
+ * no longer "refills" the buffer (since the way we do it, it's always
+ * "full" the first time through, since it points to the segment's
+ * array). So, we assign zzEndRead here.
+ */
+ //zzStartRead = zzEndRead = s.offset;
+ zzStartRead = s.offset;
+ zzEndRead = zzStartRead + s.count - 1;
+ zzCurrentPos = zzMarkedPos = s.offset;
+ zzLexicalState = YYINITIAL;
+ zzReader = reader;
+ // zzAtBOL = true;
+ zzAtEOF = false;
+ }
+
+
+
+
+ /**
+ * Creates a new scanner
+ *
+ * @param in the java.io.Reader to read input from.
+ */
+ public PlainTextTokenMaker(java.io.Reader in) {
+ this.zzReader = in;
+ }
+
+ /**
+ * Translates raw input code points to DFA table row
+ */
+ private static int zzCMap(int input) {
+ int offset = input & 255;
+ return offset == input ? ZZ_CMAP_BLOCKS[offset] : ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input >> 8] | offset];
+ }
+
+
+ /**
+ * Closes the input reader.
+ *
+ * @throws java.io.IOException if the reader could not be closed.
+ */
+ public final void yyclose() throws java.io.IOException {
+ zzAtEOF = true; // indicate end of file
+ zzEndRead = zzStartRead; // invalidate buffer
+
+ if (zzReader != null) {
+ zzReader.close();
+ }
+ }
+
+
+ /**
+ * Resets the input position.
+ */
+ private final void yyResetPosition() {
+ zzAtBOL = true;
+ zzAtEOF = false;
+ zzCurrentPos = 0;
+ zzMarkedPos = 0;
+ zzStartRead = 0;
+ zzEndRead = 0;
+ zzFinalHighSurrogate = 0;
+ yyline = 0;
+ yycolumn = 0;
+ yychar = 0L;
+ }
+
+
+ /**
+ * Returns whether the scanner has reached the end of the reader it reads from.
+ *
+ * @return whether the scanner has reached EOF.
+ */
+ public final boolean yyatEOF() {
+ return zzAtEOF;
+ }
+
+
+ /**
+ * Returns the current lexical state.
+ *
+ * @return the current lexical state.
+ */
+ public final int yystate() {
+ return zzLexicalState;
+ }
+
+
+ /**
+ * Enters a new lexical state.
+ *
+ * @param newState the new lexical state
+ */
+ public final void yybegin(int newState) {
+ zzLexicalState = newState;
+ }
+
+
+ /**
+ * Returns the text matched by the current regular expression.
+ *
+ * @return the matched text.
+ */
+ public final String yytext() {
+ return new String(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+ }
+
+
+ /**
+ * Returns the character at the given position from the matched text.
+ *
+ *
It is equivalent to {@code yytext().charAt(pos)}, but faster. + * + * @param position the position of the character to fetch. A value from 0 to {@code yylength()-1}. + * + * @return the character at {@code position}. + */ + public final char yycharat(int position) { + return zzBuffer[zzStartRead + position]; + } + + + /** + * How many characters were matched. + * + * @return the length of the matched text region. + */ + public final int yylength() { + return zzMarkedPos-zzStartRead; + } + + + /** + * Reports an error that occurred while scanning. + * + *
In a well-formed scanner (no or only correct usage of {@code yypushback(int)} and a + * match-all fallback rule) this method will only be called with things that + * "Can't Possibly Happen". + * + *
If this method is called, something is seriously wrong (e.g. a JFlex bug producing a faulty + * scanner etc.). + * + *
Usual syntax/scanner level error handling should be done in error fallback rules. + * + * @param errorCode the code of the error message to display. + */ + private static void zzScanError(int errorCode) { + String message; + try { + message = ZZ_ERROR_MSG[errorCode]; + } catch (ArrayIndexOutOfBoundsException e) { + message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; + } + + throw new Error(message); + } + + + /** + * Pushes the specified amount of characters back into the input stream. + * + *
They will be read again by then next call of the scanning method.
+ *
+ * @param number the number of characters to be read again. This number must not be greater than
+ * {@link #yylength()}.
+ */
+ public void yypushback(int number) {
+ if ( number > yylength() )
+ zzScanError(ZZ_PUSHBACK_2BIG);
+
+ zzMarkedPos -= number;
+ }
+
+
+
+
+ /**
+ * Resumes scanning until the next regular expression is matched, the end of input is encountered
+ * or an I/O-Error occurs.
+ *
+ * @return the next token.
+ * @exception java.io.IOException if any I/O-Error occurs.
+ */
+ public org.fife.ui.rsyntaxtextarea.Token yylex() throws java.io.IOException {
+ int zzInput;
+ int zzAction;
+
+ // cached fields:
+ int zzCurrentPosL;
+ int zzMarkedPosL;
+ int zzEndReadL = zzEndRead;
+ char[] zzBufferL = zzBuffer;
+
+ int [] zzTransL = ZZ_TRANS;
+ int [] zzRowMapL = ZZ_ROWMAP;
+ int [] zzAttrL = ZZ_ATTRIBUTE;
+
+ while (true) {
+ zzMarkedPosL = zzMarkedPos;
+
+ zzAction = -1;
+
+ zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+
+ zzState = ZZ_LEXSTATE[zzLexicalState];
+
+ // set up zzAction for empty match case:
+ int zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ }
+
+
+ zzForAction: {
+ while (true) {
+
+ if (zzCurrentPosL < zzEndReadL) {
+ zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
+ zzCurrentPosL += Character.charCount(zzInput);
+ }
+ else if (zzAtEOF) {
+ zzInput = YYEOF;
+ break zzForAction;
+ }
+ else {
+ // store back cached positions
+ zzCurrentPos = zzCurrentPosL;
+ zzMarkedPos = zzMarkedPosL;
+ boolean eof = zzRefill();
+ // get translated positions and possibly new buffer
+ zzCurrentPosL = zzCurrentPos;
+ zzMarkedPosL = zzMarkedPos;
+ zzBufferL = zzBuffer;
+ zzEndReadL = zzEndRead;
+ if (eof) {
+ zzInput = YYEOF;
+ break zzForAction;
+ }
+ else {
+ zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
+ zzCurrentPosL += Character.charCount(zzInput);
+ }
+ }
+ int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMap(zzInput) ];
+ if (zzNext == -1) break zzForAction;
+ zzState = zzNext;
+
+ zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ zzMarkedPosL = zzCurrentPosL;
+ if ( (zzAttributes & 8) == 8 ) break zzForAction;
+ }
+
+ }
+ }
+
+ // store back cached position
+ zzMarkedPos = zzMarkedPosL;
+
+ if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+ zzAtEOF = true;
+ switch (zzLexicalState) {
+ case YYINITIAL: {
+ addNullToken(); return firstToken;
+ } // fall though
+ case 100: break;
+ case MLC: {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_MULTILINE); return firstToken;
+ } // fall though
+ case 101: break;
+ case DOCCOMMENT: {
+ yybegin(YYINITIAL); addToken(start,zzEndRead, TokenTypes.COMMENT_DOCUMENTATION); return firstToken;
+ } // fall though
+ case 102: break;
+ case EOL_COMMENT: {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_EOL); addNullToken(); return firstToken;
+ } // fall though
+ case 103: break;
+ case TEXT_BLOCK: {
+ addToken(start,zzStartRead-1, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE); return firstToken;
+ } // fall though
+ case 104: break;
+ default:
+ return null;
+ }
+ }
+ else {
+ switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
+ case 1:
+ { addToken(TokenTypes.IDENTIFIER, false);
+ }
+ // fall through
+ case 22: break;
+ case 2:
+ { addToken(TokenTypes.WHITESPACE, false);
+ }
+ // fall through
+ case 23: break;
+ case 3:
+ { addNullToken(); return firstToken;
+ }
+ // fall through
+ case 24: break;
+ case 4:
+ {
+ }
+ // fall through
+ case 25: break;
+ case 5:
+ { addToken(start,zzStartRead-1, TokenTypes.COMMENT_MULTILINE); return firstToken;
+ }
+ // fall through
+ case 26: break;
+ case 6:
+ { addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION); return firstToken;
+ }
+ // fall through
+ case 27: break;
+ case 7:
+ { addToken(start,zzStartRead-1, TokenTypes.COMMENT_EOL); addNullToken(); return firstToken;
+ }
+ // fall through
+ case 28: break;
+ case 8:
+ { addToken(start,zzStartRead-1, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE); return firstToken;
+ }
+ // fall through
+ case 29: break;
+ case 9:
+ { /* Skip escaped chars, handles case: '\"""'. */
+ }
+ // fall through
+ case 30: break;
+ case 10:
+ { start = zzMarkedPos-2; yybegin(MLC);
+ }
+ // fall through
+ case 31: break;
+ case 11:
+ { start = zzMarkedPos-2; yybegin(EOL_COMMENT);
+ }
+ // fall through
+ case 32: break;
+ case 12:
+ { yybegin(YYINITIAL); addToken(start,zzStartRead+1, TokenTypes.COMMENT_MULTILINE);
+ }
+ // fall through
+ case 33: break;
+ case 13:
+ { yybegin(YYINITIAL); addToken(start,zzStartRead+1, TokenTypes.COMMENT_DOCUMENTATION);
+ }
+ // fall through
+ case 34: break;
+ case 14:
+ { int temp=zzStartRead; addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION); addToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_MARKUP); start = zzMarkedPos;
+ }
+ // fall through
+ case 35: break;
+ case 15:
+ { start = zzMarkedPos-3; yybegin(DOCCOMMENT);
+ }
+ // fall through
+ case 36: break;
+ case 16:
+ { yybegin(YYINITIAL); addToken(start,zzStartRead+2, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE);
+ }
+ // fall through
+ case 37: break;
+ case 17:
+ { addToken(TokenTypes.COMMENT_MULTILINE);
+ }
+ // fall through
+ case 38: break;
+ case 18:
+ { addToken(TokenTypes.IDENTIFIER, true);
+ }
+ // fall through
+ case 39: break;
+ case 19:
+ { int temp=zzStartRead; addToken(start,zzStartRead-1, TokenTypes.COMMENT_MULTILINE); addHyperlinkToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_MULTILINE); start = zzMarkedPos;
+ }
+ // fall through
+ case 40: break;
+ case 20:
+ { int temp = zzStartRead;
+ if (start <= zzStartRead - 1) {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION);
+ }
+ addHyperlinkToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_DOCUMENTATION);
+ start = zzMarkedPos;
+ }
+ // fall through
+ case 41: break;
+ case 21:
+ { int temp=zzStartRead; addToken(start,zzStartRead-1, TokenTypes.COMMENT_EOL); addHyperlinkToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_EOL); start = zzMarkedPos;
+ }
+ // fall through
+ case 42: break;
+ default:
+ zzScanError(ZZ_NO_MATCH);
+ }
+ }
+ }
+ }
+
+
+}
diff --git a/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java b/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java
index 12643f3..eb63ab4 100644
--- a/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java
+++ b/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java
@@ -41,7 +41,8 @@
AbstractTokenMakerFactory atmf = (AbstractTokenMakerFactory)TokenMakerFactory.getDefaultInstance();
atmf.putMapping("text/terminal", "istlab.KisoJikken.Test.SimpleTokenMaker");
atmf.putMapping("text/custom", "istlab.KisoJikken.Test.MyCustomTokenMaker");
- textArea.setSyntaxEditingStyle("text/terminal");
+ atmf.putMapping("text/plain", "istlab.KisoJikken.Test.PlainTextTokenMaker");
+ textArea.setSyntaxEditingStyle("text/plain");
// textArea.setCurrentLineHighlightColor(null); // カーソル行のハイライト色
@@ -55,7 +56,7 @@
scheme.getStyle(Token.COMMENT_MARKUP).background = Color.gray;
scheme.getStyle(Token.COMMENT_MULTILINE).background = Color.yellow;
- textArea.setText("initial text\ncomment // com");
+ textArea.setText("initial text\ncomment // com\n https://www.istlab.info/ ");
textArea.revalidate();
// textArea.setSyntaxEditingStyle(SyntaxConstants.SYNTAX_STYLE_NONE);
diff --git a/src/main/java/istlab/KisoJikken/Test/plaintext.flex b/src/main/java/istlab/KisoJikken/Test/plaintext.flex
new file mode 100644
index 0000000..afdda94
--- /dev/null
+++ b/src/main/java/istlab/KisoJikken/Test/plaintext.flex
@@ -0,0 +1,365 @@
+/*
+ * 11/07/2008
+ *
+ * PlainTextTokenMaker.flex - Scanner for plain text files.
+ *
+ * This library is distributed under a modified BSD license. See the included
+ * LICENSE file for details.
+ *
+ * https://github.com/bobbylight/RSyntaxTextArea/wiki/Adding-Syntax-Highlighting-for-a-new-Language
+ * There are two zzRefill() and yyreset() methods
+ * You need to delete the second of each definition (the ones generated by the lexer)
+ * 後の2つを消す
+ */
+package istlab.KisoJikken.Test;
+//package org.fife.ui.rsyntaxtextarea.modes;
+
+import java.io.*;
+import javax.swing.text.Segment;
+
+import org.fife.ui.rsyntaxtextarea.*;
+
+
+/**
+ * Scanner for plain text files.
+ *
+ * This implementation was created using
+ * JFlex 1.4.1; however, the generated file
+ * was modified for performance. Memory allocation needs to be almost
+ * completely removed to be competitive with the handwritten lexers (subclasses
+ * of AbstractTokenMaker), so this class has been modified so that
+ * Strings are never allocated (via yytext()), and the scanner never has to
+ * worry about refilling its buffer (needlessly copying chars around).
+ * We can achieve this because RText always scans exactly 1 line of tokens at a
+ * time, and hands the scanner this line as an array of characters (a Segment
+ * really). Since tokens contain pointers to char arrays instead of Strings
+ * holding their contents, there is no need for allocating new memory for
+ * Strings.
+ * + * The actual algorithm generated for scanning has, of course, not been + * modified.
+ * + * If you wish to regenerate this file yourself, keep in mind the following: + *
PlainTextTokenMaker.java file will contain
+ * two definitions of both zzRefill and yyreset.
+ * You should hand-delete the second of each definition (the ones
+ * generated by the lexer), as these generated methods modify the input
+ * buffer, which we'll never have to do.yylex() on the generated scanner
+ * directly; rather, you should use getTokenList as you would
+ * with any other TokenMaker instance.TokenTypes.NULL, as there are no multiline
+ * tokens in properties files.
+ *
+ * @param text The line of tokens to examine.
+ * @param initialTokenType The token type to start with (i.e., the value
+ * of getLastTokenTypeOnLine for the line before
+ * text).
+ * @return TokenTypes.NULL.
+ */
+ public int getLastTokenTypeOnLine(Segment text, int initialTokenType) {
+ return TokenTypes.NULL;
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @see #addToken(int, int, int)
+ */
+ private void addHyperlinkToken(int start, int end, int tokenType) {
+ int so = start + offsetShift;
+ addToken(zzBuffer, start,end, tokenType, so, true);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ */
+ private void addToken(int tokenType) {
+ addToken(zzStartRead, zzMarkedPos-1, tokenType);
+ }
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @param link Whether this token is a hyperlink.
+ */
+ private void addToken(int tokenType, boolean link) {
+ int so = zzStartRead + offsetShift;
+ super.addToken(zzBuffer, zzStartRead,zzMarkedPos-1, tokenType, so, link);
+ zzStartRead = zzMarkedPos;
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @see #addHyperlinkToken(int, int, int)
+ */
+ private void addToken(int start, int end, int tokenType) {
+ int so = start + offsetShift;
+ addToken(zzBuffer, start,end, tokenType, so, false);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param array The character array.
+ * @param start The starting offset in the array.
+ * @param end The ending offset in the array.
+ * @param tokenType The token's type.
+ * @param startOffset The offset in the document at which this token
+ * occurs.
+ * @param hyperlink Whether this token is a hyperlink.
+ */
+ @Override
+ public void addToken(char[] array, int start, int end, int tokenType, int startOffset, boolean hyperlink) {
+ super.addToken(array, start,end, tokenType, startOffset, hyperlink);
+ zzStartRead = zzMarkedPos;
+ }
+
+
+ /**
+ * Returns the text to place at the beginning and end of a
+ * line to "comment" it in a this programming language.
+ *
+ * @return null, as there are no comments in plain text.
+ */
+ @Override
+ public String[] getLineCommentStartAndEnd(int languageIndex) {
+ return new String[] { "//", null };
+ }
+
+
+ /**
+ * Always returns false, as you never want "mark occurrences"
+ * working in plain text files.
+ *
+ * @param type The token type.
+ * @return Whether tokens of this type should have "mark occurrences"
+ * enabled.
+ */
+ @Override
+ public boolean getMarkOccurrencesOfTokenType(int type) {
+ return false;
+ }
+
+
+ /**
+ * Returns the first token in the linked list of tokens generated
+ * from text. This method must be implemented by
+ * subclasses so they can correctly implement syntax highlighting.
+ *
+ * @param text The text from which to get tokens.
+ * @param initialTokenType The token type we should start with.
+ * @param startOffset The offset into the document at which
+ * text starts.
+ * @return The first Token in a linked list representing
+ * the syntax highlighted text.
+ */
+ public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
+
+ resetTokenList();
+ this.offsetShift = -text.offset + startOffset;
+
+ // Start off in the proper state.
+ s = text;
+ try {
+ yyreset(zzReader);
+ yybegin(YYINITIAL);
+ return yylex();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return new TokenImpl();
+ }
+
+ }
+
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return true if EOF was reached, otherwise
+ * false.
+ */
+ private boolean zzRefill() {
+ return zzCurrentPos>=s.offset+s.count;
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * cannot be reused (internal buffer is discarded and lost).
+ * Lexical state is set to YY_INITIAL.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(java.io.Reader reader) {
+ // 's' has been updated.
+ zzBuffer = s.array;
+ /*
+ * We replaced the line below with the two below it because zzRefill
+ * no longer "refills" the buffer (since the way we do it, it's always
+ * "full" the first time through, since it points to the segment's
+ * array). So, we assign zzEndRead here.
+ */
+ //zzStartRead = zzEndRead = s.offset;
+ zzStartRead = s.offset;
+ zzEndRead = zzStartRead + s.count - 1;
+ zzCurrentPos = zzMarkedPos = s.offset;
+ zzLexicalState = YYINITIAL;
+ zzReader = reader;
+ // zzAtBOL = true;
+ zzAtEOF = false;
+ }
+
+
+%}
+
+LetterOrDigit = ([a-zA-Z0-9])
+Identifier = ({LetterOrDigit}+)
+Separator = ([^a-zA-Z0-9 \t\n])
+WhiteSpace = ([ \t]+)
+LineTerminator = ([\n])
+
+
+Letter = ([A-Za-z])
+/* LetterOrUnderscore = ({Letter}|"_") */
+MLCBegin = "/*"
+MLCEnd = "*/"
+DocCommentBegin = "/**"
+LineCommentBegin = "//"
+
+URLGenDelim = ([:\/\?#\[\]@])
+URLSubDelim = ([\!\$&'\(\)\*\+,;=])
+URLUnreserved = ({LetterOrDigit}|[_\-\.\~])
+URLCharacter = ({URLGenDelim}|{URLSubDelim}|{URLUnreserved}|[%])
+URLCharacters = ({URLCharacter}*)
+URLEndCharacter = ([\/\$]|{LetterOrDigit})
+URL = (((https?|f(tp|ile))"://"|"www.")({URLCharacters}{URLEndCharacter})?)
+
+%state MLC
+%state DOCCOMMENT
+%state EOL_COMMENT
+%state TEXT_BLOCK
+
+%%
+
+