AbstractTokenMaker), so this class has been modified so that
+ * Strings are never allocated (via yytext()), and the scanner never has to
+ * worry about refilling its buffer (needlessly copying chars around).
+ * We can achieve this because RText always scans exactly 1 line of tokens at a
+ * time, and hands the scanner this line as an array of characters (a Segment
+ * really). Since tokens contain pointers to char arrays instead of Strings
+ * holding their contents, there is no need for allocating new memory for
+ * Strings.+ * + * The actual algorithm generated for scanning has, of course, not been + * modified.
+ * + * If you wish to regenerate this file yourself, keep in mind the following: + *
PlainTextTokenMaker.java file will contain
+ * two definitions of both zzRefill and yyreset.
+ * You should hand-delete the second of each definition (the ones
+ * generated by the lexer), as these generated methods modify the input
+ * buffer, which we'll never have to do.yylex() on the generated scanner
+ * directly; rather, you should use getTokenList as you would
+ * with any other TokenMaker instance.When a lead/high surrogate has been read from the input stream into the final
+ * {@link #zzBuffer} position, this will have a value of 1; otherwise, it will have a value of 0.
+ */
+ private int zzFinalHighSurrogate = 0;
+
+ /** Number of newlines encountered up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private int yyline;
+
+ /** Number of characters from the last newline up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private int yycolumn;
+
+ /** Number of characters up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private long yychar;
+
+ /** Whether the scanner is currently at the beginning of a line. */
+ @SuppressWarnings("unused")
+ private boolean zzAtBOL = true;
+
+ /** Whether the user-EOF-code has already been executed. */
+ @SuppressWarnings("unused")
+ private boolean zzEOFDone;
+
+ /* user code: */
+
+
+ /**
+ * Constructor. This must be here because JFlex does not generate a
+ * no-parameter constructor.
+ */
+ public MyCustomTokenMaker() {
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @param link Whether this token is a hyperlink.
+ */
+ private void addToken(int tokenType, boolean link) {
+ int so = zzStartRead + offsetShift;
+ super.addToken(zzBuffer, zzStartRead,zzMarkedPos-1, tokenType, so, link);
+ zzStartRead = zzMarkedPos;
+ }
+
+
+ /**
+ * Always returns TokenTypes.NULL, as there are no multiline
+ * tokens in properties files.
+ *
+ * @param text The line of tokens to examine.
+ * @param initialTokenType The token type to start with (i.e., the value
+ * of getLastTokenTypeOnLine for the line before
+ * text).
+ * @return TokenTypes.NULL.
+ */
+ public int getLastTokenTypeOnLine(Segment text, int initialTokenType) {
+ return TokenTypes.NULL;
+ }
+
+
+ /**
+ * Returns the text to place at the beginning and end of a
+ * line to "comment" it in a this programming language.
+ *
+ * @return null, as there are no comments in plain text.
+ */
+ @Override
+ public String[] getLineCommentStartAndEnd(int languageIndex) {
+ return null;
+ }
+
+
+ /**
+ * Always returns false, as you never want "mark occurrences"
+ * working in plain text files.
+ *
+ * @param type The token type.
+ * @return Whether tokens of this type should have "mark occurrences"
+ * enabled.
+ */
+ @Override
+ public boolean getMarkOccurrencesOfTokenType(int type) {
+ return false;
+ }
+
+
+ /**
+ * Returns the first token in the linked list of tokens generated
+ * from text. This method must be implemented by
+ * subclasses so they can correctly implement syntax highlighting.
+ *
+ * @param text The text from which to get tokens.
+ * @param initialTokenType The token type we should start with.
+ * @param startOffset The offset into the document at which
+ * text starts.
+ * @return The first Token in a linked list representing
+ * the syntax highlighted text.
+ */
+ public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
+
+ resetTokenList();
+ this.offsetShift = -text.offset + startOffset;
+
+ // Start off in the proper state.
+ s = text;
+ try {
+ yyreset(zzReader);
+ yybegin(YYINITIAL);
+ return yylex();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return new TokenImpl();
+ }
+
+ }
+
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return true if EOF was reached, otherwise
+ * false.
+ */
+ private boolean zzRefill() {
+ return zzCurrentPos>=s.offset+s.count;
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * cannot be reused (internal buffer is discarded and lost).
+ * Lexical state is set to YY_INITIAL.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(java.io.Reader reader) {
+ // 's' has been updated.
+ zzBuffer = s.array;
+ /*
+ * We replaced the line below with the two below it because zzRefill
+ * no longer "refills" the buffer (since the way we do it, it's always
+ * "full" the first time through, since it points to the segment's
+ * array). So, we assign zzEndRead here.
+ */
+ //zzStartRead = zzEndRead = s.offset;
+ zzStartRead = s.offset;
+ zzEndRead = zzStartRead + s.count - 1;
+ zzCurrentPos = zzMarkedPos = /*zzPushbackPos =*/ s.offset;
+ zzLexicalState = YYINITIAL;
+ zzReader = reader;
+ zzAtBOL = true;
+ zzAtEOF = false;
+ }
+
+
+
+
+ /**
+ * Creates a new scanner
+ *
+ * @param in the java.io.Reader to read input from.
+ */
+ public MyCustomTokenMaker(java.io.Reader in) {
+ this.zzReader = in;
+ }
+
+ /**
+ * Translates raw input code points to DFA table row
+ */
+ private static int zzCMap(int input) {
+ int offset = input & 255;
+ return offset == input ? ZZ_CMAP_BLOCKS[offset] : ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input >> 8] | offset];
+ }
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return {@code false} iff there was new input.
+ * @exception java.io.IOException if any I/O-Error occurs
+ */
+ // private boolean zzRefill() throws java.io.IOException {
+
+ // /* first: make room (if you can) */
+ // if (zzStartRead > 0) {
+ // zzEndRead += zzFinalHighSurrogate;
+ // zzFinalHighSurrogate = 0;
+ // System.arraycopy(zzBuffer, zzStartRead,
+ // zzBuffer, 0,
+ // zzEndRead - zzStartRead);
+
+ // /* translate stored positions */
+ // zzEndRead -= zzStartRead;
+ // zzCurrentPos -= zzStartRead;
+ // zzMarkedPos -= zzStartRead;
+ // zzStartRead = 0;
+ // }
+
+ // /* is the buffer big enough? */
+ // if (zzCurrentPos >= zzBuffer.length - zzFinalHighSurrogate) {
+ // /* if not: blow it up */
+ // char newBuffer[] = new char[zzBuffer.length * 2];
+ // System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
+ // zzBuffer = newBuffer;
+ // zzEndRead += zzFinalHighSurrogate;
+ // zzFinalHighSurrogate = 0;
+ // }
+
+ // /* fill the buffer with new input */
+ // int requested = zzBuffer.length - zzEndRead;
+ // int numRead = zzReader.read(zzBuffer, zzEndRead, requested);
+
+ // /* not supposed to occur according to specification of java.io.Reader */
+ // if (numRead == 0) {
+ // throw new java.io.IOException(
+ // "Reader returned 0 characters. See JFlex examples/zero-reader for a workaround.");
+ // }
+ // if (numRead > 0) {
+ // zzEndRead += numRead;
+ // if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) {
+ // if (numRead == requested) { // We requested too few chars to encode a full Unicode character
+ // --zzEndRead;
+ // zzFinalHighSurrogate = 1;
+ // } else { // There is room in the buffer for at least one more char
+ // int c = zzReader.read(); // Expecting to read a paired low surrogate char
+ // if (c == -1) {
+ // return true;
+ // } else {
+ // zzBuffer[zzEndRead++] = (char)c;
+ // }
+ // }
+ // }
+ // /* potentially more input available */
+ // return false;
+ // }
+
+ // /* numRead < 0 ==> end of stream */
+ // return true;
+ // }
+
+
+ /**
+ * Closes the input reader.
+ *
+ * @throws java.io.IOException if the reader could not be closed.
+ */
+ public final void yyclose() throws java.io.IOException {
+ zzAtEOF = true; // indicate end of file
+ zzEndRead = zzStartRead; // invalidate buffer
+
+ if (zzReader != null) {
+ zzReader.close();
+ }
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ *
+ *
Does not close the old reader. + * + *
All internal variables are reset, the old input stream cannot be reused (internal + * buffer is discarded and lost). Lexical state is set to {@code ZZ_INITIAL}. + * + *
Internal scan buffer is resized down to its initial length, if it has grown. + * + * @param reader The new input stream. + */ + // public final void yyreset(java.io.Reader reader) { + // zzReader = reader; + // zzEOFDone = false; + // yyResetPosition(); + // zzLexicalState = YYINITIAL; + // if (zzBuffer.length > ZZ_BUFFERSIZE) { + // zzBuffer = new char[ZZ_BUFFERSIZE]; + // } + // } + + /** + * Resets the input position. + */ + private final void yyResetPosition() { + zzAtBOL = true; + zzAtEOF = false; + zzCurrentPos = 0; + zzMarkedPos = 0; + zzStartRead = 0; + zzEndRead = 0; + zzFinalHighSurrogate = 0; + yyline = 0; + yycolumn = 0; + yychar = 0L; + } + + + /** + * Returns whether the scanner has reached the end of the reader it reads from. + * + * @return whether the scanner has reached EOF. + */ + public final boolean yyatEOF() { + return zzAtEOF; + } + + + /** + * Returns the current lexical state. + * + * @return the current lexical state. + */ + public final int yystate() { + return zzLexicalState; + } + + + /** + * Enters a new lexical state. + * + * @param newState the new lexical state + */ + public final void yybegin(int newState) { + zzLexicalState = newState; + } + + + /** + * Returns the text matched by the current regular expression. + * + * @return the matched text. + */ + public final String yytext() { + return new String(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); + } + + + /** + * Returns the character at the given position from the matched text. + * + *
It is equivalent to {@code yytext().charAt(pos)}, but faster. + * + * @param position the position of the character to fetch. A value from 0 to {@code yylength()-1}. + * + * @return the character at {@code position}. + */ + public final char yycharat(int position) { + return zzBuffer[zzStartRead + position]; + } + + + /** + * How many characters were matched. + * + * @return the length of the matched text region. + */ + public final int yylength() { + return zzMarkedPos-zzStartRead; + } + + + /** + * Reports an error that occurred while scanning. + * + *
In a well-formed scanner (no or only correct usage of {@code yypushback(int)} and a + * match-all fallback rule) this method will only be called with things that + * "Can't Possibly Happen". + * + *
If this method is called, something is seriously wrong (e.g. a JFlex bug producing a faulty + * scanner etc.). + * + *
Usual syntax/scanner level error handling should be done in error fallback rules. + * + * @param errorCode the code of the error message to display. + */ + private static void zzScanError(int errorCode) { + String message; + try { + message = ZZ_ERROR_MSG[errorCode]; + } catch (ArrayIndexOutOfBoundsException e) { + message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; + } + + throw new Error(message); + } + + + /** + * Pushes the specified amount of characters back into the input stream. + * + *
They will be read again by then next call of the scanning method. + * + * @param number the number of characters to be read again. This number must not be greater than + * {@link #yylength()}. + */ + public void yypushback(int number) { + if ( number > yylength() ) + zzScanError(ZZ_PUSHBACK_2BIG); + + zzMarkedPos -= number; + } + + + + + /** + * Resumes scanning until the next regular expression is matched, the end of input is encountered + * or an I/O-Error occurs. + * + * @return the next token. + * @exception java.io.IOException if any I/O-Error occurs. + */ + public org.fife.ui.rsyntaxtextarea.Token yylex() throws java.io.IOException { + int zzInput; + int zzAction; + + // cached fields: + int zzCurrentPosL; + int zzMarkedPosL; + int zzEndReadL = zzEndRead; + char[] zzBufferL = zzBuffer; + + int [] zzTransL = ZZ_TRANS; + int [] zzRowMapL = ZZ_ROWMAP; + int [] zzAttrL = ZZ_ATTRIBUTE; + + while (true) { + zzMarkedPosL = zzMarkedPos; + + zzAction = -1; + + zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; + + zzState = ZZ_LEXSTATE[zzLexicalState]; + + // set up zzAction for empty match case: + int zzAttributes = zzAttrL[zzState]; + if ( (zzAttributes & 1) == 1 ) { + zzAction = zzState; + } + + + zzForAction: { + while (true) { + + if (zzCurrentPosL < zzEndReadL) { + zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); + zzCurrentPosL += Character.charCount(zzInput); + } + else if (zzAtEOF) { + zzInput = YYEOF; + break zzForAction; + } + else { + // store back cached positions + zzCurrentPos = zzCurrentPosL; + zzMarkedPos = zzMarkedPosL; + boolean eof = zzRefill(); + // get translated positions and possibly new buffer + zzCurrentPosL = zzCurrentPos; + zzMarkedPosL = zzMarkedPos; + zzBufferL = zzBuffer; + zzEndReadL = zzEndRead; + if (eof) { + zzInput = YYEOF; + break zzForAction; + } + else { + zzInput = zzBufferL[zzCurrentPosL++]; +// zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); + zzCurrentPosL += Character.charCount(zzInput); + } + } + int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMap(zzInput) ]; + if (zzNext == -1) break zzForAction; + zzState = zzNext; + + zzAttributes = zzAttrL[zzState]; + if ( (zzAttributes & 1) == 1 ) { + zzAction = zzState; + zzMarkedPosL = zzCurrentPosL; + if ( (zzAttributes & 8) == 8 ) break zzForAction; + } + + } + } + + // store back cached position + zzMarkedPos = zzMarkedPosL; + + if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { + zzAtEOF = true; + switch (zzLexicalState) { + case YYINITIAL: { + addNullToken(); return firstToken; + } // fall though + case 22: break; + default: + return null; + } + } + else { + switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { + case 1: + { addToken(TokenTypes.IDENTIFIER, false); + } + // fall through + case 5: break; + case 2: + { addToken(TokenTypes.WHITESPACE, false); + } + // fall through + case 6: break; + case 3: + { addNullToken(); return firstToken; + } + // fall through + case 7: break; + case 4: + { addToken(TokenTypes.IDENTIFIER, true); + } + // fall through + case 8: break; + default: + zzScanError(ZZ_NO_MATCH); + } + } + } + } + + +} diff --git a/src/main/java/istlab/KisoJikken/Test/SimpleTokenMaker.java b/src/main/java/istlab/KisoJikken/Test/SimpleTokenMaker.java new file mode 100644 index 0000000..c05bb77 --- /dev/null +++ b/src/main/java/istlab/KisoJikken/Test/SimpleTokenMaker.java @@ -0,0 +1,1353 @@ +// DO NOT EDIT +// Generated by JFlex 1.8.2 http://jflex.de/ +// source: simple_token_maker.flex + +/* + * 11/13/2004 + * + * SimpleTokenMaker.java - Scanner for the Java programming language. + * + * This library is distributed under a modified BSD license. See the included + * LICENSE file for details. + */ +package istlab.KisoJikken.Test; + +import java.io.IOException; +import java.io.Reader; + +import javax.swing.text.Segment; + +import org.fife.ui.rsyntaxtextarea.AbstractJFlexCTokenMaker; +import org.fife.ui.rsyntaxtextarea.Token; +import org.fife.ui.rsyntaxtextarea.TokenImpl; +import org.fife.ui.rsyntaxtextarea.TokenTypes; + + +/** + * Scanner for the Java programming language.
+ *
+ * This implementation was created using
+ * JFlex 1.4.1; however, the generated file
+ * was modified for performance. Memory allocation needs to be almost
+ * completely removed to be competitive with the handwritten lexers (subclasses
+ * of AbstractTokenMaker), so this class has been modified so that
+ * Strings are never allocated (via yytext()), and the scanner never has to
+ * worry about refilling its buffer (needlessly copying chars around).
+ * We can achieve this because RText always scans exactly 1 line of tokens at a
+ * time, and hands the scanner this line as an array of characters (a Segment
+ * really). Since tokens contain pointers to char arrays instead of Strings
+ * holding their contents, there is no need for allocating new memory for
+ * Strings.
+ * + * The actual algorithm generated for scanning has, of course, not been + * modified.
+ * + * If you wish to regenerate this file yourself, keep in mind the following: + *
JavaTokenMaker.java file will contain two
+ * definitions of both zzRefill and yyreset.
+ * You should hand-delete the second of each definition (the ones
+ * generated by the lexer), as these generated methods modify the input
+ * buffer, which we'll never have to do.yylex() on the generated scanner
+ * directly; rather, you should use getTokenList as you would
+ * with any other TokenMaker instance.When a lead/high surrogate has been read from the input stream into the final
+ * {@link #zzBuffer} position, this will have a value of 1; otherwise, it will have a value of 0.
+ */
+ private int zzFinalHighSurrogate = 0;
+
+ /** Number of newlines encountered up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private int yyline;
+
+ /** Number of characters from the last newline up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private int yycolumn;
+
+ /** Number of characters up to the start of the matched text. */
+ @SuppressWarnings("unused")
+ private long yychar;
+
+ /** Whether the scanner is currently at the beginning of a line. */
+ @SuppressWarnings("unused")
+ private boolean zzAtBOL = true;
+
+ /** Whether the user-EOF-code has already been executed. */
+ @SuppressWarnings("unused")
+ private boolean zzEOFDone;
+
+ /* user code: */
+
+
+ /**
+ * Constructor. This must be here because JFlex does not generate a
+ * no-parameter constructor.
+ */
+ public SimpleTokenMaker() {
+ }
+
+ /**
+ * Creates a new scanner
+ *
+ * @param in the java.io.Reader to read input from.
+ */
+ public SimpleTokenMaker(Reader in) {
+ this.zzReader = in;
+ }
+ // public SimpleTokenMaker(java.io.Reader in) {
+ // this.zzReader = in;
+ // }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @see #addToken(int, int, int)
+ */
+ private void addHyperlinkToken(int start, int end, int tokenType) {
+ int so = start + offsetShift;
+ addToken(zzBuffer, start,end, tokenType, so, true);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ */
+ private void addToken(int tokenType) {
+ addToken(zzStartRead, zzMarkedPos-1, tokenType);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param tokenType The token's type.
+ * @see #addHyperlinkToken(int, int, int)
+ */
+ private void addToken(int start, int end, int tokenType) {
+ int so = start + offsetShift;
+ addToken(zzBuffer, start,end, tokenType, so, false);
+ }
+
+
+ /**
+ * Adds the token specified to the current linked list of tokens.
+ *
+ * @param array The character array.
+ * @param start The starting offset in the array.
+ * @param end The ending offset in the array.
+ * @param tokenType The token's type.
+ * @param startOffset The offset in the document at which this token
+ * occurs.
+ * @param hyperlink Whether this token is a hyperlink.
+ */
+ @Override
+ public void addToken(char[] array, int start, int end, int tokenType,
+ int startOffset, boolean hyperlink) {
+ super.addToken(array, start,end, tokenType, startOffset, hyperlink);
+ zzStartRead = zzMarkedPos;
+ }
+
+
+ @Override
+ public String[] getLineCommentStartAndEnd(int languageIndex) {
+ return new String[] { "//", null };
+ }
+
+
+ /**
+ * Returns the first token in the linked list of tokens generated
+ * from text. This method must be implemented by
+ * subclasses so they can correctly implement syntax highlighting.
+ *
+ * @param text The text from which to get tokens.
+ * @param initialTokenType The token type we should start with.
+ * @param startOffset The offset into the document at which
+ * text starts.
+ * @return The first Token in a linked list representing
+ * the syntax highlighted text.
+ */
+ public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
+
+ resetTokenList();
+ this.offsetShift = -text.offset + startOffset;
+
+ // Start off in the proper state.
+ int state;
+ switch (initialTokenType) {
+ case TokenTypes.COMMENT_MULTILINE:
+ state = MLC;
+ start = text.offset;
+ break;
+ case TokenTypes.COMMENT_DOCUMENTATION:
+ state = DOCCOMMENT;
+ start = text.offset;
+ break;
+ case TokenTypes.LITERAL_STRING_DOUBLE_QUOTE:
+ state = TEXT_BLOCK;
+ start = text.offset;
+ break;
+ default:
+ state = YYINITIAL;
+ }
+
+ s = text;
+ try {
+ yyreset(zzReader);
+ yybegin(state);
+ return yylex();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return new TokenImpl();
+ }
+
+ }
+
+
+ // /**
+ // * Refills the input buffer.
+ // *
+ // * @return true if EOF was reached, otherwise
+ // * false.
+ // */
+ private boolean zzRefill() {
+ return zzCurrentPos>=s.offset+s.count;
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * cannot be reused (internal buffer is discarded and lost).
+ * Lexical state is set to YY_INITIAL.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(Reader reader) {
+ // 's' has been updated.
+ zzBuffer = s.array;
+ /*
+ * We replaced the line below with the two below it because zzRefill
+ * no longer "refills" the buffer (since the way we do it, it's always
+ * "full" the first time through, since it points to the segment's
+ * array). So, we assign zzEndRead here.
+ */
+ //zzStartRead = zzEndRead = s.offset;
+ zzStartRead = s.offset;
+ zzEndRead = zzStartRead + s.count - 1;
+ zzCurrentPos = zzMarkedPos = /*zzPushbackPos =*/ s.offset;
+ zzLexicalState = YYINITIAL;
+ zzReader = reader;
+ zzAtBOL = true;
+ zzAtEOF = false;
+ }
+
+
+ /**
+ * Translates raw input code points to DFA table row
+ */
+ private static int zzCMap(int input) {
+ int offset = input & 255;
+ return offset == input ? ZZ_CMAP_BLOCKS[offset] : ZZ_CMAP_BLOCKS[ZZ_CMAP_TOP[input >> 8] | offset];
+ }
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return {@code false} iff there was new input.
+ * @exception java.io.IOException if any I/O-Error occurs
+ */
+ // private boolean zzRefill() throws java.io.IOException {
+
+ // /* first: make room (if you can) */
+ // if (zzStartRead > 0) {
+ // zzEndRead += zzFinalHighSurrogate;
+ // zzFinalHighSurrogate = 0;
+ // System.arraycopy(zzBuffer, zzStartRead,
+ // zzBuffer, 0,
+ // zzEndRead - zzStartRead);
+
+ // /* translate stored positions */
+ // zzEndRead -= zzStartRead;
+ // zzCurrentPos -= zzStartRead;
+ // zzMarkedPos -= zzStartRead;
+ // zzStartRead = 0;
+ // }
+
+ // /* is the buffer big enough? */
+ // if (zzCurrentPos >= zzBuffer.length - zzFinalHighSurrogate) {
+ // /* if not: blow it up */
+ // char newBuffer[] = new char[zzBuffer.length * 2];
+ // System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
+ // zzBuffer = newBuffer;
+ // zzEndRead += zzFinalHighSurrogate;
+ // zzFinalHighSurrogate = 0;
+ // }
+
+ // /* fill the buffer with new input */
+ // int requested = zzBuffer.length - zzEndRead;
+ // int numRead = zzReader.read(zzBuffer, zzEndRead, requested);
+
+ // /* not supposed to occur according to specification of java.io.Reader */
+ // if (numRead == 0) {
+ // throw new java.io.IOException(
+ // "Reader returned 0 characters. See JFlex examples/zero-reader for a workaround.");
+ // }
+ // if (numRead > 0) {
+ // zzEndRead += numRead;
+ // if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) {
+ // if (numRead == requested) { // We requested too few chars to encode a full Unicode character
+ // --zzEndRead;
+ // zzFinalHighSurrogate = 1;
+ // } else { // There is room in the buffer for at least one more char
+ // int c = zzReader.read(); // Expecting to read a paired low surrogate char
+ // if (c == -1) {
+ // return true;
+ // } else {
+ // zzBuffer[zzEndRead++] = (char)c;
+ // }
+ // }
+ // }
+ // /* potentially more input available */
+ // return false;
+ // }
+
+ // /* numRead < 0 ==> end of stream */
+ // return true;
+ // }
+
+
+ /**
+ * Closes the input reader.
+ *
+ * @throws java.io.IOException if the reader could not be closed.
+ */
+ public final void yyclose() throws java.io.IOException {
+ zzAtEOF = true; // indicate end of file
+ zzEndRead = zzStartRead; // invalidate buffer
+
+ if (zzReader != null) {
+ zzReader.close();
+ }
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ *
+ *
Does not close the old reader. + * + *
All internal variables are reset, the old input stream cannot be reused (internal + * buffer is discarded and lost). Lexical state is set to {@code ZZ_INITIAL}. + * + *
Internal scan buffer is resized down to its initial length, if it has grown. + * + * @param reader The new input stream. + */ + // public final void yyreset(java.io.Reader reader) { + // zzReader = reader; + // zzEOFDone = false; + // yyResetPosition(); + // zzLexicalState = YYINITIAL; + // if (zzBuffer.length > ZZ_BUFFERSIZE) { + // zzBuffer = new char[ZZ_BUFFERSIZE]; + // } + // } + + /** + * Resets the input position. + */ + private final void yyResetPosition() { + zzAtBOL = true; + zzAtEOF = false; + zzCurrentPos = 0; + zzMarkedPos = 0; + zzStartRead = 0; + zzEndRead = 0; + zzFinalHighSurrogate = 0; + yyline = 0; + yycolumn = 0; + yychar = 0L; + } + + + /** + * Returns whether the scanner has reached the end of the reader it reads from. + * + * @return whether the scanner has reached EOF. + */ + public final boolean yyatEOF() { + return zzAtEOF; + } + + + /** + * Returns the current lexical state. + * + * @return the current lexical state. + */ + public final int yystate() { + return zzLexicalState; + } + + + /** + * Enters a new lexical state. + * + * @param newState the new lexical state + */ + public final void yybegin(int newState) { + zzLexicalState = newState; + } + + + /** + * Returns the text matched by the current regular expression. + * + * @return the matched text. + */ + public final String yytext() { + return new String(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); + } + + + /** + * Returns the character at the given position from the matched text. + * + *
It is equivalent to {@code yytext().charAt(pos)}, but faster. + * + * @param position the position of the character to fetch. A value from 0 to {@code yylength()-1}. + * + * @return the character at {@code position}. + */ + public final char yycharat(int position) { + return zzBuffer[zzStartRead + position]; + } + + + /** + * How many characters were matched. + * + * @return the length of the matched text region. + */ + public final int yylength() { + return zzMarkedPos-zzStartRead; + } + + + /** + * Reports an error that occurred while scanning. + * + *
In a well-formed scanner (no or only correct usage of {@code yypushback(int)} and a + * match-all fallback rule) this method will only be called with things that + * "Can't Possibly Happen". + * + *
If this method is called, something is seriously wrong (e.g. a JFlex bug producing a faulty + * scanner etc.). + * + *
Usual syntax/scanner level error handling should be done in error fallback rules. + * + * @param errorCode the code of the error message to display. + */ + private static void zzScanError(int errorCode) { + String message; + try { + message = ZZ_ERROR_MSG[errorCode]; + } catch (ArrayIndexOutOfBoundsException e) { + message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; + } + + throw new Error(message); + } + + + /** + * Pushes the specified amount of characters back into the input stream. + * + *
They will be read again by then next call of the scanning method.
+ *
+ * @param number the number of characters to be read again. This number must not be greater than
+ * {@link #yylength()}.
+ */
+ public void yypushback(int number) {
+ if ( number > yylength() )
+ zzScanError(ZZ_PUSHBACK_2BIG);
+
+ zzMarkedPos -= number;
+ }
+
+
+
+
+ /**
+ * Resumes scanning until the next regular expression is matched, the end of input is encountered
+ * or an I/O-Error occurs.
+ *
+ * @return the next token.
+ * @exception java.io.IOException if any I/O-Error occurs.
+ */
+ public org.fife.ui.rsyntaxtextarea.Token yylex() throws java.io.IOException {
+ int zzInput;
+ int zzAction;
+
+ // cached fields:
+ int zzCurrentPosL;
+ int zzMarkedPosL;
+ int zzEndReadL = zzEndRead;
+ char[] zzBufferL = zzBuffer;
+
+ int [] zzTransL = ZZ_TRANS;
+ int [] zzRowMapL = ZZ_ROWMAP;
+ int [] zzAttrL = ZZ_ATTRIBUTE;
+
+ while (true) {
+ zzMarkedPosL = zzMarkedPos;
+
+ zzAction = -1;
+
+ zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+
+ zzState = ZZ_LEXSTATE[zzLexicalState];
+
+ // set up zzAction for empty match case:
+ int zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ }
+
+
+ zzForAction: {
+ while (true) {
+
+ if (zzCurrentPosL < zzEndReadL) {
+ zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
+ zzCurrentPosL += Character.charCount(zzInput);
+ }
+ else if (zzAtEOF) {
+ zzInput = YYEOF;
+ break zzForAction;
+ }
+ else {
+ // store back cached positions
+ zzCurrentPos = zzCurrentPosL;
+ zzMarkedPos = zzMarkedPosL;
+ boolean eof = zzRefill();
+ // get translated positions and possibly new buffer
+ zzCurrentPosL = zzCurrentPos;
+ zzMarkedPosL = zzMarkedPos;
+ zzBufferL = zzBuffer;
+ zzEndReadL = zzEndRead;
+ if (eof) {
+ zzInput = YYEOF;
+ break zzForAction;
+ }
+ else {
+ zzInput = zzBufferL[zzCurrentPosL++];
+ // zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
+ zzCurrentPosL += Character.charCount(zzInput);
+ }
+ }
+ int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMap(zzInput) ];
+ if (zzNext == -1) break zzForAction;
+ zzState = zzNext;
+
+ zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ zzMarkedPosL = zzCurrentPosL;
+ if ( (zzAttributes & 8) == 8 ) break zzForAction;
+ }
+
+ }
+ }
+
+ // store back cached position
+ zzMarkedPos = zzMarkedPosL;
+
+ if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+ zzAtEOF = true;
+ switch (zzLexicalState) {
+ case YYINITIAL: {
+ addNullToken(); return firstToken;
+ } // fall though
+ case 216: break;
+ case MLC: {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_MULTILINE); return firstToken;
+ } // fall though
+ case 217: break;
+ case DOCCOMMENT: {
+ yybegin(YYINITIAL); addToken(start,zzEndRead, TokenTypes.COMMENT_DOCUMENTATION); return firstToken;
+ } // fall though
+ case 218: break;
+ case EOL_COMMENT: {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_EOL); addNullToken(); return firstToken;
+ } // fall though
+ case 219: break;
+ case TEXT_BLOCK: {
+ addToken(start,zzStartRead-1, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE); return firstToken;
+ } // fall though
+ case 220: break;
+ default:
+ return null;
+ }
+ }
+ else {
+ switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
+ case 1:
+ {
+ }
+ // fall through
+ case 23: break;
+ case 2:
+ { addToken(TokenTypes.ERROR_IDENTIFIER);
+ }
+ // fall through
+ case 24: break;
+ case 3:
+ { addToken(TokenTypes.WHITESPACE);
+ }
+ // fall through
+ case 25: break;
+ case 4:
+ { addNullToken(); return firstToken;
+ }
+ // fall through
+ case 26: break;
+ case 5:
+ { addToken(TokenTypes.ANNOTATION);
+ }
+ // fall through
+ case 27: break;
+ case 6:
+ { addToken(start,zzStartRead-1, TokenTypes.COMMENT_MULTILINE); return firstToken;
+ }
+ // fall through
+ case 28: break;
+ case 7:
+ { addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION); return firstToken;
+ }
+ // fall through
+ case 29: break;
+ case 8:
+ { addToken(start,zzStartRead-1, TokenTypes.COMMENT_EOL); addNullToken(); return firstToken;
+ }
+ // fall through
+ case 30: break;
+ case 9:
+ { addToken(start,zzStartRead-1, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE); return firstToken;
+ }
+ // fall through
+ case 31: break;
+ case 10:
+ { /* Skip escaped chars, handles case: '\"""'. */
+ }
+ // fall through
+ case 32: break;
+ case 11:
+ { start = zzMarkedPos-2; yybegin(MLC);
+ }
+ // fall through
+ case 33: break;
+ case 12:
+ { start = zzMarkedPos-2; yybegin(EOL_COMMENT);
+ }
+ // fall through
+ case 34: break;
+ case 13:
+ { yybegin(YYINITIAL); addToken(start,zzStartRead+1, TokenTypes.COMMENT_MULTILINE);
+ }
+ // fall through
+ case 35: break;
+ case 14:
+ { yybegin(YYINITIAL); addToken(start,zzStartRead+1, TokenTypes.COMMENT_DOCUMENTATION);
+ }
+ // fall through
+ case 36: break;
+ case 15:
+ { int temp=zzStartRead; addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION); addToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_MARKUP); start = zzMarkedPos;
+ }
+ // fall through
+ case 37: break;
+ case 16:
+ { start = zzMarkedPos-3; yybegin(DOCCOMMENT);
+ }
+ // fall through
+ case 38: break;
+ case 17:
+ { yybegin(YYINITIAL); addToken(start,zzStartRead+2, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE);
+ }
+ // fall through
+ case 39: break;
+ case 18:
+ { addToken(TokenTypes.COMMENT_MULTILINE);
+ }
+ // fall through
+ case 40: break;
+ case 19:
+ { int temp=zzStartRead; addToken(start,zzStartRead-1, TokenTypes.COMMENT_MULTILINE); addHyperlinkToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_MULTILINE); start = zzMarkedPos;
+ }
+ // fall through
+ case 41: break;
+ case 20:
+ { int temp = zzStartRead;
+ if (start <= zzStartRead - 1) {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION);
+ }
+ addToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_KEYWORD);
+ start = zzMarkedPos;
+ }
+ // fall through
+ case 42: break;
+ case 21:
+ { int temp = zzStartRead;
+ if (start <= zzStartRead - 1) {
+ addToken(start,zzStartRead-1, TokenTypes.COMMENT_DOCUMENTATION);
+ }
+ addHyperlinkToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_DOCUMENTATION);
+ start = zzMarkedPos;
+ }
+ // fall through
+ case 43: break;
+ case 22:
+ { int temp=zzStartRead; addToken(start,zzStartRead-1, TokenTypes.COMMENT_EOL); addHyperlinkToken(temp,zzMarkedPos-1, TokenTypes.COMMENT_EOL); start = zzMarkedPos;
+ }
+ // fall through
+ case 44: break;
+ default:
+ zzScanError(ZZ_NO_MATCH);
+ }
+ }
+ }
+ }
+
+
+}
diff --git a/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java b/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java
index 29ff72f..12643f3 100644
--- a/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java
+++ b/src/main/java/istlab/KisoJikken/Test/StyleTestDemo.java
@@ -17,14 +17,11 @@
import javax.swing.UIManager;
import javax.swing.plaf.FontUIResource;
-import org.fife.ui.rsyntaxtextarea.RSyntaxDocument;
+import org.fife.ui.rsyntaxtextarea.AbstractTokenMakerFactory;
import org.fife.ui.rsyntaxtextarea.RSyntaxTextArea;
-import org.fife.ui.rsyntaxtextarea.SyntaxConstants;
import org.fife.ui.rsyntaxtextarea.SyntaxScheme;
import org.fife.ui.rsyntaxtextarea.Token;
-import org.fife.ui.rsyntaxtextarea.TokenMaker;
-import org.fife.ui.rsyntaxtextarea.modes.JavaTokenMaker;
-import org.fife.ui.rsyntaxtextarea.modes.PlainTextTokenMaker;
+import org.fife.ui.rsyntaxtextarea.TokenMakerFactory;
import org.fife.ui.rtextarea.RTextScrollPane;
import org.mdkt.compiler.InMemoryJavaCompiler;
@@ -35,13 +32,21 @@
public StyleTestDemo() {
JPanel contentPane = new JPanel(new BorderLayout());
- RSyntaxDocument doc = new RSyntaxDocument("text/java");
- TokenMaker tokenMaker = new PlainTextTokenMaker();
- doc.setSyntaxStyle(tokenMaker);
- textArea = new RSyntaxTextArea(doc);
+ // RSyntaxDocument doc = new RSyntaxDocument();
+ // TokenMaker tokenMaker = new PlainTextTokenMaker();
+ // TokenMaker tokenMaker = new SimpleTokenMaker();
+ // doc.setSyntaxStyle(tokenMaker);
+ textArea = new RSyntaxTextArea();
+
+ AbstractTokenMakerFactory atmf = (AbstractTokenMakerFactory)TokenMakerFactory.getDefaultInstance();
+ atmf.putMapping("text/terminal", "istlab.KisoJikken.Test.SimpleTokenMaker");
+ atmf.putMapping("text/custom", "istlab.KisoJikken.Test.MyCustomTokenMaker");
+ textArea.setSyntaxEditingStyle("text/terminal");
+
+
// textArea.setCurrentLineHighlightColor(null); // カーソル行のハイライト色
textArea.setHighlightCurrentLine(false); // カーソル行のハイライトを消す
- textArea.setSyntaxEditingStyle(SyntaxConstants.SYNTAX_STYLE_JAVA);
+ // textArea.setSyntaxEditingStyle(SyntaxConstants.SYNTAX_STYLE_NONE);
SyntaxScheme scheme = textArea.getSyntaxScheme();
scheme.getStyle(Token.COMMENT_EOL).background = Color.cyan;
@@ -112,23 +117,23 @@
north.add(execB);
// JOptionPane.showMessageDialog(null, "test");
- JButton javaB = new JButton("Java");
- javaB.addActionListener(e -> {
- doc.setSyntaxStyle(new JavaTokenMaker());
- textArea.setSyntaxEditingStyle("text/c");
- // doc.setSyntaxStyle("text/java");
- // textArea.setDocument(doc);
- } );
- north.add(javaB);
+ // JButton javaB = new JButton("Java");
+ // javaB.addActionListener(e -> {
+ // doc.setSyntaxStyle(new JavaTokenMaker());
+ // textArea.setSyntaxEditingStyle("text/c");
+ // // doc.setSyntaxStyle("text/java");
+ // // textArea.setDocument(doc);
+ // } );
+ // north.add(javaB);
- JButton plainB = new JButton("Plain");
- javaB.addActionListener(e -> {
- doc.setSyntaxStyle(new PlainTextTokenMaker());
- textArea.setSyntaxEditingStyle("text/plain");
- // doc.setSyntaxStyle("text/plain");
- // textArea.setDocument(doc);
- } );
- north.add(plainB);
+ // JButton plainB = new JButton("Plain");
+ // javaB.addActionListener(e -> {
+ // doc.setSyntaxStyle(new PlainTextTokenMaker());
+ // textArea.setSyntaxEditingStyle("text/plain");
+ // // doc.setSyntaxStyle("text/plain");
+ // // textArea.setDocument(doc);
+ // } );
+ // north.add(plainB);
add(north, BorderLayout.NORTH);
diff --git a/src/main/java/istlab/KisoJikken/Test/my_custom_token_maker.flex b/src/main/java/istlab/KisoJikken/Test/my_custom_token_maker.flex
new file mode 100644
index 0000000..cfc4eaf
--- /dev/null
+++ b/src/main/java/istlab/KisoJikken/Test/my_custom_token_maker.flex
@@ -0,0 +1,226 @@
+/*
+ * 11/07/2008
+ *
+ * PlainTextTokenMaker.flex - Scanner for plain text files.
+ *
+ * This library is distributed under a modified BSD license. See the included
+ * LICENSE file for details.
+ */
+package istlab.KisoJikken.Test;
+
+import java.io.*;
+import javax.swing.text.Segment;
+
+import org.fife.ui.rsyntaxtextarea.*;
+
+
+/**
+ * Scanner for plain text files.
+ *
+ * This implementation was created using
+ * JFlex 1.4.1; however, the generated file
+ * was modified for performance. Memory allocation needs to be almost
+ * completely removed to be competitive with the handwritten lexers (subclasses
+ * of AbstractTokenMaker), so this class has been modified so that
+ * Strings are never allocated (via yytext()), and the scanner never has to
+ * worry about refilling its buffer (needlessly copying chars around).
+ * We can achieve this because RText always scans exactly 1 line of tokens at a
+ * time, and hands the scanner this line as an array of characters (a Segment
+ * really). Since tokens contain pointers to char arrays instead of Strings
+ * holding their contents, there is no need for allocating new memory for
+ * Strings.
+ * + * The actual algorithm generated for scanning has, of course, not been + * modified.
+ * + * If you wish to regenerate this file yourself, keep in mind the following: + *
PlainTextTokenMaker.java file will contain
+ * two definitions of both zzRefill and yyreset.
+ * You should hand-delete the second of each definition (the ones
+ * generated by the lexer), as these generated methods modify the input
+ * buffer, which we'll never have to do.yylex() on the generated scanner
+ * directly; rather, you should use getTokenList as you would
+ * with any other TokenMaker instance.TokenTypes.NULL, as there are no multiline
+ * tokens in properties files.
+ *
+ * @param text The line of tokens to examine.
+ * @param initialTokenType The token type to start with (i.e., the value
+ * of getLastTokenTypeOnLine for the line before
+ * text).
+ * @return TokenTypes.NULL.
+ */
+ public int getLastTokenTypeOnLine(Segment text, int initialTokenType) {
+ return TokenTypes.NULL;
+ }
+
+
+ /**
+ * Returns the text to place at the beginning and end of a
+ * line to "comment" it in a this programming language.
+ *
+ * @return null, as there are no comments in plain text.
+ */
+ @Override
+ public String[] getLineCommentStartAndEnd(int languageIndex) {
+ return null;
+ }
+
+
+ /**
+ * Always returns false, as you never want "mark occurrences"
+ * working in plain text files.
+ *
+ * @param type The token type.
+ * @return Whether tokens of this type should have "mark occurrences"
+ * enabled.
+ */
+ @Override
+ public boolean getMarkOccurrencesOfTokenType(int type) {
+ return false;
+ }
+
+
+ /**
+ * Returns the first token in the linked list of tokens generated
+ * from text. This method must be implemented by
+ * subclasses so they can correctly implement syntax highlighting.
+ *
+ * @param text The text from which to get tokens.
+ * @param initialTokenType The token type we should start with.
+ * @param startOffset The offset into the document at which
+ * text starts.
+ * @return The first Token in a linked list representing
+ * the syntax highlighted text.
+ */
+ public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
+
+ resetTokenList();
+ this.offsetShift = -text.offset + startOffset;
+
+ // Start off in the proper state.
+ s = text;
+ try {
+ yyreset(zzReader);
+ yybegin(YYINITIAL);
+ return yylex();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return new TokenImpl();
+ }
+
+ }
+
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return true if EOF was reached, otherwise
+ * false.
+ */
+ private boolean zzRefill() {
+ return zzCurrentPos>=s.offset+s.count;
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * cannot be reused (internal buffer is discarded and lost).
+ * Lexical state is set to YY_INITIAL.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(java.io.Reader reader) {
+ // 's' has been updated.
+ zzBuffer = s.array;
+ /*
+ * We replaced the line below with the two below it because zzRefill
+ * no longer "refills" the buffer (since the way we do it, it's always
+ * "full" the first time through, since it points to the segment's
+ * array). So, we assign zzEndRead here.
+ */
+ //zzStartRead = zzEndRead = s.offset;
+ zzStartRead = s.offset;
+ zzEndRead = zzStartRead + s.count - 1;
+ zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
+ zzLexicalState = YYINITIAL;
+ zzReader = reader;
+ zzAtBOL = true;
+ zzAtEOF = false;
+ }
+
+
+%}
+
+LetterOrDigit = ([a-zA-Z0-9])
+Identifier = ({LetterOrDigit}+)
+Separator = ([^a-zA-Z0-9 \t\n])
+WhiteSpace = ([ \t]+)
+LineTerminator = ([\n])
+
+URLGenDelim = ([:\/\?#\[\]@])
+URLSubDelim = ([\!\$&'\(\)\*\+,;=])
+URLUnreserved = ({LetterOrDigit}|[_\-\.\~])
+URLCharacter = ({URLGenDelim}|{URLSubDelim}|{URLUnreserved}|[%])
+URLCharacters = ({URLCharacter}*)
+URLEndCharacter = ([\/\$]|{LetterOrDigit})
+URL = (((https?|f(tp|ile))"://"|"www.")({URLCharacters}{URLEndCharacter})?)
+
+%%
+
+
+ *
+ * This implementation was created using
+ * JFlex 1.4.1; however, the generated file
+ * was modified for performance. Memory allocation needs to be almost
+ * completely removed to be competitive with the handwritten lexers (subclasses
+ * of AbstractTokenMaker), so this class has been modified so that
+ * Strings are never allocated (via yytext()), and the scanner never has to
+ * worry about refilling its buffer (needlessly copying chars around).
+ * We can achieve this because RText always scans exactly 1 line of tokens at a
+ * time, and hands the scanner this line as an array of characters (a Segment
+ * really). Since tokens contain pointers to char arrays instead of Strings
+ * holding their contents, there is no need for allocating new memory for
+ * Strings.
+ * + * The actual algorithm generated for scanning has, of course, not been + * modified.
+ * + * If you wish to regenerate this file yourself, keep in mind the following: + *
JavaTokenMaker.java file will contain two
+ * definitions of both zzRefill and yyreset.
+ * You should hand-delete the second of each definition (the ones
+ * generated by the lexer), as these generated methods modify the input
+ * buffer, which we'll never have to do.yylex() on the generated scanner
+ * directly; rather, you should use getTokenList as you would
+ * with any other TokenMaker instance.text. This method must be implemented by
+ * subclasses so they can correctly implement syntax highlighting.
+ *
+ * @param text The text from which to get tokens.
+ * @param initialTokenType The token type we should start with.
+ * @param startOffset The offset into the document at which
+ * text starts.
+ * @return The first Token in a linked list representing
+ * the syntax highlighted text.
+ */
+ public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
+
+ resetTokenList();
+ this.offsetShift = -text.offset + startOffset;
+
+ // Start off in the proper state.
+ int state;
+ switch (initialTokenType) {
+ case TokenTypes.COMMENT_MULTILINE:
+ state = MLC;
+ start = text.offset;
+ break;
+ case TokenTypes.COMMENT_DOCUMENTATION:
+ state = DOCCOMMENT;
+ start = text.offset;
+ break;
+ case TokenTypes.LITERAL_STRING_DOUBLE_QUOTE:
+ state = TEXT_BLOCK;
+ start = text.offset;
+ break;
+ default:
+ state = YYINITIAL;
+ }
+
+ s = text;
+ try {
+ yyreset(zzReader);
+ yybegin(state);
+ return yylex();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return new TokenImpl();
+ }
+
+ }
+
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return true if EOF was reached, otherwise
+ * false.
+ */
+ private boolean zzRefill() {
+ return zzCurrentPos>=s.offset+s.count;
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * cannot be reused (internal buffer is discarded and lost).
+ * Lexical state is set to YY_INITIAL.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(Reader reader) {
+ // 's' has been updated.
+ zzBuffer = s.array;
+ /*
+ * We replaced the line below with the two below it because zzRefill
+ * no longer "refills" the buffer (since the way we do it, it's always
+ * "full" the first time through, since it points to the segment's
+ * array). So, we assign zzEndRead here.
+ */
+ //zzStartRead = zzEndRead = s.offset;
+ zzStartRead = s.offset;
+ zzEndRead = zzStartRead + s.count - 1;
+ zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
+ zzLexicalState = YYINITIAL;
+ zzReader = reader;
+ zzAtBOL = true;
+ zzAtEOF = false;
+ }
+
+
+%}
+
+Letter = ([A-Za-z])
+LetterOrUnderscore = ({Letter}|"_")
+NonzeroDigit = ([1-9])
+BinaryDigit = ([0-1])
+Digit = ("0"|{NonzeroDigit})
+HexDigit = ({Digit}|[A-Fa-f])
+OctalDigit = ([0-7])
+AnyCharacterButApostropheOrBackSlash = ([^\\'])
+AnyCharacterButDoubleQuoteOrBackSlash = ([^\\\"\n])
+EscapedSourceCharacter = ("u"{HexDigit}{HexDigit}{HexDigit}{HexDigit})
+Escape = ("\\"(([bstnfr\"'\\])|([0123]{OctalDigit}?{OctalDigit}?)|({OctalDigit}{OctalDigit}?)|{EscapedSourceCharacter}))
+NonSeparator = ([^\t\f\r\n\ \(\)\{\}\[\]\;\,\.\=\>\<\!\~\?\:\+\-\*\/\&\|\^\%\"\']|"#"|"\\")
+IdentifierStart = ([:jletter:])
+IdentifierPart = ([:jletterdigit:]|("\\"{EscapedSourceCharacter}))
+
+LineTerminator = (\n)
+WhiteSpace = ([ \t\f])
+
+CharLiteral = ([\']({AnyCharacterButApostropheOrBackSlash}|{Escape})[\'])
+UnclosedCharLiteral = ([\'][^\'\n]*)
+ErrorCharLiteral = ({UnclosedCharLiteral}[\'])
+StringLiteral = ([\"]({AnyCharacterButDoubleQuoteOrBackSlash}|{Escape})*[\"])
+UnclosedStringLiteral = ([\"]([\\].|[^\\\"])*[^\"]?)
+ErrorStringLiteral = ({UnclosedStringLiteral}[\"])
+
+MLCBegin = "/*"
+MLCEnd = "*/"
+DocCommentBegin = "/**"
+LineCommentBegin = "//"
+
+DigitOrUnderscore = ({Digit}|[_])
+DigitsAndUnderscoresEnd = ({DigitOrUnderscore}*{Digit})
+IntegerHelper = (({NonzeroDigit}{DigitsAndUnderscoresEnd}?)|"0")
+IntegerLiteral = ({IntegerHelper}[lL]?)
+
+BinaryDigitOrUnderscore = ({BinaryDigit}|[_])
+BinaryDigitsAndUnderscores = ({BinaryDigit}({BinaryDigitOrUnderscore}*{BinaryDigit})?)
+BinaryLiteral = ("0"[bB]{BinaryDigitsAndUnderscores})
+
+HexDigitOrUnderscore = ({HexDigit}|[_])
+HexDigitsAndUnderscores = ({HexDigit}({HexDigitOrUnderscore}*{HexDigit})?)
+OctalDigitOrUnderscore = ({OctalDigit}|[_])
+OctalDigitsAndUnderscoresEnd= ({OctalDigitOrUnderscore}*{OctalDigit})
+HexHelper = ("0"(([xX]{HexDigitsAndUnderscores})|({OctalDigitsAndUnderscoresEnd})))
+HexLiteral = ({HexHelper}[lL]?)
+
+FloatHelper1 = ([fFdD]?)
+FloatHelper2 = ([eE][+-]?{Digit}+{FloatHelper1})
+FloatLiteral1 = ({Digit}+"."({FloatHelper1}|{FloatHelper2}|{Digit}+({FloatHelper1}|{FloatHelper2})))
+FloatLiteral2 = ("."{Digit}+({FloatHelper1}|{FloatHelper2}))
+FloatLiteral3 = ({Digit}+{FloatHelper2})
+FloatLiteral = ({FloatLiteral1}|{FloatLiteral2}|{FloatLiteral3}|({Digit}+[fFdD]))
+
+ErrorNumberFormat = (({IntegerLiteral}|{HexLiteral}|{FloatLiteral}){NonSeparator}+)
+BooleanLiteral = ("true"|"false")
+
+Separator = ([\(\)\{\}\[\]])
+Separator2 = ([\;,.])
+
+NonAssignmentOperator = ("+"|"-"|"<="|"^"|"++"|"<"|"*"|">="|"%"|"--"|">"|"/"|"!="|"?"|">>"|"!"|"&"|"=="|":"|">>"|"~"|"|"|"&&"|">>>")
+AssignmentOperator = ("="|"-="|"*="|"/="|"|="|"&="|"^="|"+="|"%="|"<<="|">>="|">>>=")
+Operator = ({NonAssignmentOperator}|{AssignmentOperator})
+
+CurrentBlockTag = ("author"|"deprecated"|"exception"|"param"|"return"|"see"|"serial"|"serialData"|"serialField"|"since"|"throws"|"version")
+ProposedBlockTag = ("category"|"example"|"tutorial"|"index"|"exclude"|"todo"|"internal"|"obsolete"|"threadsafety")
+BlockTag = ({CurrentBlockTag}|{ProposedBlockTag})
+InlineTag = ("code"|"docRoot"|"inheritDoc"|"link"|"linkplain"|"literal"|"value")
+
+Identifier = ({IdentifierStart}{IdentifierPart}*)
+ErrorIdentifier = ({NonSeparator}+)
+
+Annotation = ("@"{Identifier}?)
+
+URLGenDelim = ([:\/\?#\[\]@])
+URLSubDelim = ([\!\$&'\(\)\*\+,;=])
+URLUnreserved = ({LetterOrUnderscore}|{Digit}|[\-\.\~])
+URLCharacter = ({URLGenDelim}|{URLSubDelim}|{URLUnreserved}|[%])
+URLCharacters = ({URLCharacter}*)
+URLEndCharacter = ([\/\$]|{Letter}|{Digit})
+URL = (((https?|f(tp|ile))"://"|"www.")({URLCharacters}{URLEndCharacter})?)
+
+
+%state MLC
+%state DOCCOMMENT
+%state EOL_COMMENT
+%state TEXT_BLOCK
+
+%%
+
+