Class RubyLex
In: lib/irb/ruby-lex.rb
lib/rdoc/parsers/parse_rb.rb
Parent: Object

Lexical analyzer for Ruby source

Methods

Included Modules

RubyToken RubyToken IRB

Classes and Modules

Class RubyLex::BufferedReader

Constants

ENINDENT_CLAUSE = [ "case", "class", "def", "do", "for", "if", "module", "unless", "until", "while", "begin"
DEINDENT_CLAUSE = ["end"
PERCENT_LTYPE = { "q" => "\'", "Q" => "\"", "x" => "\`", "r" => "/", "w" => "]", "W" => "]", "s" => ":"
PERCENT_PAREN = { "{" => "}", "[" => "]", "<" => ">", "(" => ")"
Ltype2Token = { "\'" => TkSTRING, "\"" => TkSTRING, "\`" => TkXSTRING, "/" => TkREGEXP, "]" => TkDSTRING, ":" => TkSYMBOL
DLtype2Token = { "\"" => TkDSTRING, "\`" => TkDXSTRING, "/" => TkDREGEXP, }
ENINDENT_CLAUSE = [ "case", "class", "def", "do", "for", "if", "module", "unless", "until", "while", "begin"
DEINDENT_CLAUSE = ["end"
PERCENT_LTYPE = { "q" => "\'", "Q" => "\"", "x" => "\`", "r" => "/", "w" => "]"
PERCENT_PAREN = { "{" => "}", "[" => "]", "<" => ">", "(" => ")"
Ltype2Token = { "\'" => TkSTRING, "\"" => TkSTRING, "\`" => TkXSTRING, "/" => TkREGEXP, "]" => TkDSTRING
DLtype2Token = { "\"" => TkDSTRING, "\`" => TkDXSTRING, "/" => TkDREGEXP, }

Attributes

char_no  [R] 
continue  [R] 
debug_level  [RW] 
exception_on_syntax_error  [RW] 
exception_on_syntax_error  [RW] 
indent  [R] 
indent  [R] 
lex_state  [R] 
line_no  [R] 
read_auto_clean_up  [RW] 
readed_auto_clean_up  [RW] 
seek  [R] 
skip_space  [RW] 
skip_space  [RW] 

Public Class methods

[Source]

    # File lib/irb/ruby-lex.rb, line 34
34:     def debug?
35:       @debug_level > 0
36:     end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 443
443:   def RubyLex.debug?
444:     false
445:   end

[Source]

    # File lib/irb/ruby-lex.rb, line 40
40:   def initialize
41:     lex_init
42:     set_input(STDIN)
43: 
44:     @seek = 0
45:     @exp_line_no = @line_no = 1
46:     @base_char_no = 0
47:     @char_no = 0
48:     @rests = []
49:     @readed = []
50:     @here_readed = []
51: 
52:     @indent = 0
53:     @indent_stack = []
54:     @lex_state = EXPR_BEG
55:     @space_seen = false
56:     @here_header = false
57:     
58:     @continue = false
59:     @line = ""
60: 
61:     @skip_space = false
62:     @readed_auto_clean_up = false
63:     @exception_on_syntax_error = true
64: 
65:     @prompt = nil
66:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 447
447:   def initialize(content)
448:     lex_init
449: 
450:     @reader = BufferedReader.new(content)
451: 
452:     @exp_line_no = @line_no = 1
453:     @base_char_no = 0
454:     @indent = 0
455: 
456:     @ltype = nil
457:     @quoted = nil
458:     @lex_state = EXPR_BEG
459:     @space_seen = false
460:     
461:     @continue = false
462:     @line = ""
463: 
464:     @skip_space = false
465:     @read_auto_clean_up = false
466:     @exception_on_syntax_error = true
467:   end

Public Instance methods

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 480
480:   def char_no
481:     @reader.column
482:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 227
227:   def each_top_level_statement
228:     initialize_input
229:     catch(:TERM_INPUT) do
230:       loop do
231:         begin
232:           @continue = false
233:           prompt
234:           unless l = lex
235:             throw :TERM_INPUT if @line == ''
236:           else
237:             #p l
238:             @line.concat l
239:             if @ltype or @continue or @indent > 0
240:               next
241:             end
242:           end
243:           if @line != "\n"
244:             yield @line, @exp_line_no
245:           end
246:           break unless l
247:           @line = ''
248:           @exp_line_no = @line_no
249: 
250:           @indent = 0
251:           @indent_stack = []
252:           prompt
253:         rescue TerminateLineInput
254:           initialize_input
255:           prompt
256:           get_readed
257:         end
258:       end
259:     end
260:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 132
132:   def eof?
133:     @io.eof?
134:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 484
484:   def get_read
485:     @reader.get_read
486:   end

[Source]

    # File lib/irb/ruby-lex.rb, line 89
89:   def get_readed
90:     if idx = @readed.reverse.index("\n")
91:       @base_char_no = idx
92:     else
93:       @base_char_no += @readed.size
94:     end
95:     
96:     readed = @readed.join("")
97:     @readed = []
98:     readed
99:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 488
488:   def getc
489:     @reader.getc
490:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 101
101:   def getc
102:     while @rests.empty?
103: #      return nil unless buf_input
104:       @rests.push nil unless buf_input
105:     end
106:     c = @rests.shift
107:     if @here_header
108:       @here_readed.push c
109:     else
110:       @readed.push c
111:     end
112:     @seek += 1
113:     if c == "\n"
114:       @line_no += 1 
115:       @char_no = 0
116:     else
117:       @char_no += 1
118:     end
119:     c
120:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 492
492:   def getc_of_rests
493:     @reader.getc_already_read
494:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 136
136:   def getc_of_rests
137:     if @rests.empty?
138:       nil
139:     else
140:       getc
141:     end
142:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 122
122:   def gets
123:     l = ""
124:     while c = getc
125:       l.concat(c)
126:       break if c == "\n"
127:     end
128:     return nil if l == "" and c.nil?
129:     l
130:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 496
496:   def gets
497:     c = getc or return
498:     l = ""
499:     begin
500:       l.concat c unless c == "\r"
501:       break if c == "\n"
502:     end while c = getc
503:     l
504:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1269
1269:   def identify_comment
1270:     @ltype = "#"
1271:     comment = "#"
1272:     while ch = getc
1273:       if ch == "\\"
1274:         ch = getc
1275:         if ch == "\n"
1276:           ch = " "
1277:         else
1278:           comment << "\\" 
1279:         end
1280:       else
1281:         if ch == "\n"
1282:           @ltype = nil
1283:           ungetc
1284:           break
1285:         end
1286:       end
1287:       comment << ch
1288:     end
1289:     return Token(TkCOMMENT).set_text(comment)
1290:   end

[Source]

      # File lib/irb/ruby-lex.rb, line 1085
1085:   def identify_comment
1086:     @ltype = "#"
1087: 
1088:     while ch = getc
1089: #      if ch == "\\" #"
1090: #       read_escape
1091: #      end
1092:       if ch == "\n"
1093:         @ltype = nil
1094:         ungetc
1095:         break
1096:       end
1097:     end
1098:     return Token(TkCOMMENT)
1099:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 964
964:   def identify_gvar
965:     @lex_state = EXPR_END
966:     str = "$"
967: 
968:     tk = case ch = getc
969:          when /[~_*$?!@\/\\;,=:<>".]/   #"
970:            str << ch
971:            Token(TkGVAR, str)
972:            
973:          when "-"
974:            str << "-" << getc
975:            Token(TkGVAR, str)
976:            
977:          when "&", "`", "'", "+"
978:            str << ch
979:            Token(TkBACK_REF, str)
980:            
981:          when /[1-9]/
982:            str << ch
983:            while (ch = getc) =~ /[0-9]/
984:              str << ch
985:            end
986:            ungetc
987:            Token(TkNTH_REF)
988:          when /\w/
989:            ungetc
990:            ungetc
991:            return identify_identifier
992:          else 
993:            ungetc
994:            Token("$")     
995:          end
996:     tk.set_text(str)
997:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 746
746:   def identify_gvar
747:     @lex_state = EXPR_END
748:     
749:     case ch = getc
750:     when /[~_*$?!@\/\\;,=:<>".]/   #"
751:       Token(TkGVAR, "$" + ch)
752:     when "-"
753:       Token(TkGVAR, "$-" + getc)
754:     when "&", "`", "'", "+"
755:       Token(TkBACK_REF, "$"+ch)
756:     when /[1-9]/
757:       while getc =~ /[0-9]/; end
758:       ungetc
759:       Token(TkNTH_REF)
760:     when /\w/
761:       ungetc
762:       ungetc
763:       identify_identifier
764:     else 
765:       ungetc
766:       Token("$")
767:     end
768:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1074
1074:   def identify_here_document
1075:     ch = getc
1076:     if ch == "-"
1077:       ch = getc
1078:       indent = true
1079:     end
1080:     if /['"`]/ =~ ch            # '
1081:       lt = ch
1082:       quoted = ""
1083:       while (c = getc) && c != lt
1084:         quoted.concat c
1085:       end
1086:     else
1087:       lt = '"'
1088:       quoted = ch.dup
1089:       while (c = getc) && c =~ /\w/
1090:         quoted.concat c
1091:       end
1092:       ungetc
1093:     end
1094: 
1095:     ltback, @ltype = @ltype, lt
1096:     reserve = ""
1097: 
1098:     while ch = getc
1099:       reserve << ch
1100:       if ch == "\\"    #"
1101:         ch = getc
1102:         reserve << ch
1103:       elsif ch == "\n"
1104:         break
1105:       end
1106:     end
1107: 
1108:     str = ""
1109:     while (l = gets)
1110:       l.chomp!
1111:       l.strip! if indent
1112:       break if l == quoted
1113:       str << l.chomp << "\n"
1114:     end
1115: 
1116:     @reader.divert_read_from(reserve)
1117: 
1118:     @ltype = ltback
1119:     @lex_state = EXPR_END
1120:     Token(Ltype2Token[lt], str).set_text(str.dump)
1121:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 879
879:   def identify_here_document
880:     ch = getc
881: #    if lt = PERCENT_LTYPE[ch]
882:     if ch == "-"
883:       ch = getc
884:       indent = true
885:     end
886:     if /['"`]/ =~ ch
887:       lt = ch
888:       quoted = ""
889:       while (c = getc) && c != lt
890:         quoted.concat c
891:       end
892:     else
893:       lt = '"'
894:       quoted = ch.dup
895:       while (c = getc) && c =~ /\w/
896:         quoted.concat c
897:       end
898:       ungetc
899:     end
900: 
901:     ltback, @ltype = @ltype, lt
902:     reserve = []
903:     while ch = getc
904:       reserve.push ch
905:       if ch == "\\"
906:         reserve.push ch = getc
907:       elsif ch == "\n"
908:         break
909:       end
910:     end
911: 
912:     @here_header = false
913:     while l = gets
914:       l = l.sub(/(:?\r)?\n\z/, '')
915:       if (indent ? l.strip : l) == quoted
916:         break
917:       end
918:     end
919: 
920:     @here_header = true
921:     @here_readed.concat reserve
922:     while ch = reserve.pop
923:       ungetc ch
924:     end
925: 
926:     @ltype = ltback
927:     @lex_state = EXPR_END
928:     Token(Ltype2Token[lt])
929:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 999
 999:   def identify_identifier
1000:     token = ""
1001:     token.concat getc if peek(0) =~ /[$@]/
1002:     token.concat getc if peek(0) == "@"
1003: 
1004:     while (ch = getc) =~ /\w|_/
1005:       print ":", ch, ":" if RubyLex.debug?
1006:       token.concat ch
1007:     end
1008:     ungetc
1009:     
1010:     if ch == "!" or ch == "?"
1011:       token.concat getc
1012:     end
1013:     # fix token
1014: 
1015:     # $stderr.puts "identifier - #{token}, state = #@lex_state"
1016: 
1017:     case token
1018:     when /^\$/
1019:       return Token(TkGVAR, token).set_text(token)
1020:     when /^\@/
1021:       @lex_state = EXPR_END
1022:       return Token(TkIVAR, token).set_text(token)
1023:     end
1024:     
1025:     if @lex_state != EXPR_DOT
1026:       print token, "\n" if RubyLex.debug?
1027: 
1028:       token_c, *trans = TkReading2Token[token]
1029:       if token_c
1030:         # reserved word?
1031: 
1032:         if (@lex_state != EXPR_BEG &&
1033:             @lex_state != EXPR_FNAME &&
1034:             trans[1])
1035:           # modifiers
1036:           token_c = TkSymbol2Token[trans[1]]
1037:           @lex_state = trans[0]
1038:         else
1039:           if @lex_state != EXPR_FNAME
1040:             if ENINDENT_CLAUSE.include?(token)
1041:               @indent += 1
1042:             elsif DEINDENT_CLAUSE.include?(token)
1043:               @indent -= 1
1044:             end
1045:             @lex_state = trans[0]
1046:           else
1047:             @lex_state = EXPR_END
1048:           end
1049:         end
1050:         return Token(token_c, token).set_text(token)
1051:       end
1052:     end
1053: 
1054:     if @lex_state == EXPR_FNAME
1055:       @lex_state = EXPR_END
1056:       if peek(0) == '='
1057:         token.concat getc
1058:       end
1059:     elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
1060:       @lex_state = EXPR_ARG
1061:     else
1062:       @lex_state = EXPR_END
1063:     end
1064: 
1065:     if token[0, 1] =~ /[A-Z]/
1066:       return Token(TkCONSTANT, token).set_text(token)
1067:     elsif token[token.size - 1, 1] =~ /[!?]/
1068:       return Token(TkFID, token).set_text(token)
1069:     else
1070:       return Token(TkIDENTIFIER, token).set_text(token)
1071:     end
1072:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 770
770:   def identify_identifier
771:     token = ""
772:     if peek(0) =~ /[$@]/
773:       token.concat(c = getc)
774:       if c == "@" and peek(0) == "@"
775:         token.concat getc
776:       end
777:     end
778: 
779:     while (ch = getc) =~ /\w|_/
780:       print ":", ch, ":" if RubyLex.debug?
781:       token.concat ch
782:     end
783:     ungetc
784:     
785:     if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
786:       token.concat getc
787:     end
788: 
789:     # almost fix token
790: 
791:     case token
792:     when /^\$/
793:       return Token(TkGVAR, token)
794:     when /^\@\@/
795:       @lex_state = EXPR_END
796:       # p Token(TkCVAR, token)
797:       return Token(TkCVAR, token)
798:     when /^\@/
799:       @lex_state = EXPR_END
800:       return Token(TkIVAR, token)
801:     end
802:     
803:     if @lex_state != EXPR_DOT
804:       print token, "\n" if RubyLex.debug?
805: 
806:       token_c, *trans = TkReading2Token[token]
807:       if token_c
808:         # reserved word?
809: 
810:         if (@lex_state != EXPR_BEG &&
811:             @lex_state != EXPR_FNAME &&
812:             trans[1])
813:           # modifiers
814:           token_c = TkSymbol2Token[trans[1]]
815:           @lex_state = trans[0]
816:         else
817:           if @lex_state != EXPR_FNAME
818:             if ENINDENT_CLAUSE.include?(token)
819:               # check for ``class = val'' etc.
820:               valid = true
821:               case token
822:               when "class"
823:                 valid = false unless peek_match?(/^\s*(<<|\w|::)/)
824:               when "def"
825:                 valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
826:               when "do"
827:                 valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
828:               when *ENINDENT_CLAUSE
829:                 valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
830:               else
831:                 # no nothing
832:               end
833:               if valid
834:                 if token == "do"
835:                   if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
836:                     @indent += 1
837:                     @indent_stack.push token_c
838:                   end
839:                 else
840:                   @indent += 1
841:                   @indent_stack.push token_c
842:                 end
843: #               p @indent_stack
844:               end
845: 
846:             elsif DEINDENT_CLAUSE.include?(token)
847:               @indent -= 1
848:               @indent_stack.pop
849:             end
850:             @lex_state = trans[0]
851:           else
852:             @lex_state = EXPR_END
853:           end
854:         end
855:         return Token(token_c, token)
856:       end
857:     end
858: 
859:     if @lex_state == EXPR_FNAME
860:       @lex_state = EXPR_END
861:       if peek(0) == '='
862:         token.concat getc
863:       end
864:     elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
865:       @lex_state = EXPR_ARG
866:     else
867:       @lex_state = EXPR_END
868:     end
869: 
870:     if token[0, 1] =~ /[A-Z]/
871:       return Token(TkCONSTANT, token)
872:     elsif token[token.size - 1, 1] =~ /[!?]/
873:       return Token(TkFID, token)
874:     else
875:       return Token(TkIDENTIFIER, token)
876:     end
877:   end

[Source]

      # File lib/irb/ruby-lex.rb, line 949
 949:   def identify_number
 950:     @lex_state = EXPR_END
 951: 
 952:     if peek(0) == "0" && peek(1) !~ /[.eE]/
 953:       getc
 954:       case peek(0)
 955:       when /[xX]/
 956:         ch = getc
 957:         match = /[0-9a-fA-F_]/
 958:       when /[bB]/
 959:         ch = getc
 960:         match = /[01_]/
 961:       when /[oO]/
 962:         ch = getc
 963:         match = /[0-7_]/
 964:       when /[dD]/
 965:         ch = getc
 966:         match = /[0-9_]/
 967:       when /[0-7]/
 968:         match = /[0-7_]/
 969:       when /[89]/
 970:         RubyLex.fail SyntaxError, "Illegal octal digit"
 971:       else 
 972:         return Token(TkINTEGER)
 973:       end
 974:       
 975:       len0 = true
 976:       non_digit = false
 977:       while ch = getc
 978:         if match =~ ch
 979:           if ch == "_"
 980:             if non_digit
 981:               RubyLex.fail SyntaxError, "trailing `#{ch}' in number"
 982:             else
 983:               non_digit = ch
 984:             end
 985:           else
 986:             non_digit = false
 987:             len0 = false
 988:           end
 989:         else
 990:           ungetc
 991:           if len0
 992:             RubyLex.fail SyntaxError, "numeric literal without digits"
 993:           end
 994:           if non_digit
 995:             RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
 996:           end
 997:           break
 998:         end
 999:       end
1000:       return Token(TkINTEGER)
1001:     end
1002:     
1003:     type = TkINTEGER
1004:     allow_point = true
1005:     allow_e = true
1006:     non_digit = false
1007:     while ch = getc
1008:       case ch
1009:       when /[0-9]/
1010:         non_digit = false
1011:       when "_"
1012:         non_digit = ch
1013:       when allow_point && "."
1014:         if non_digit
1015:           RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1016:         end
1017:         type = TkFLOAT
1018:         if peek(0) !~ /[0-9]/
1019:           type = TkINTEGER
1020:           ungetc
1021:           break
1022:         end
1023:         allow_point = false
1024:       when allow_e && "e", allow_e && "E"
1025:         if non_digit
1026:           RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1027:         end
1028:         type = TkFLOAT
1029:         if peek(0) =~ /[+-]/
1030:           getc
1031:         end
1032:         allow_e = false
1033:         allow_point = false
1034:         non_digit = ch
1035:       else
1036:         if non_digit
1037:           RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1038:         end
1039:         ungetc
1040:         break
1041:       end
1042:     end
1043:     Token(type)
1044:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1142
1142:   def identify_number(start)
1143:     str = start.dup
1144: 
1145:     if start == "+" or start == "-" or start == ""
1146:       start = getc
1147:       str << start
1148:     end
1149: 
1150:     @lex_state = EXPR_END
1151: 
1152:     if start == "0"
1153:       if peek(0) == "x"
1154:         ch = getc
1155:         str << ch
1156:         match = /[0-9a-f_]/
1157:       else
1158:         match = /[0-7_]/
1159:       end
1160:       while ch = getc
1161:         if ch !~ match
1162:           ungetc
1163:           break
1164:         else
1165:           str << ch
1166:         end
1167:       end
1168:       return Token(TkINTEGER).set_text(str)
1169:     end
1170: 
1171:     type = TkINTEGER
1172:     allow_point = TRUE
1173:     allow_e = TRUE
1174:     while ch = getc
1175:       case ch
1176:       when /[0-9_]/
1177:         str << ch
1178: 
1179:       when allow_point && "."
1180:         type = TkFLOAT
1181:         if peek(0) !~ /[0-9]/
1182:           ungetc
1183:           break
1184:         end
1185:         str << ch
1186:         allow_point = false
1187: 
1188:       when allow_e && "e", allow_e && "E"
1189:         str << ch
1190:         type = TkFLOAT
1191:         if peek(0) =~ /[+-]/
1192:           str << getc
1193:         end
1194:         allow_e = false
1195:         allow_point = false
1196:       else
1197:         ungetc
1198:         break
1199:       end
1200:     end
1201:     Token(type).set_text(str)
1202:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1123
1123:   def identify_quotation(initial_char)
1124:     ch = getc
1125:     if lt = PERCENT_LTYPE[ch]
1126:       initial_char += ch
1127:       ch = getc
1128:     elsif ch =~ /\W/
1129:       lt = "\""
1130:     else
1131:       RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
1132:     end
1133: #     if ch !~ /\W/
1134: #       ungetc
1135: #       next
1136: #     end
1137:     #@ltype = lt
1138:     @quoted = ch unless @quoted = PERCENT_PAREN[ch]
1139:     identify_string(lt, @quoted, ch, initial_char)
1140:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 931
931:   def identify_quotation
932:     ch = getc
933:     if lt = PERCENT_LTYPE[ch]
934:       ch = getc
935:     elsif ch =~ /\W/
936:       lt = "\""
937:     else
938:       RubyLex.fail SyntaxError, "unknown type of %string"
939:     end
940: #     if ch !~ /\W/
941: #       ungetc
942: #       next
943: #     end
944:     #@ltype = lt
945:     @quoted = ch unless @quoted = PERCENT_PAREN[ch]
946:     identify_string(lt, @quoted)
947:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1204
1204:   def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
1205:     @ltype = ltype
1206:     @quoted = quoted
1207:     subtype = nil
1208: 
1209:     str = ""
1210:     str << initial_char if initial_char
1211:     str << (opener||quoted)
1212: 
1213:     nest = 0
1214:     begin
1215:       while ch = getc 
1216:         str << ch
1217:         if @quoted == ch 
1218:           if nest == 0
1219:             break
1220:           else
1221:             nest -= 1
1222:           end
1223:         elsif opener == ch
1224:           nest += 1
1225:         elsif @ltype != "'" && @ltype != "]" and ch == "#"
1226:           ch = getc
1227:           if ch == "{"
1228:             subtype = true
1229:             str << ch << skip_inner_expression
1230:           else
1231:             ungetc(ch)
1232:           end
1233:         elsif ch == '\\' #'
1234:           str << read_escape
1235:         end
1236:       end
1237:       if @ltype == "/"
1238:         if peek(0) =~ /i|o|n|e|s/
1239:           str << getc
1240:         end
1241:       end
1242:       if subtype
1243:         Token(DLtype2Token[ltype], str)
1244:       else
1245:         Token(Ltype2Token[ltype], str)
1246:       end.set_text(str)
1247:     ensure
1248:       @ltype = nil
1249:       @quoted = nil
1250:       @lex_state = EXPR_END
1251:     end
1252:   end

[Source]

      # File lib/irb/ruby-lex.rb, line 1046
1046:   def identify_string(ltype, quoted = ltype)
1047:     @ltype = ltype
1048:     @quoted = quoted
1049:     subtype = nil
1050:     begin
1051:       nest = 0
1052:       while ch = getc
1053:         if @quoted == ch and nest == 0
1054:           break
1055:         elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
1056:           subtype = true
1057:         elsif ch == '\\' #'
1058:           read_escape
1059:         end
1060:         if PERCENT_PAREN.values.include?(@quoted) 
1061:           if PERCENT_PAREN[ch] == @quoted
1062:             nest += 1
1063:           elsif ch == @quoted
1064:             nest -= 1
1065:           end
1066:         end
1067:       end
1068:       if @ltype == "/"
1069:         if peek(0) =~ /i|m|x|o|e|s|u|n/
1070:           getc
1071:         end
1072:       end
1073:       if subtype
1074:         Token(DLtype2Token[ltype])
1075:       else
1076:         Token(Ltype2Token[ltype])
1077:       end
1078:     ensure
1079:       @ltype = nil
1080:       @quoted = nil
1081:       @lex_state = EXPR_END
1082:     end
1083:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 211
211:   def initialize_input
212:     @ltype = nil
213:     @quoted = nil
214:     @indent = 0
215:     @indent_stack = []
216:     @lex_state = EXPR_BEG
217:     @space_seen = false
218:     @here_header = false
219:     
220:     @continue = false
221:     prompt
222: 
223:     @line = ""
224:     @exp_line_no = @line_no
225:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 262
262:   def lex
263:     until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
264:              !@continue or
265:              tk.nil?)
266:       #p tk
267:       #p @lex_state
268:       #p self
269:     end
270:     line = get_readed
271:     #      print self.inspect
272:     if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
273:       nil
274:     else
275:       line
276:     end
277:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 519
519:   def lex
520:     until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
521:              !@continue or
522:              tk.nil?)
523:     end
524:     line = get_read
525: 
526:     if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
527:       nil
528:     else
529:       line
530:     end
531:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 339
339:   def lex_init()
340:     @OP = IRB::SLex.new
341:     @OP.def_rules("\0", "\004", "\032") do |op, io|
342:       Token(TkEND_OF_SCRIPT)
343:     end
344: 
345:     @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
346:       @space_seen = true
347:       while getc =~ /[ \t\f\r\13]/; end
348:       ungetc
349:       Token(TkSPACE)
350:     end
351: 
352:     @OP.def_rule("#") do |op, io|
353:       identify_comment
354:     end
355: 
356:     @OP.def_rule("=begin",
357:                  proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do 
358:       |op, io|
359:       @ltype = "="
360:       until getc == "\n"; end
361:       until peek_equal?("=end") && peek(4) =~ /\s/
362:         until getc == "\n"; end
363:       end
364:       gets
365:       @ltype = nil
366:       Token(TkRD_COMMENT)
367:     end
368: 
369:     @OP.def_rule("\n") do |op, io|
370:       print "\\n\n" if RubyLex.debug?
371:       case @lex_state
372:       when EXPR_BEG, EXPR_FNAME, EXPR_DOT
373:         @continue = true
374:       else
375:         @continue = false
376:         @lex_state = EXPR_BEG
377:         until (@indent_stack.empty? || 
378:                [TkLPAREN, TkLBRACK, TkLBRACE, 
379:                  TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
380:           @indent_stack.pop
381:         end
382:       end
383:       @here_header = false
384:       @here_readed = []
385:       Token(TkNL)
386:     end
387: 
388:     @OP.def_rules("*", "**",    
389:                   "=", "==", "===", 
390:                   "=~", "<=>",        
391:                   "<", "<=",
392:                   ">", ">=", ">>") do
393:       |op, io|
394:       case @lex_state
395:       when EXPR_FNAME, EXPR_DOT
396:         @lex_state = EXPR_ARG
397:       else
398:         @lex_state = EXPR_BEG
399:       end
400:       Token(op)
401:     end
402: 
403:     @OP.def_rules("!", "!=", "!~") do
404:       |op, io|
405:       @lex_state = EXPR_BEG
406:       Token(op)
407:     end
408: 
409:     @OP.def_rules("<<") do
410:       |op, io|
411:       tk = nil
412:       if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
413:           (@lex_state != EXPR_ARG || @space_seen)
414:         c = peek(0)
415:         if /\S/ =~ c && (/["'`]/ =~ c || /[\w_]/ =~ c || c == "-")
416:           tk = identify_here_document
417:         end
418:       end
419:       unless tk
420:         tk = Token(op)
421:         case @lex_state
422:         when EXPR_FNAME, EXPR_DOT
423:           @lex_state = EXPR_ARG
424:         else
425:           @lex_state = EXPR_BEG
426:         end
427:       end
428:       tk
429:     end
430: 
431:     @OP.def_rules("'", '"') do
432:       |op, io|
433:       identify_string(op)
434:     end
435: 
436:     @OP.def_rules("`") do
437:       |op, io|
438:       if @lex_state == EXPR_FNAME
439:         @lex_state = EXPR_END
440:         Token(op)
441:       else
442:         identify_string(op)
443:       end
444:     end
445: 
446:     @OP.def_rules('?') do
447:       |op, io|
448:       if @lex_state == EXPR_END
449:         @lex_state = EXPR_BEG
450:         Token(TkQUESTION)
451:       else
452:         ch = getc
453:         if @lex_state == EXPR_ARG && ch =~ /\s/
454:           ungetc
455:           @lex_state = EXPR_BEG;
456:           Token(TkQUESTION)
457:         else
458:           if (ch == '\\') 
459:             read_escape
460:           end
461:           @lex_state = EXPR_END
462:           Token(TkINTEGER)
463:         end
464:       end
465:     end
466: 
467:     @OP.def_rules("&", "&&", "|", "||") do
468:       |op, io|
469:       @lex_state = EXPR_BEG
470:       Token(op)
471:     end
472:     
473:     @OP.def_rules("+=", "-=", "*=", "**=", 
474:                   "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
475:       |op, io|
476:       @lex_state = EXPR_BEG
477:       op =~ /^(.*)=$/
478:       Token(TkOPASGN, $1)
479:     end
480: 
481:     @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
482:       |op, io|
483:       @lex_state = EXPR_ARG
484:       Token(op)
485:     end
486: 
487:     @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
488:       |op, io|
489:       @lex_state = EXPR_ARG
490:       Token(op)
491:     end
492: 
493:     @OP.def_rules("+", "-") do
494:       |op, io|
495:       catch(:RET) do
496:         if @lex_state == EXPR_ARG
497:           if @space_seen and peek(0) =~ /[0-9]/
498:             throw :RET, identify_number
499:           else
500:             @lex_state = EXPR_BEG
501:           end
502:         elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
503:           throw :RET, identify_number
504:         else
505:           @lex_state = EXPR_BEG
506:         end
507:         Token(op)
508:       end
509:     end
510: 
511:     @OP.def_rule(".") do
512:       |op, io|
513:       @lex_state = EXPR_BEG
514:       if peek(0) =~ /[0-9]/
515:         ungetc
516:         identify_number
517:       else
518:         # for "obj.if" etc.
519:         @lex_state = EXPR_DOT
520:         Token(TkDOT)
521:       end
522:     end
523: 
524:     @OP.def_rules("..", "...") do
525:       |op, io|
526:       @lex_state = EXPR_BEG
527:       Token(op)
528:     end
529: 
530:     lex_int2
531:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 589
589:   def lex_init()
590:     @OP = SLex.new
591:     @OP.def_rules("\0", "\004", "\032") do |chars, io|
592:       Token(TkEND_OF_SCRIPT).set_text(chars)
593:     end
594: 
595:     @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
596:       @space_seen = TRUE
597:       while (ch = getc) =~ /[ \t\f\r\13]/
598:         chars << ch
599:       end
600:       ungetc
601:       Token(TkSPACE).set_text(chars)
602:     end
603: 
604:     @OP.def_rule("#") do
605:       |op, io|
606:       identify_comment
607:     end
608: 
609:     @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
610:       |op, io|
611:       str = op
612:       @ltype = "="
613: 
614: 
615:       begin
616:         line = ""
617:         begin
618:           ch = getc
619:           line << ch
620:         end until ch == "\n"
621:         str << line
622:       end until line =~ /^=end/
623: 
624:       ungetc
625: 
626:       @ltype = nil
627: 
628:       if str =~ /\A=begin\s+rdoc/i
629:         str.sub!(/\A=begin.*\n/, '')
630:         str.sub!(/^=end.*/m, '')
631:         Token(TkCOMMENT).set_text(str)
632:       else
633:         Token(TkRD_COMMENT)#.set_text(str)
634:       end
635:     end
636: 
637:     @OP.def_rule("\n") do
638:       print "\\n\n" if RubyLex.debug?
639:       case @lex_state
640:       when EXPR_BEG, EXPR_FNAME, EXPR_DOT
641:         @continue = TRUE
642:       else
643:         @continue = FALSE
644:         @lex_state = EXPR_BEG
645:       end
646:       Token(TkNL).set_text("\n")
647:     end
648: 
649:     @OP.def_rules("*", "**",    
650:                   "!", "!=", "!~",
651:                   "=", "==", "===", 
652:                   "=~", "<=>",        
653:                   "<", "<=",
654:                   ">", ">=", ">>") do
655:       |op, io|
656:       @lex_state = EXPR_BEG
657:       Token(op).set_text(op)
658:     end
659: 
660:     @OP.def_rules("<<") do
661:       |op, io|
662:       tk = nil
663:       if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
664:           (@lex_state != EXPR_ARG || @space_seen)
665:         c = peek(0)
666:         if /[-\w_\"\'\`]/ =~ c
667:           tk = identify_here_document
668:         end
669:       end
670:       if !tk
671:         @lex_state = EXPR_BEG
672:         tk = Token(op).set_text(op)
673:       end
674:       tk
675:     end
676: 
677:     @OP.def_rules("'", '"') do
678:       |op, io|
679:       identify_string(op)
680:     end
681: 
682:     @OP.def_rules("`") do
683:       |op, io|
684:       if @lex_state == EXPR_FNAME
685:         Token(op).set_text(op)
686:       else
687:         identify_string(op)
688:       end
689:     end
690: 
691:     @OP.def_rules('?') do
692:       |op, io|
693:       if @lex_state == EXPR_END
694:         @lex_state = EXPR_BEG
695:         Token(TkQUESTION).set_text(op)
696:       else
697:         ch = getc
698:         if @lex_state == EXPR_ARG && ch !~ /\s/
699:           ungetc
700:           @lex_state = EXPR_BEG;
701:           Token(TkQUESTION).set_text(op)
702:         else
703:           str = op
704:           str << ch
705:           if (ch == '\\') #'
706:             str << read_escape
707:           end
708:           @lex_state = EXPR_END
709:           Token(TkINTEGER).set_text(str)
710:         end
711:       end
712:     end
713: 
714:     @OP.def_rules("&", "&&", "|", "||") do
715:       |op, io|
716:       @lex_state = EXPR_BEG
717:       Token(op).set_text(op)
718:     end
719:     
720:     @OP.def_rules("+=", "-=", "*=", "**=", 
721:                   "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
722:       |op, io|
723:       @lex_state = EXPR_BEG
724:       op =~ /^(.*)=$/
725:       Token(TkOPASGN, $1).set_text(op)
726:     end
727: 
728:     @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
729:       Token(TkUPLUS).set_text(op)
730:     end
731: 
732:     @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
733:       Token(TkUMINUS).set_text(op)
734:     end
735: 
736:     @OP.def_rules("+", "-") do
737:       |op, io|
738:       catch(:RET) do
739:         if @lex_state == EXPR_ARG
740:           if @space_seen and peek(0) =~ /[0-9]/
741:             throw :RET, identify_number(op)
742:           else
743:             @lex_state = EXPR_BEG
744:           end
745:         elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
746:           throw :RET, identify_number(op)
747:         else
748:           @lex_state = EXPR_BEG
749:         end
750:         Token(op).set_text(op)
751:       end
752:     end
753: 
754:     @OP.def_rule(".") do
755:       @lex_state = EXPR_BEG
756:       if peek(0) =~ /[0-9]/
757:         ungetc
758:         identify_number("")
759:       else
760:         # for obj.if
761:         @lex_state = EXPR_DOT
762:         Token(TkDOT).set_text(".")
763:       end
764:     end
765: 
766:     @OP.def_rules("..", "...") do
767:       |op, io|
768:       @lex_state = EXPR_BEG
769:       Token(op).set_text(op)
770:     end
771: 
772:     lex_int2
773:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 533
533:   def lex_int2
534:     @OP.def_rules("]", "}", ")") do
535:       |op, io|
536:       @lex_state = EXPR_END
537:       @indent -= 1
538:       @indent_stack.pop
539:       Token(op)
540:     end
541: 
542:     @OP.def_rule(":") do
543:       |op, io|
544:       if @lex_state == EXPR_END || peek(0) =~ /\s/
545:         @lex_state = EXPR_BEG
546:         Token(TkCOLON)
547:       else
548:         @lex_state = EXPR_FNAME;
549:         Token(TkSYMBEG)
550:       end
551:     end
552: 
553:     @OP.def_rule("::") do
554:        |op, io|
555: #      p @lex_state.id2name, @space_seen
556:       if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
557:         @lex_state = EXPR_BEG
558:         Token(TkCOLON3)
559:       else
560:         @lex_state = EXPR_DOT
561:         Token(TkCOLON2)
562:       end
563:     end
564: 
565:     @OP.def_rule("/") do
566:       |op, io|
567:       if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
568:         identify_string(op)
569:       elsif peek(0) == '='
570:         getc
571:         @lex_state = EXPR_BEG
572:         Token(TkOPASGN, "/") #/)
573:       elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
574:         identify_string(op)
575:       else 
576:         @lex_state = EXPR_BEG
577:         Token("/") #/)
578:       end
579:     end
580: 
581:     @OP.def_rules("^") do
582:       |op, io|
583:       @lex_state = EXPR_BEG
584:       Token("^")
585:     end
586: 
587:     #       @OP.def_rules("^=") do
588:     #   @lex_state = EXPR_BEG
589:     #   Token(OP_ASGN, :^)
590:     #       end
591:     
592:     @OP.def_rules(",") do
593:       |op, io|
594:       @lex_state = EXPR_BEG
595:       Token(op)
596:     end
597: 
598:     @OP.def_rules(";") do
599:       |op, io|
600:       @lex_state = EXPR_BEG
601:       until (@indent_stack.empty? || 
602:              [TkLPAREN, TkLBRACK, TkLBRACE, 
603:                TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
604:         @indent_stack.pop
605:       end
606:       Token(op)
607:     end
608: 
609:     @OP.def_rule("~") do
610:       |op, io|
611:       @lex_state = EXPR_BEG
612:       Token("~")
613:     end
614: 
615:     @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
616:       |op, io|
617:       @lex_state = EXPR_BEG
618:       Token("~")
619:     end
620:     
621:     @OP.def_rule("(") do
622:       |op, io|
623:       @indent += 1
624:       if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
625:         @lex_state = EXPR_BEG
626:         tk_c = TkfLPAREN
627:       else
628:         @lex_state = EXPR_BEG
629:         tk_c = TkLPAREN
630:       end
631:       @indent_stack.push tk_c
632:       tk = Token(tk_c)
633:     end
634: 
635:     @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
636:       |op, io|
637:       @lex_state = EXPR_ARG
638:       Token("[]")
639:     end
640: 
641:     @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
642:       |op, io|
643:       @lex_state = EXPR_ARG
644:       Token("[]=")
645:     end
646: 
647:     @OP.def_rule("[") do
648:       |op, io|
649:       @indent += 1
650:       if @lex_state == EXPR_FNAME
651:         tk_c = TkfLBRACK
652:       else
653:         if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
654:           tk_c = TkLBRACK
655:         elsif @lex_state == EXPR_ARG && @space_seen
656:           tk_c = TkLBRACK
657:         else
658:           tk_c = TkfLBRACK
659:         end
660:         @lex_state = EXPR_BEG
661:       end
662:       @indent_stack.push tk_c
663:       Token(tk_c)
664:     end
665: 
666:     @OP.def_rule("{") do
667:       |op, io|
668:       @indent += 1
669:       if @lex_state != EXPR_END && @lex_state != EXPR_ARG
670:         tk_c = TkLBRACE
671:       else
672:         tk_c = TkfLBRACE
673:       end
674:       @lex_state = EXPR_BEG
675:       @indent_stack.push tk_c
676:       Token(tk_c)
677:     end
678: 
679:     @OP.def_rule('\\') do
680:       |op, io|
681:       if getc == "\n"
682:         @space_seen = true
683:         @continue = true
684:         Token(TkSPACE)
685:       else
686:         ungetc
687:         Token("\\")
688:       end
689:     end
690: 
691:     @OP.def_rule('%') do
692:       |op, io|
693:       if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
694:         identify_quotation
695:       elsif peek(0) == '='
696:         getc
697:         Token(TkOPASGN, :%)
698:       elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
699:         identify_quotation
700:       else
701:         @lex_state = EXPR_BEG
702:         Token("%") #))
703:       end
704:     end
705: 
706:     @OP.def_rule('$') do
707:       |op, io|
708:       identify_gvar
709:     end
710: 
711:     @OP.def_rule('@') do
712:       |op, io|
713:       if peek(0) =~ /[\w_@]/
714:         ungetc
715:         identify_identifier
716:       else
717:         Token("@")
718:       end
719:     end
720: 
721:     #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
722:     #   |op, io|
723:     #   @indent += 1
724:     #   @lex_state = EXPR_FNAME
725:     # # @lex_state = EXPR_END
726:     # # until @rests[0] == "\n" or @rests[0] == ";"
727:     # #   rests.shift
728:     # # end
729:     #       end
730: 
731:     @OP.def_rule("") do
732:       |op, io|
733:       printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
734:       if peek(0) =~ /[0-9]/
735:         t = identify_number
736:       elsif peek(0) =~ /[\w_]/
737:         t = identify_identifier
738:       end
739:       printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
740:       t
741:     end
742:     
743:     p @OP if RubyLex.debug?
744:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 775
775:   def lex_int2
776:     @OP.def_rules("]", "}", ")") do
777:       |op, io|
778:       @lex_state = EXPR_END
779:       @indent -= 1
780:       Token(op).set_text(op)
781:     end
782: 
783:     @OP.def_rule(":") do
784:       if @lex_state == EXPR_END || peek(0) =~ /\s/
785:         @lex_state = EXPR_BEG
786:         tk = Token(TkCOLON)
787:       else
788:         @lex_state = EXPR_FNAME;
789:         tk = Token(TkSYMBEG)
790:       end
791:       tk.set_text(":")
792:     end
793: 
794:     @OP.def_rule("::") do
795: #      p @lex_state.id2name, @space_seen
796:       if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
797:         @lex_state = EXPR_BEG
798:         tk = Token(TkCOLON3)
799:       else
800:         @lex_state = EXPR_DOT
801:         tk = Token(TkCOLON2)
802:       end
803:       tk.set_text("::")
804:     end
805: 
806:     @OP.def_rule("/") do
807:       |op, io|
808:       if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
809:         identify_string(op)
810:       elsif peek(0) == '='
811:         getc
812:         @lex_state = EXPR_BEG
813:         Token(TkOPASGN, :/).set_text("/=") #")
814:       elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
815:         identify_string(op)
816:       else 
817:         @lex_state = EXPR_BEG
818:         Token("/").set_text(op)
819:       end
820:     end
821: 
822:     @OP.def_rules("^") do
823:       @lex_state = EXPR_BEG
824:       Token("^").set_text("^")
825:     end
826: 
827:     #       @OP.def_rules("^=") do
828:     #   @lex_state = EXPR_BEG
829:     #   Token(TkOPASGN, :^)
830:     #       end
831:     
832:     @OP.def_rules(",", ";") do
833:       |op, io|
834:       @lex_state = EXPR_BEG
835:       Token(op).set_text(op)
836:     end
837: 
838:     @OP.def_rule("~") do
839:       @lex_state = EXPR_BEG
840:       Token("~").set_text("~")
841:     end
842: 
843:     @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
844:       @lex_state = EXPR_BEG
845:       Token("~").set_text("~@")
846:     end
847:     
848:     @OP.def_rule("(") do
849:       @indent += 1
850:       if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
851:         @lex_state = EXPR_BEG
852:         tk = Token(TkfLPAREN)
853:       else
854:         @lex_state = EXPR_BEG
855:         tk = Token(TkLPAREN)
856:       end
857:       tk.set_text("(")
858:     end
859: 
860:     @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
861:       Token("[]").set_text("[]")
862:     end
863: 
864:     @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
865:       Token("[]=").set_text("[]=")
866:     end
867: 
868:     @OP.def_rule("[") do
869:       @indent += 1
870:       if @lex_state == EXPR_FNAME
871:         t = Token(TkfLBRACK)
872:       else
873:         if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
874:           t = Token(TkLBRACK)
875:         elsif @lex_state == EXPR_ARG && @space_seen
876:           t = Token(TkLBRACK)
877:         else
878:           t = Token(TkfLBRACK)
879:         end
880:         @lex_state = EXPR_BEG
881:       end
882:       t.set_text("[")
883:     end
884: 
885:     @OP.def_rule("{") do
886:       @indent += 1
887:       if @lex_state != EXPR_END && @lex_state != EXPR_ARG
888:         t = Token(TkLBRACE)
889:       else
890:         t = Token(TkfLBRACE)
891:       end
892:       @lex_state = EXPR_BEG
893:       t.set_text("{")
894:     end
895: 
896:     @OP.def_rule('\\') do   #'
897:       if getc == "\n" 
898:         @space_seen = true
899:         @continue = true
900:         Token(TkSPACE).set_text("\\\n")
901:       else 
902:         ungetc
903:         Token("\\").set_text("\\")  #"
904:       end 
905:     end 
906: 
907:     @OP.def_rule('%') do
908:       |op, io|
909:       if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
910:         identify_quotation('%')
911:       elsif peek(0) == '='
912:         getc
913:         Token(TkOPASGN, "%").set_text("%=")
914:       elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
915:         identify_quotation('%')
916:       else
917:         @lex_state = EXPR_BEG
918:         Token("%").set_text("%")
919:       end
920:     end
921: 
922:     @OP.def_rule('$') do  #'
923:       identify_gvar
924:     end
925: 
926:     @OP.def_rule('@') do
927:       if peek(0) =~ /[@\w_]/
928:         ungetc
929:         identify_identifier
930:       else
931:         Token("@").set_text("@")
932:       end
933:     end
934: 
935:     #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
936:     #   |op, io|
937:     #   @indent += 1
938:     #   @lex_state = EXPR_FNAME
939:     # # @lex_state = EXPR_END
940:     # # until @rests[0] == "\n" or @rests[0] == ";"
941:     # #   rests.shift
942:     # # end
943:     #       end
944: 
945:     @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
946:       throw :eof
947:     end
948: 
949:     @OP.def_rule("") do
950:       |op, io|
951:       printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
952:       if peek(0) =~ /[0-9]/
953:         t = identify_number("")
954:       elsif peek(0) =~ /[\w_]/
955:         t = identify_identifier
956:       end
957:       printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
958:       t
959:     end
960:     
961:     p @OP if RubyLex.debug?
962:   end

io functions

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 476
476:   def line_no
477:     @reader.line_num
478:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 180
180:   def peek(i = 0)
181:     while @rests.size <= i
182:       return nil unless buf_input
183:     end
184:     @rests[i]
185:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 515
515:   def peek(i = 0)
516:     @reader.peek(i)
517:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 165
165:   def peek_equal?(str)
166:     chrs = str.split(//)
167:     until @rests.size >= chrs.size
168:       return false unless buf_input
169:     end
170:     @rests[0, chrs.size] == chrs
171:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 511
511:   def peek_equal?(str)
512:     @reader.peek_equal(str)
513:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 173
173:   def peek_match?(regexp)
174:     while @rests.empty?
175:       return false unless buf_input
176:     end
177:     regexp =~ @rests.join("")
178:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 205
205:   def prompt
206:     if @prompt
207:       @prompt.call(@ltype, @indent, @continue, @line_no)
208:     end
209:   end

[Source]

      # File lib/irb/ruby-lex.rb, line 1101
1101:   def read_escape
1102:     case ch = getc
1103:     when "\n", "\r", "\f"
1104:     when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
1105:     when /[0-7]/
1106:       ungetc ch
1107:       3.times do
1108:         case ch = getc
1109:         when /[0-7]/
1110:         when nil
1111:           break
1112:         else
1113:           ungetc
1114:           break
1115:         end
1116:       end
1117:       
1118:     when "x"
1119:       2.times do
1120:         case ch = getc
1121:         when /[0-9a-fA-F]/
1122:         when nil
1123:           break
1124:         else
1125:           ungetc
1126:           break
1127:         end
1128:       end
1129: 
1130:     when "M"
1131:       if (ch = getc) != '-'
1132:         ungetc
1133:       else
1134:         if (ch = getc) == "\\" #"
1135:           read_escape
1136:         end
1137:       end
1138: 
1139:     when "C", "c" #, "^"
1140:       if ch == "C" and (ch = getc) != "-"
1141:         ungetc
1142:       elsif (ch = getc) == "\\" #"
1143:         read_escape
1144:       end
1145:     else
1146:       # other characters 
1147:     end
1148:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1292
1292:   def read_escape
1293:     res = ""
1294:     case ch = getc
1295:     when /[0-7]/
1296:       ungetc ch
1297:       3.times do
1298:         case ch = getc
1299:         when /[0-7]/
1300:         when nil
1301:           break
1302:         else
1303:           ungetc
1304:           break
1305:         end
1306:         res << ch
1307:       end
1308:       
1309:     when "x"
1310:       res << ch
1311:       2.times do
1312:         case ch = getc
1313:         when /[0-9a-fA-F]/
1314:         when nil
1315:           break
1316:         else
1317:           ungetc
1318:           break
1319:         end
1320:         res << ch
1321:       end
1322: 
1323:     when "M"
1324:       res << ch
1325:       if (ch = getc) != '-'
1326:         ungetc
1327:       else
1328:         res << ch
1329:         if (ch = getc) == "\\" #"
1330:           res << ch
1331:           res << read_escape
1332:         else
1333:           res << ch
1334:         end
1335:       end
1336: 
1337:     when "C", "c" #, "^"
1338:       res << ch
1339:       if ch == "C" and (ch = getc) != "-"
1340:         ungetc
1341:       else
1342:         res << ch
1343:         if (ch = getc) == "\\" #"
1344:           res << ch
1345:           res << read_escape
1346:         else
1347:           res << ch
1348:         end
1349:       end
1350:     else
1351:       res << ch
1352:     end
1353:     res
1354:   end

io functions

[Source]

    # File lib/irb/ruby-lex.rb, line 78
78:   def set_input(io, p = nil, &block)
79:     @io = io
80:     if p.respond_to?(:call)
81:       @input = p
82:     elsif block_given?
83:       @input = block
84:     else
85:       @input = Proc.new{@io.gets}
86:     end
87:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 196
196:   def set_prompt(p = nil, &block)
197:     p = block if block_given?
198:     if p.respond_to?(:call)
199:       @prompt = p
200:     else
201:       @prompt = Proc.new{print p}
202:     end
203:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1254
1254:   def skip_inner_expression
1255:     res = ""
1256:     nest = 0
1257:     while (ch = getc)
1258:       res << ch
1259:       if ch == '}'
1260:         break if nest.zero?
1261:         nest -= 1
1262:       elsif ch == '{'
1263:         nest += 1
1264:       end
1265:     end
1266:     res
1267:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 279
279:   def token
280:     #      require "tracer"
281:     #      Tracer.on
282:     @prev_seek = @seek
283:     @prev_line_no = @line_no
284:     @prev_char_no = @char_no
285:     begin
286:       begin
287:         tk = @OP.match(self)
288:         @space_seen = tk.kind_of?(TkSPACE)
289:       rescue SyntaxError
290:         raise if @exception_on_syntax_error
291:         tk = TkError.new(@seek, @line_no, @char_no)
292:       end
293:     end while @skip_space and tk.kind_of?(TkSPACE)
294:     if @readed_auto_clean_up
295:       get_readed
296:     end
297:     #      Tracer.off
298:     tk
299:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 533
533:   def token
534:     set_token_position(line_no, char_no)
535:     begin
536:       begin
537:         tk = @OP.match(self)
538:         @space_seen = tk.kind_of?(TkSPACE)
539:       rescue SyntaxError
540:         abort if @exception_on_syntax_error
541:         tk = TkError.new(line_no, char_no)
542:       end
543:     end while @skip_space and tk.kind_of?(TkSPACE)
544:     if @read_auto_clean_up
545:       get_read
546:     end
547: #   throw :eof unless tk
548:     p tk if $DEBUG
549:     tk
550:   end

[Source]

     # File lib/irb/ruby-lex.rb, line 144
144:   def ungetc(c = nil)
145:     if @here_readed.empty?
146:       c2 = @readed.pop
147:     else
148:       c2 = @here_readed.pop
149:     end
150:     c = c2 unless c
151:     @rests.unshift c #c = 
152:       @seek -= 1
153:     if c == "\n"
154:       @line_no -= 1 
155:       if idx = @readed.reverse.index("\n")
156:         @char_no = @readed.size - idx
157:       else
158:         @char_no = @base_char_no + @readed.size
159:       end
160:     else
161:       @char_no -= 1
162:     end
163:   end

[Source]

     # File lib/rdoc/parsers/parse_rb.rb, line 507
507:   def ungetc(c = nil)
508:     @reader.ungetc(c)
509:   end

Private Instance methods

[Source]

     # File lib/irb/ruby-lex.rb, line 187
187:   def buf_input
188:     prompt
189:     line = @input.call
190:     return nil unless line
191:     @rests.concat line.split(//)
192:     true
193:   end

[Validate]