# File gtk/sample/gtk-demo/main.rb, line 320
    def tokenize(str, index = 0)
      until str.empty?
        tag = nil

        case str
        when /".+?"/, /'.+?'/
          tag = :string
        when /#.*$/
          tag = :comment
        when RESERVED_WORDS_PATTERN
          tag = :reserved
        when /[A-Z][A-Za-z0-9_]+/
          tag = :const
        end

        if tag
          tokenize($~.pre_match, index) do |*args|
            yield(*args)
          end
          yield(tag, index + $~.begin(0), index + $~.end(0))
          index += (str.length - $~.post_match.length)
          str = $~.post_match
        else
          index += str.length
          str = ''
        end
      end
    end