Merge branch 'master' into autoload

Author: korny

.travis.yml

@@ -2,7 +2,10 @@ rvm:
   - 1.8.7
   - ree
   - 1.9.3
-  - 2.0.0
+  - 2.0
+  - 2.1
+  - 2.2
+  - 2.3.0
   - ruby-head
   - jruby-18mode
   - jruby-19mode
@@ -19,3 +22,4 @@ matrix:
     - rvm: rbx-18mode
     - rvm: rbx-19mode
 script: "rake test" # test:scanners"
+sudo: false

Changes.textile

@@ -2,6 +2,17 @@ h1=. CodeRay Version History
 
 p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._
 
+h2. Changes in 1.1.1
+
+* SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson]
+* SQL scanner: Fix open strings [#163, thanks to Adam]
+* Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1)
+* Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2)
+* Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3)
+* Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3)
+* Diff scanner: Prevent running out of regexp stack.
+* HTML encoder: You can keep tabs intact now by setting @tab_width: false@.
+
 h2. Changes in 1.1
 
 New scanners:

Gemfile

@@ -6,11 +6,13 @@ gemspec
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "bundler", ">= 1.0.0"
+  gem "bundler"
   gem "rake"
   gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3"
-  gem "term-ansicolor", '~> 1.2.2'
-  gem "shoulda-context", "~> 1.1.2"
+  gem "term-ansicolor"
+  gem 'tins', '~> 1.6.0'
+  gem "shoulda-context"
+  gem "test-unit"
   gem "json" if RUBY_VERSION < '1.9'
   gem "rdoc"
 end

README.markdown

@@ -16,7 +16,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co
 
 ### Dependencies
 
-CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby.
+CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby.
 
 ## Example Usage
 

bin/coderay

@@ -35,7 +35,7 @@ defaults:
 
 common:
   coderay file.rb                      # highlight file to terminal
-  coderay file.rb > file.html          # highlight file to HTML page
+  coderay file.rb -page > file.html    # highlight file to HTML page
   coderay file.rb -div > file.html     # highlight file to HTML snippet
 
 configure output:

lib/coderay/encoders/html.rb

@@ -25,7 +25,8 @@ module Encoders
   # == Options
   #
   # === :tab_width
-  # Convert \t characters to +n+ spaces (a number.)
+  # Convert \t characters to +n+ spaces (a number or false.)
+  # false will keep tab characters untouched.
   # 
   # Default: 8
   #
@@ -180,7 +181,7 @@ def setup options
 
       @break_lines = (options[:break_lines] == true)
 
-      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
+      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t")
 
       @opened = []
       @last_opened = nil

lib/coderay/scanners/diff.rb

@@ -100,7 +100,7 @@ def scan_tokens encoder, options
             next
           elsif match = scan(/-/)
             deleted_lines_count += 1
-            if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
+            if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
               deleted_lines  = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) }
               inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/)         ; scan(/.*/) }
 

lib/coderay/scanners/ruby.rb

@@ -164,15 +164,19 @@ def scan_tokens encoder, options
               end
 
             elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx)
-              encoder.begin_group :string
               if match.size == 1
+                kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string
+                encoder.begin_group kind
                 encoder.text_token match, :delimiter
-                state = self.class::StringState.new :string, match == '"', match  # important for streaming
+                state = self.class::StringState.new kind, match == '"', match  # important for streaming
               else
+                kind = value_expected == true && scan(/:/) ? :key : :string
+                encoder.begin_group kind
                 encoder.text_token match[0,1], :delimiter
                 encoder.text_token match[1..-2], :content if match.size > 2
                 encoder.text_token match[-1,1], :delimiter
-                encoder.end_group :string
+                encoder.end_group kind
+                encoder.text_token ':', :operator if kind == :key
                 value_expected = false
               end
 
@@ -191,11 +195,14 @@ def scan_tokens encoder, options
                 encoder.text_token match, :error
                 method_call_expected = false
               else
-                encoder.text_token match, self[1] ? :float : :integer  # TODO: send :hex/:octal/:binary
+                kind = self[1] ? :float : :integer  # TODO: send :hex/:octal/:binary
+                match << 'r' if match !~ /e/i && scan(/r/)
+                match << 'i' if scan(/i/)
+                encoder.text_token match, kind
               end
               value_expected = false
 
-            elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x)
+            elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x)
               value_expected = true
               encoder.text_token match, :operator
 
@@ -208,7 +215,7 @@ def scan_tokens encoder, options
               encoder.end_group kind
               heredocs ||= []  # create heredocs if empty
               heredocs << self.class::StringState.new(kind, quote != "'", delim,
-                self[1] == '-' ? :indented : :linestart)
+                self[1] ? :indented : :linestart)
               value_expected = false
 
             elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o)

lib/coderay/scanners/ruby/patterns.rb

@@ -114,7 +114,7 @@ module Ruby::Patterns  # :nodoc: all
     # NOTE: This is not completely correct, but
     # nobody needs heredoc delimiters ending with \n.
     HEREDOC_OPEN = /
-      << (-)?              # $1 = float
+      << ([-~])?           # $1 = float
       (?:
         ( [A-Za-z_0-9]+ )  # $2 = delim
       |

lib/coderay/scanners/ruby/string_state.rb

@@ -37,6 +37,14 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
         end
       end
 
+      def self.simple_key_pattern delim
+        if delim == "'"
+          / (?> (?: [^\\']+ | \\. )* ) ' : /mx
+        else
+          / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx
+        end
+      end
+      
       def initialize kind, interpreted, delim, heredoc = false
         if heredoc
           pattern = heredoc_pattern delim, interpreted, heredoc == :indented

lib/coderay/scanners/sql.rb

@@ -57,6 +57,12 @@ class SQL < Scanner
 
     STRING_PREFIXES = /[xnb]|_\w+/i
 
+    STRING_CONTENT_PATTERN = {
+      '"' => / (?: [^\\"] | "" )+ /x,
+      "'" => / (?: [^\\'] | '' )+ /x,
+      '`' => / (?: [^\\`] | `` )+ /x,
+    }
+    
     def scan_tokens encoder, options
 
       state = :initial
@@ -90,7 +96,7 @@ def scan_tokens encoder, options
             state = :string
             encoder.text_token match, :delimiter
 
-          elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
+          elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x)
             encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match])
             name_expected = false
 
@@ -115,40 +121,26 @@ def scan_tokens encoder, options
           end
 
         elsif state == :string
-          if match = scan(/[^\\"'`]+/)
-            string_content << match
-            next
+          if match = scan(STRING_CONTENT_PATTERN[string_type])
+            encoder.text_token match, :content
           elsif match = scan(/["'`]/)
             if string_type == match
               if peek(1) == string_type  # doubling means escape
-                string_content << string_type << getch
-                next
-              end
-              unless string_content.empty?
-                encoder.text_token string_content, :content
-                string_content = ''
+                encoder.text_token match + getch, :content
+              else
+                encoder.text_token match, :delimiter
+                encoder.end_group :string
+                state = :initial
+                string_type = nil
               end
-              encoder.text_token match, :delimiter
-              encoder.end_group :string
-              state = :initial
-              string_type = nil
             else
-              string_content << match
+              encoder.text_token match, :content
             end
           elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
-            unless string_content.empty?
-              encoder.text_token string_content, :content
-              string_content = ''
-            end
             encoder.text_token match, :char
           elsif match = scan(/ \\ . /mox)
-            string_content << match
-            next
+            encoder.text_token match, :content
           elsif match = scan(/ \\ | $ /x)
-            unless string_content.empty?
-              encoder.text_token string_content, :content
-              string_content = ''
-            end
             encoder.text_token match, :error unless match.empty?
             encoder.end_group :string
             state = :initial

lib/coderay/version.rb

@@ -1,3 +1,3 @@
 module CodeRay
-  VERSION = '1.1.0'
+  VERSION = '1.1.1'
 end