From 419727a73af94057ca0980733e69ac8b4d52fdf4 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 14 May 2023 18:01:54 +0200 Subject: [PATCH 01/73] ERB: Managed to format first example - Based on syntax_tree-xml - Added an example that uses multiple ERB-features and blocks --- .gitignore | 2 +- CHANGELOG.md | 8 +- Gemfile.lock | 9 +- README.md | 77 +-- lib/syntax_tree/{xml.rb => erb.rb} | 17 +- lib/syntax_tree/erb/format.rb | 188 +++++++ lib/syntax_tree/{xml => erb}/nodes.rb | 335 +++++++------ lib/syntax_tree/erb/parser.rb | 539 +++++++++++++++++++++ lib/syntax_tree/erb/pretty_print.rb | 168 +++++++ lib/syntax_tree/{xml => erb}/version.rb | 4 +- lib/syntax_tree/{xml => erb}/visitor.rb | 30 +- lib/syntax_tree/xml/format.rb | 238 --------- lib/syntax_tree/xml/parser.rb | 384 --------------- lib/syntax_tree/xml/pretty_print.rb | 88 ---- syntax_tree-erb.gemspec | 33 ++ syntax_tree-xml.gemspec | 33 -- test/erb_test.rb | 18 + test/fixture/example1_formatted.html.erb | 20 + test/fixture/example1_unformatted.html.erb | 17 + test/fixture/formatted.xml | 69 --- test/fixture/unformatted.xml | 63 --- test/test_helper.rb | 2 +- test/xml_test.rb | 16 - 23 files changed, 1232 insertions(+), 1126 deletions(-) rename lib/syntax_tree/{xml.rb => erb.rb} (54%) create mode 100644 lib/syntax_tree/erb/format.rb rename lib/syntax_tree/{xml => erb}/nodes.rb (67%) create mode 100644 lib/syntax_tree/erb/parser.rb create mode 100644 lib/syntax_tree/erb/pretty_print.rb rename lib/syntax_tree/{xml => erb}/version.rb (62%) rename lib/syntax_tree/{xml => erb}/visitor.rb (67%) delete mode 100644 lib/syntax_tree/xml/format.rb delete mode 100644 lib/syntax_tree/xml/parser.rb delete mode 100644 lib/syntax_tree/xml/pretty_print.rb create mode 100644 syntax_tree-erb.gemspec delete mode 100644 syntax_tree-xml.gemspec create mode 100644 test/erb_test.rb create mode 100644 test/fixture/example1_formatted.html.erb create mode 100644 test/fixture/example1_unformatted.html.erb delete mode 100644 test/fixture/formatted.xml delete mode 100644 test/fixture/unformatted.xml delete mode 100644 test/xml_test.rb diff --git a/.gitignore b/.gitignore index 7d84d59..3bd06d9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ /spec/reports/ /tmp/ -test.xml +test.erb diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b79387..a128997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] -## [0.1.0] - 2022-05-25 +## [0.0.1] - 2023-05-25 ### Added -- 🎉 Initial release! 🎉 +- 🎉 First version based on syntax_tree-xml 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree-xml/compare/v0.1.0...HEAD -[0.1.0]: https://github.com/ruby-syntax-tree/syntax_tree-xml/compare/c34baa...v0.1.0 +[unreleased]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.1.0...HEAD +[0.0.1]: https://github.com/davidwessman/syntax_tree-erb/compare/b280a...v0.1.0 diff --git a/Gemfile.lock b/Gemfile.lock index 8dcfc3c..3963b8d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,9 @@ PATH remote: . specs: - syntax_tree-xml (0.1.0) + syntax_tree-erb (0.0.1) prettier_print (>= 1.0.2) - syntax_tree (>= 4.0.1) + syntax_tree (>= 6.1.1) GEM remote: https://rubygems.org/ @@ -24,6 +24,7 @@ GEM PLATFORMS arm64-darwin-21 x86_64-darwin-21 + x86_64-darwin-22 x86_64-linux DEPENDENCIES @@ -31,7 +32,7 @@ DEPENDENCIES minitest rake simplecov - syntax_tree-xml! + syntax_tree-erb! BUNDLED WITH - 2.3.6 + 2.4.1 diff --git a/README.md b/README.md index ca86216..54ed91b 100644 --- a/README.md +++ b/README.md @@ -1,75 +1,50 @@ # SyntaxTree::XML -[![Build Status](https://github.com/ruby-syntax-tree/syntax_tree-xml/actions/workflows/main.yml/badge.svg)](https://github.com/ruby-syntax-tree/syntax_tree-xml/actions/workflows/main.yml) -[![Gem Version](https://img.shields.io/gem/v/syntax_tree-xml.svg)](https://rubygems.org/gems/syntax_tree-xml) +[![Build Status](https://github.com/davidwessman/syntax_tree-erb/actions/workflows/main.yml/badge.svg)](https://github.com/davidwessman/syntax_tree-erb/actions/workflows/main.yml) -[Syntax Tree](https://github.com/ruby-syntax-tree/syntax_tree) support for XML. +[Syntax Tree](https://github.com/ruby-syntax-tree/syntax_tree) support for ERB. -## Installation - -Add this line to your application's Gemfile: +## Work in progress! -```ruby -gem "syntax_tree-xml" -``` +This is not ready for production use just yet, still need to work on: -And then execute: +- Comments +- Blocks using `do` +- Blank lines +- Probably more - $ bundle install +Currently handles -Or install it yourself as: +- ERB tags with and without output +- ERB tags inside strings +- HTML tags with attributes +- HTML tags with and without closing tags +- ERB `if`, `elsif` and `else` statements +- Text output +- Formatting the ruby code inside the ERB tags (using syntax_tree itself) - $ gem install syntax_tree-xml - -## Usage +## Installation -From code: +Add this line to your application's Gemfile: ```ruby -require "syntax_tree/xml" - -pp SyntaxTree::XML.parse(source) # print out the AST -puts SyntaxTree::XML.format(source) # format the AST -``` - -From the CLI: - -```sh -$ stree ast --plugins=xml file.xml -(document - (misc "\n"), - (element - (opening_tag "<", "message", ">"), - (char_data "\n" + " "), - (element (opening_tag "<", "hello", ">"), (char_data "Hello"), (closing_tag "")), - (char_data "\n" + " "), - (element (opening_tag "<", "world", ">"), (char_data "World"), (closing_tag "")), - (char_data "\n"), - (closing_tag "") - ) -) +gem github: "davidwessman/syntax_tree-erb" ``` -or +## Usage -```sh -$ stree format --plugins=xml file.xml - - Hello - World - -``` +From code: -or +```ruby +require "syntax_tree/erb" -```sh -$ stree write --plugins=xml file.xml -file.xml 1ms +pp SyntaxTree::ERB.parse(source) # print out the AST +puts SyntaxTree::ERB.format(source) # format the AST ``` ## Contributing -Bug reports and pull requests are welcome on GitHub at https://github.com/ruby-syntax-tree/syntax_tree-xml. +Bug reports and pull requests are welcome on GitHub at https://github.com/davidwessman/syntax_tree-erb. ## License diff --git a/lib/syntax_tree/xml.rb b/lib/syntax_tree/erb.rb similarity index 54% rename from lib/syntax_tree/xml.rb rename to lib/syntax_tree/erb.rb index 8f24808..ac6bb4c 100644 --- a/lib/syntax_tree/xml.rb +++ b/lib/syntax_tree/erb.rb @@ -3,16 +3,17 @@ require "prettier_print" require "syntax_tree" -require_relative "xml/nodes" -require_relative "xml/parser" -require_relative "xml/visitor" +require_relative "erb/nodes" +require_relative "erb/parser" +require_relative "erb/visitor" -require_relative "xml/format" -require_relative "xml/pretty_print" +require_relative "erb/format" +require_relative "erb/pretty_print" module SyntaxTree - module XML - def self.format(source, maxwidth = 80) + module ERB + MAX_WIDTH=80 + def self.format(source, maxwidth = MAX_WIDTH) PrettierPrint.format(+"", maxwidth) { |q| parse(source).format(q) } end @@ -25,5 +26,5 @@ def self.read(filepath) end end - register_handler(".xml", XML) + register_handler(".erb", ERB) end diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb new file mode 100644 index 0000000..c4bc4ff --- /dev/null +++ b/lib/syntax_tree/erb/format.rb @@ -0,0 +1,188 @@ +# frozen_string_literal: true + +module SyntaxTree + module ERB + class Format < Visitor + attr_reader :q + + def initialize(q) + @q = q + end + + # Visit a Token node. + def visit_token(node) + q.text(node.value.strip) + end + + # Visit a Document node. + def visit_document(node) + child_nodes = node.child_nodes.sort_by(&:location) + + q.seplist(child_nodes, -> { q.breakable(force: true) }) do |child_node| + visit(child_node) + end + + q.breakable(force: true) + end + + def visit_html(node) + q.group do + visit(node.opening_tag) + + if node.content + q.indent do + q.breakable("") + q.seplist(node.content, -> { q.breakable(force: true) }) do |child_node| + visit(child_node) + end + end + end + + q.breakable("") + visit(node.closing_tag) + end + end + + # Visit an ErbNode node. + def visit_erb(node) + visit(node.opening_tag) + visit(node.content) + visit(node.closing_tag) + end + + # Visit an ErbIf node. + def visit_erb_if(node, key: "if") + q.group do + q.text("<% #{key}") + visit(node.erb_node.content) + q.text("%>") + + if node.elements.any? + q.indent do + q.breakable + q.seplist(node.elements, -> { q.breakable(force: true) }) do |child_node| + visit(child_node) + end + end + end + + q.breakable("") + visit(node.consequent) + end + end + + # Visit an ErbElsIf node. + def visit_erb_elsif(node) + visit_erb_if(node, key: "elsif") + end + + # Visit an ErbElse node. + def visit_erb_else(node) + q.group do + q.text("<% else %>") + + q.indent do + q.breakable + visit_all(node.elements) + end + + q.breakable_force + visit(node.consequent) + end + end + + # Visit an ErbEnd node. + def visit_erb_end(node) + q.text("<% end %>") + end + + def visit_erb_content(node) + if (node.value.is_a?(String)) + q.text(node.value) + else + formatter = + SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) + formatter.format(node.value.statements) + formatter.flush + formatted = [" ", *formatter.output, " "].join + q.text(formatted) + end + end + + # Visit an HtmlNode::OpeningTag node. + def visit_opening_tag(node) + q.group do + visit(node.opening) + visit(node.name) + + if node.attributes.any? + q.indent do + q.breakable + q.seplist(node.attributes, -> { q.breakable }) do |child_node| + visit(child_node) + end + end + end + + q.breakable(node.closing.value == "/>" ? " " : "") + visit(node.closing) + end + end + + # Visit an HtmlNode::ClosingTag node. + def visit_closing_tag(node) + q.group do + visit(node.opening) + visit(node.name) + visit(node.closing) + end + end + + # Visit a Reference node. + def visit_reference(node) + visit(node.value) + end + + # Visit an Attribute node. + def visit_attribute(node) + q.group do + visit(node.key) + visit(node.equals) + visit(node.value) + end + end + + # Visit an ErbString node. + def visit_erb_string(node) + q.group do + visit(node.opening) + q.seplist(node.contents, -> { "" }) { |child_node| visit(child_node) } + visit(node.closing) + end + end + + # Visit a CharData node. + def visit_char_data(node) + lines = node.value.value.strip.split("\n") + + if lines.size > 0 + q.seplist(lines, -> { q.breakable(indent: false) }) do |line| + q.text(line) + end + end + end + + private + + # Format a text by splitting nicely at newlines and spaces. + def format_text(q, value) + sep_line = -> { q.breakable(force: true, indent: false) } + sep_word = -> { q.group { q.breakable } } + + q.seplist(value.strip.split("\n"), sep_line) do |line| + q.seplist(line.split(/\b(?: +)\b/), sep_word) { |word| q.text(word) } + end + end + end + end +end diff --git a/lib/syntax_tree/xml/nodes.rb b/lib/syntax_tree/erb/nodes.rb similarity index 67% rename from lib/syntax_tree/xml/nodes.rb rename to lib/syntax_tree/erb/nodes.rb index 104a65b..3c18417 100644 --- a/lib/syntax_tree/xml/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module SyntaxTree - module XML + module ERB # A Location represents a position for a node in the source file. class Location attr_reader :start_char, :end_char, :start_line, :end_line @@ -74,18 +74,16 @@ def deconstruct_keys(keys) end end - # The Document node is the top of the syntax tree. It contains an optional - # prolog, an optional doctype declaration, any number of optional - # miscellenous elements like comments, whitespace, or processing - # instructions, and a root element. + # The Document node is the top of the syntax tree. + # It contains any number of: + # - Text + # - HtmlNode + # - ErbNodes class Document < Node - attr_reader :prolog, :miscs, :doctype, :element, :location + attr_reader :elements, :location - def initialize(prolog:, miscs:, doctype:, element:, location:) - @prolog = prolog - @miscs = miscs - @doctype = doctype - @element = element + def initialize(elements:, location:) + @elements = elements @location = location end @@ -94,120 +92,13 @@ def accept(visitor) end def child_nodes - [prolog, *miscs, doctype, element].compact + [*elements].compact end alias deconstruct child_nodes def deconstruct_keys(keys) - { - prolog: prolog, - miscs: miscs, - doctype: doctype, - element: element, - location: location - } - end - end - - # The prolog to the document includes an XML declaration which opens the - # tag, any number of attributes, and a closing of the tag. - class Prolog < Node - attr_reader :opening, :attributes, :closing, :location - - def initialize(opening:, attributes:, closing:, location:) - @opening = opening - @attributes = attributes - @closing = closing - @location = location - end - - def accept(visitor) - visitor.visit_prolog(self) - end - - def child_nodes - [opening, *attributes, closing] - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { - opening: opening, - attributes: attributes, - closing: closing, - location: location - } - end - end - - # A document type declaration is a special kind of tag that specifies the - # type of the document. It contains an opening declaration, the name of - # the document type, an optional external identifier, and a closing of the - # tag. - class DocType < Node - attr_reader :opening, :name, :external_id, :closing, :location - - def initialize(opening:, name:, external_id:, closing:, location:) - @opening = opening - @name = name - @external_id = external_id - @closing = closing - @location = location - end - - def accept(visitor) - visitor.visit_doctype(self) - end - - def child_nodes - [opening, name, external_id, closing].compact - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { - opening: opening, - name: name, - external_id: external_id, - closing: closing, - location: location - } - end - end - - # An external ID is a child of a document type declaration. It represents - # the location where the external identifier is located. It contains a - # type (either system or public), an optional public id literal, and the - # system literal. - class ExternalID < Node - attr_reader :type, :public_id, :system_id, :location - - def initialize(type:, public_id:, system_id:, location:) - @type = type - @public_id = public_id - @system_id = system_id - end - - def accept(visitor) - visitor.visit_external_id(self) - end - - def child_nodes - [type, public_id, system_id].compact - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { - type: type, - public_id: public_id, - system_id: system_id, - location: location - } + { elements: elements, location: location } end end @@ -215,7 +106,7 @@ def deconstruct_keys(keys) # optional content within the tag, and a closing tag. It can also # potentially contain an opening tag that self-closes, in which case the # content and closing tag will be nil. - class Element < Node + class HtmlNode < Node # The opening tag of an element. It contains the opening character (<), # the name of the element, any optional attributes, and the closing # token (either > or />). @@ -288,7 +179,7 @@ def initialize(opening_tag:, content:, closing_tag:, location:) end def accept(visitor) - visitor.visit_element(self) + visitor.visit_html(self) end def child_nodes @@ -307,34 +198,179 @@ def deconstruct_keys(keys) end end - # A Reference is either a character or entity reference. It contains a - # single value that is the token it contains. - class Reference < Node - attr_reader :value, :location + class ErbNode < Node + attr_reader :opening_tag, :content, :closing_tag, :location - def initialize(value:, location:) - @value = value + def initialize(opening_tag:, content:, closing_tag:, location:) + @opening_tag = opening_tag + @content = ErbContent.new(value: content) + + @closing_tag = closing_tag @location = location end def accept(visitor) - visitor.visit_reference(self) + visitor.visit_erb(self) end def child_nodes - [value] + [opening_tag, content, closing_tag].compact end alias deconstruct child_nodes def deconstruct_keys(keys) - { value: value, location: location } + { + opening_tag: opening_tag, + content: content, + closing_tag: closing_tag, + location: location + } + end + end + + class ErbBlock < Node + attr_reader :opening_tag, :content, :closing_tag, :location + + def initialize(opening_tag:, content:, closing_tag:, location:) + @opening_tag = opening_tag + @content = content + @closing_tag = closing_tag + @location = location + end + + def accept(visitor) + visitor.visit_erb(self) + end + + def child_nodes + [opening_tag, content, closing_tag].compact + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { + opening_tag: opening_tag, + content: content, + closing_tag: closing_tag, + location: location + } + end + end + + class ErbControl < Node + attr_reader :erb_node + + def initialize(erb_node:) + @erb_node = erb_node + end + + def location + erb_node.location + end + end + + class ErbIf < ErbControl + attr_reader :erb_node + + # [[HtmlNode | ErbNode | CharDataNode]] the child elements + attr_reader :elements + + # [nil | ErbElsif | ErbElse] the next clause in the chain + attr_reader :consequent + + def initialize(erb_node:, elements:, consequent:) + super(erb_node: erb_node) + @elements = elements + @consequent = consequent + end + + def accept(visitor) + visitor.visit_erb_if(self) + end + + def child_nodes + elements + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { + erb_node: erb_node, + elements: elements, + consequent: consequent, + location: location + } + end + end + + class ErbElsif < ErbIf + def accept(visitor) + visitor.visit_erb_elsif(self) + end + end + + class ErbElse < ErbIf + def accept(visitor) + visitor.visit_erb_else(self) + end + end + + class ErbEnd < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_erb_end(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { location: location } + end + end + + class ErbContent < Node + attr_reader(:value, :parsed) + + def initialize(value:) + @value = value + begin + @value = SyntaxTree.parse(@value) + @parsed = true + rescue SyntaxTree::Parser::ParseError + @parsed = false + end + end + + def accept(visitor) + visitor.visit_erb_content(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: value } end end - # An Attribute is a key-value pair within a tag. It contains the key, the + # An HtmlAttribute is a key-value pair within a tag. It contains the key, the # equals sign, and the value. - class Attribute < Node + class HtmlAttribute < Node attr_reader :key, :equals, :value, :location def initialize(key:, equals:, value:, location:) @@ -359,35 +395,40 @@ def deconstruct_keys(keys) end end - # A CharData contains either plain text or whitespace within an element. - # It wraps a single token value. - class CharData < Node - attr_reader :value, :location + # An ErbString can include ERB-tags + class ErbString < Node + attr_reader :opening, :contents, :closing, :location - def initialize(value:, location:) - @value = value + def initialize(opening:, contents:, closing:, location:) + @opening = opening + @contents = contents + @closing = closing @location = location end def accept(visitor) - visitor.visit_char_data(self) + visitor.visit_erb_string(self) end def child_nodes - [value] + [*contents] end alias deconstruct child_nodes def deconstruct_keys(keys) - { value: value, location: location } + { + opening: opening, + contents: contents, + closing: closing, + location: location + } end end - # A Misc is a catch-all for miscellaneous content outside the root tag of - # the XML document. It contains a single token which can be either a - # comment, a processing instruction, or whitespace. - class Misc < Node + # A CharData contains either plain text or whitespace within an element. + # It wraps a single token value. + class CharData < Node attr_reader :value, :location def initialize(value:, location:) @@ -396,7 +437,7 @@ def initialize(value:, location:) end def accept(visitor) - visitor.visit_misc(self) + visitor.visit_char_data(self) end def child_nodes diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb new file mode 100644 index 0000000..4e4e8f6 --- /dev/null +++ b/lib/syntax_tree/erb/parser.rb @@ -0,0 +1,539 @@ +# frozen_string_literal: true + +module SyntaxTree + module ERB + class Parser + NAME_START = + "[:a-zA-Z_\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFFD}]" + + NAME_CHAR = + "[#{NAME_START}-\\.\\d\u{00B7}\u{0300}-\u{036F}\u{203F}-\u{2040}]" + + NAME = "#{NAME_START}(?:#{NAME_CHAR})*" + + # This is the parent class of any kind of errors that will be raised by + # the parser. + class ParseError < StandardError + end + + # This error occurs when a certain token is expected in a certain place + # but is not found. Sometimes this is handled internally because some + # elements are optional. Other times it is not and it is raised to end the + # parsing process. + class MissingTokenError < ParseError + end + + class ErbKeywordError < ParseError + end + + # This error is thrown when an erb-tag with a do-statement is parsed. + # It is used to control the flow of the parser. + class ErbDoTokenError < ParseError + attr_reader(:tag) + + def initialize(tag:) + @tag = tag + end + end + + attr_reader :source, :tokens + + def initialize(source) + @source = source + @tokens = make_tokens + end + + def parse + elements = many { parse_any_tag } + + Document.new( + elements: elements, + location: elements.first.location.to(elements.last.location) + ) + end + + def debug_tokens + @tokens.each do |key, value, index, line| + puts("#{key} #{value.inspect} #{index} #{line}") + end + end + + private + + def parse_any_tag + atleast do + maybe { parse_erb_if } || maybe { parse_erb } || + maybe { parse_html_element } || maybe { parse_chardata } + end + end + + def make_tokens + Enumerator.new do |enum| + index = 0 + line = 1 + state = %i[outside] + + while index < source.length + case state.last + in :outside + case source[index..] + when /\A(?: |\t|\n|\r\n)+/m + # whitespace + # enum.yield :whitespace, $&, index, line + line += $&.count("\n") + when /\A/m + # comments + # + enum.yield :comment, $&, index, line + line += $&.count("\n") + when /\A/ + # ERB else statements + # <% else %> + enum.yield :erb_else, $&, index, line + line += $&.count("\n") + when /\A<%\s*elsif/ + # ERB elsif statements + # <% elsif + enum.yield :erb_elsif_open, $&, index, line + state << :erb + line += $&.count("\n") + when /\A<%\s*if/ + # ERB elsif statements + # <% elsif + enum.yield :erb_if_open, $&, index, line + state << :erb + line += $&.count("\n") + when /\A<%\s*end\s*%>/ + # ERB end statements + # <% end %> + enum.yield :erb_end, $&, index, line + line += $&.count("\n") + when /\A<%[=]?/ + # the beginning of an ERB tag + # <% + enum.yield :erb_open, $&, index, line + state << :erb + line += $&.count("\n") + when %r{\A/ + enum.yield :erb_close, $&, index, line + state.pop + else + enum.yield :erb_code, source[index], index, line + index += 1 + next + end + in :string + case source[index..] + when /\A(?: |\t|\n|\r\n)+/m + # whitespace + enum.yield :whitespace, $&, index, line + line += $&.count("\n") + when /\A\"/ + # the end of a quoted string + enum.yield :string_close, $&, index, line + state.pop + when /\A<%[=]?/ + # the beginning of an ERB tag + # <% + enum.yield :erb_open, $&, index, line + state << :erb + when /\A[^<&"]+/ + # plain text content + # abc + enum.yield :text, $&, index, line + else + raise ParseError, + "Unexpected character in string at #{index}: #{source[index]}" + end + in :inside + case source[index..] + when /\A[ \t\r\n]+/ + # whitespace + line += $&.count("\n") + when /\A%>/ + # the end of an ERB tag + # %> + enum.yield :erb_close, $&, index, line + state.pop + when /\A>/ + # the end of a tag + # > + enum.yield :close, $&, index, line + state.pop + when /\A\?>/ + # the end of a tag + # ?> + enum.yield :special_close, $&, index, line + state.pop + when %r{\A/>} + # the end of a self-closing tag + enum.yield :slash_close, $&, index, line + state.pop + when %r{\A/} + # a forward slash + # / + enum.yield :slash, $&, index, line + when /\A=/ + # an equals sign + # = + enum.yield :equals, $&, index, line + when /\A#{NAME}/ + # a name + # abc + enum.yield :name, $&, index, line + when /\A<%/ + # the beginning of an ERB tag + # <% + enum.yield :erb_open, $&, index, line + state << :erb + when /\A"/ + # the beginning of a string + enum.yield :string_open, $&, index, line + state << :string + else + raise ParseError, + "Unexpected character at #{index}: #{source[index]}" + end + end + + index += $&.length + end + + enum.yield :EOF, nil, index, line + end + end + + # If the next token in the list of tokens matches the expected type, then + # we're going to create a new Token, advance the token enumerator, and + # return the new Token. Otherwise we're going to raise a + # MissingTokenError. + def consume(expected) + type, value, index, line = tokens.peek + + if expected != type + raise MissingTokenError, "expected #{expected} got #{type}" + end + + tokens.next + + Token.new( + type: type, + value: value, + location: + Location.new( + start_char: index, + end_char: index + value.length, + start_line: line, + end_line: line + value.count("\n") + ) + ) + end + + # We're going to yield to the block which should attempt to consume some + # number of tokens. If any of them are missing, then we're going to return + # nil from this block. + def maybe + yield + rescue MissingTokenError + end + + # We're going to attempt to parse everything by yielding to the block. If + # nothing is returned by the block, then we're going to raise an error. + # Otherwise we'll return the value returned by the block. + def atleast + result = yield + raise MissingTokenError if result.nil? + result + end + + # We're going to attempt to parse with the block many times. We'll stop + # parsing once we get an error back from the block. + def many + items = [] + + loop do + begin + items << yield + rescue MissingTokenError + break + end + end + + items + end + + def parse_until_erb_end + items = [] + + loop do + begin + result = maybe { parse_erb_end } || maybe { parse_any_tag } + items << result + break if result.is_a?(ErbEnd) + rescue ErbKeywordError + break + end + end + + items + end + + def parse_any_tag_until_erb_keyword + items = [] + + loop do + result = + maybe { parse_erb_elsif } || maybe { parse_erb_else } || + maybe { parse_erb_end } || maybe { parse_any_tag } + items << result + break if result.is_a?(ErbControl) || result.is_a?(ErbEnd) + end + + items + end + + def parse_content + many do + atleast do + maybe { parse_html_element } || maybe { parse_chardata } || + maybe { parse_erb } || maybe { consume(:comment) } + end + end + end + + def parse_html_opening_tag + opening = consume(:open) + name = consume(:name) + attributes = many { parse_html_attribute } + + closing = + atleast do + maybe { consume(:close) } || maybe { consume(:slash_close) } + end + + HtmlNode::OpeningTag.new( + opening: opening, + name: name, + attributes: attributes, + closing: closing, + location: opening.location.to(closing.location) + ) + end + + def parse_html_closing_tag + opening = consume(:slash_open) + name = consume(:name) + closing = consume(:close) + + HtmlNode::ClosingTag.new( + opening: opening, + name: name, + closing: closing, + location: opening.location.to(closing.location) + ) + end + + def parse_html_element + opening_tag = parse_html_opening_tag + + if opening_tag.closing.value == ">" + content = parse_content + closing_tag = parse_html_closing_tag + + HtmlNode.new( + opening_tag: opening_tag, + content: content, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ) + else + HtmlNode.new( + opening_tag: opening_tag, + content: nil, + closing_tag: nil, + location: opening_tag.location + ) + end + end + + def parse_erb + parse_erb_tag + end + + def parse_erb_if + opening_tag = consume(:erb_if_open) + statement = many { consume(:erb_code) } + closing_tag = consume(:erb_close) + + contents = parse_any_tag_until_erb_keyword + + erb_tag = contents.pop + + unless erb_tag.is_a?(ErbControl) || erb_tag.is_a?(ErbEnd) + raise(ErbKeywordError, "Found no matching tag to the if-tag") + end + + ErbIf.new( + erb_node: + ErbNode.new( + opening_tag: opening_tag, + content: statement.map(&:value).join, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ), + elements: contents, + consequent: erb_tag + ) + end + + def parse_erb_elsif + opening_tag = consume(:erb_elsif_open) + statement = many { consume(:erb_code) } + closing_tag = consume(:erb_close) + + contents = parse_any_tag_until_erb_keyword + + erb_tag = contents.pop + + unless erb_tag.is_a?(ErbControl) + raise(ErbKeywordError, "Found no matching end-tag to the elsif-tag") + end + + ErbElsif.new( + erb_node: + ErbNode.new( + opening_tag: opening_tag, + content: statement.map(&:value).join, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ), + elements: contents, + consequent: erb_tag + ) + end + + def parse_erb_else + tag = consume(:erb_else) + child_nodes = parse_until_erb_end + + erb_end = child_nodes.pop + + unless erb_end.is_a?(ErbEnd) + raise(ErbKeywordError, "Found no matching end-tag for the else-tag") + end + + ErbElse.new(erb_node: tag, elements: child_nodes, consequent: erb_end) + end + + def parse_erb_end + tag = consume(:erb_end) + ErbEnd.new(location: tag.location) + end + + def parse_ruby_or_string(content) + SyntaxTree.parse(content).statements + rescue SyntaxTree::Parser::ParseError + content + end + + def parse_erb_tag + opening_tag = consume(:erb_open) + content = many { consume(:erb_code) } + closing_tag = consume(:erb_close) + + ErbNode.new( + opening_tag: opening_tag, + content: content.map(&:value).join, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ) + end + + def parse_string + opening = consume(:string_open) + contents = + many do + atleast do + maybe { consume(:text) } || maybe { consume(:whitespace) } || + maybe { parse_erb } + end + end + closing = consume(:string_close) + + ErbString.new( + opening: opening, + contents: contents, + closing: closing, + location: opening.location.to(closing.location) + ) + end + + def parse_html_attribute + key = consume(:name) + equals = consume(:equals) + value = parse_string + + HtmlAttribute.new( + key: key, + equals: equals, + value: value, + location: key.location.to(value.location) + ) + end + + def parse_chardata + values = + many do + atleast do + maybe { consume(:text) } || maybe { consume(:whitespace) } + end + end + + token = + if values.size > 1 + Token.new( + type: :text, + value: values.map(&:value).join(""), + location: values.first.location.to(values.last.location) + ) + else + values.first + end + + CharData.new(value: token, location: token.location) if token + end + end + end +end diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb new file mode 100644 index 0000000..42256da --- /dev/null +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +module SyntaxTree + module ERB + class PrettyPrint < Visitor + attr_reader :q + + def initialize(q) + @q = q + end + + # Visit a Token node. + def visit_token(node) + q.pp(node.value) + end + + # Visit a Document node. + def visit_document(node) + visit_node("document", node) + end + + # Visit an HtmlNode. + def visit_html(node) + visit_node("html", node) + end + + # Visit an HtmlNode::OpeningTag node. + def visit_opening_tag(node) + visit_node("opening_tag", node) + end + + # Visit an HtmlNode::ClosingTag node. + def visit_closing_tag(node) + visit_node("closing_tag", node) + end + + # Visit an ErbNode node. + def visit_erb(node) + q.group do + q.text("(erb") + q.nest(2) do + q.breakable + visit(node.opening_tag) + q.breakable + q.text("content") + q.breakable + visit(node.closing_tag) + end + q.breakable("") + q.text(")") + end + end + + def visit_erb_if(node, key = "erb_if") + q.group do + q.text("(#{key}") + q.nest(2) do + q.breakable + q.seplist(node.child_nodes) { |child_node| visit(child_node) } + end + q.breakable + visit(node.consequent) + q.breakable("") + q.text(")") + end + end + + def visit_erb_elsif(node) + visit_erb_if(node, "erb_elsif") + end + + def visit_erb_else(node) + visit_erb_if(node, "erb_else") + end + + def visit_erb_end(node) + q.text("(erb_end)") + end + + # Visit an ErbContent node. + def visit_erb_content(node) + q.text(node.value) + end + + # Visit a Reference node. + def visit_reference(node) + visit_node("reference", node) + end + + # Visit an Attribute node. + def visit_attribute(node) + visit_node("attribute", node) + end + + # Visit an ErbString node. + def visit_erb_string(node) + visit_node("erb_string", node) + end + + # Visit a CharData node. + def visit_char_data(node) + visit_node("char_data", node) + end + + private + + # A generic visit node function for how we pretty print nodes. + def visit_node(type, node) + q.group do + q.text("(#{type}") + q.nest(2) do + q.breakable + q.seplist(node.child_nodes) { |child_node| visit(child_node) } + end + q.breakable("") + q.text(")") + end + end + + def comments(node) + return if node.comments.empty? + + q.breakable + q.group(2, "(", ")") do + q.seplist(node.comments) { |comment| q.pp(comment) } + end + end + + def field(_name, value) + q.breakable + q.pp(value) + end + + def list(_name, values) + q.breakable + q.group(2, "(", ")") { q.seplist(values) { |value| q.pp(value) } } + end + + def node(_node, type) + q.group(2, "(", ")") do + q.text(type) + yield + end + end + + def pairs(_name, values) + q.group(2, "(", ")") do + q.seplist(values) do |(key, value)| + q.pp(key) + + if value + q.text("=") + q.group(2) do + q.breakable("") + q.pp(value) + end + end + end + end + end + + def text(_name, value) + q.breakable + q.text(value) + end + end + end +end diff --git a/lib/syntax_tree/xml/version.rb b/lib/syntax_tree/erb/version.rb similarity index 62% rename from lib/syntax_tree/xml/version.rb rename to lib/syntax_tree/erb/version.rb index 26871f5..14e53a4 100644 --- a/lib/syntax_tree/xml/version.rb +++ b/lib/syntax_tree/erb/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module SyntaxTree - module XML - VERSION = "0.1.0" + module ERB + VERSION = "0.0.1" end end diff --git a/lib/syntax_tree/xml/visitor.rb b/lib/syntax_tree/erb/visitor.rb similarity index 67% rename from lib/syntax_tree/xml/visitor.rb rename to lib/syntax_tree/erb/visitor.rb index cbf0006..fbc9350 100644 --- a/lib/syntax_tree/xml/visitor.rb +++ b/lib/syntax_tree/erb/visitor.rb @@ -1,16 +1,18 @@ # frozen_string_literal: true module SyntaxTree - module XML + module ERB # Provides a visitor interface for visiting certain nodes. It's used # internally to implement formatting and pretty-printing. It could also be # used externally to visit a subset of nodes that are relevant to a certain # task. - class Visitor + class Visitor < SyntaxTree::Visitor def visit(node) node&.accept(self) end + alias visit_statements visit_child_nodes + private def visit_all(nodes) @@ -27,22 +29,13 @@ def visit_child_nodes(node) # Visit a Document node. alias visit_document visit_child_nodes - # Visit a Prolog node. - alias visit_prolog visit_child_nodes - - # Visit a Doctype node. - alias visit_doctype visit_child_nodes - - # Visit an ExternalID node. - alias visit_external_id visit_child_nodes + # Visit an Html node. + alias visit_html visit_child_nodes - # Visit an Element node. - alias visit_element visit_child_nodes - - # Visit an Element::OpeningTag node. + # Visit an HtmlNode::OpeningTag node. alias visit_opening_tag visit_child_nodes - # Visit an Element::ClosingTag node. + # Visit an HtmlNode::ClosingTag node. alias visit_closing_tag visit_child_nodes # Visit a Reference node. @@ -54,8 +47,11 @@ def visit_child_nodes(node) # Visit a CharData node. alias visit_char_data visit_child_nodes - # Visit a Misc node. - alias visit_misc visit_child_nodes + # Visit an ErbNode node. + alias visit_erb visit_child_nodes + + # Visit an ErbString node. + alias visit_erb_string visit_child_nodes end end end diff --git a/lib/syntax_tree/xml/format.rb b/lib/syntax_tree/xml/format.rb deleted file mode 100644 index 401b455..0000000 --- a/lib/syntax_tree/xml/format.rb +++ /dev/null @@ -1,238 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module XML - class Format < Visitor - attr_reader :q - - def initialize(q) - @q = q - end - - # Visit a Token node. - def visit_token(node) - q.text(node.value.strip) - end - - # Visit a Document node. - def visit_document(node) - child_nodes = - node - .child_nodes - .select do |child_node| - case child_node - in Misc[value: Token[type: :whitespace]] - false - else - true - end - end - .sort_by(&:location) - - q.seplist(child_nodes, -> { q.breakable(force: true) }) do |child_node| - visit(child_node) - end - - q.breakable(force: true) - end - - # Visit a Prolog node. - def visit_prolog(node) - q.group do - visit(node.opening) - - if node.attributes.any? - q.indent do - q.breakable - q.seplist(node.attributes, -> { q.breakable }) do |child_node| - visit(child_node) - end - end - end - - q.breakable("") - visit(node.closing) - end - end - - # Visit a Doctype node. - def visit_doctype(node) - q.group do - visit(node.opening) - q.text(" ") - visit(node.name) - - if node.external_id - q.text(" ") - visit(node.external_id) - end - - visit(node.closing) - end - end - - # Visit an ExternalID node. - def visit_external_id(node) - q.group do - q.group do - visit(node.type) - - if node.public_id - q.indent do - q.breakable - visit(node.public_id) - end - end - end - - q.indent do - q.breakable - visit(node.system_id) - end - end - end - - # Visit an Element node. - def visit_element(node) - inner_nodes = - node.content&.select do |child_node| - case child_node - in CharData[value: Token[type: :whitespace]] - false - in CharData[value: Token[value:]] if value.strip.empty? - false - else - true - end - end - - case inner_nodes - in nil - visit(node.opening_tag) - in [] - visit( - Element::OpeningTag.new( - opening: node.opening_tag.opening, - name: node.opening_tag.name, - attributes: node.opening_tag.attributes, - closing: - Token.new( - type: :close, - value: "/>", - location: node.opening_tag.closing.location - ), - location: node.opening_tag.location - ) - ) - in [CharData[value: Token[type: :text, value:]]] - q.group do - visit(node.opening_tag) - q.indent do - q.breakable("") - format_text(q, value) - end - - q.breakable("") - visit(node.closing_tag) - end - else - q.group do - visit(node.opening_tag) - q.indent do - q.breakable("") - - inner_nodes.each_with_index do |child_node, index| - if index != 0 - q.breakable(force: true) - - end_line = inner_nodes[index - 1].location.end_line - start_line = child_node.location.start_line - q.breakable(force: true) if (start_line - end_line) >= 2 - end - - case child_node - in CharData[value: Token[type: :text, value:]] - format_text(q, value) - else - visit(child_node) - end - end - end - - q.breakable(force: true) - visit(node.closing_tag) - end - end - end - - # Visit an Element::OpeningTag node. - def visit_opening_tag(node) - q.group do - visit(node.opening) - visit(node.name) - - if node.attributes.any? - q.indent do - q.breakable - q.seplist(node.attributes, -> { q.breakable }) do |child_node| - visit(child_node) - end - end - end - - q.breakable(node.closing.value == "/>" ? " " : "") - visit(node.closing) - end - end - - # Visit an Element::ClosingTag node. - def visit_closing_tag(node) - q.group do - visit(node.opening) - visit(node.name) - visit(node.closing) - end - end - - # Visit a Reference node. - def visit_reference(node) - visit(node.value) - end - - # Visit an Attribute node. - def visit_attribute(node) - q.group do - visit(node.key) - visit(node.equals) - visit(node.value) - end - end - - # Visit a CharData node. - def visit_char_data(node) - lines = node.value.value.strip.split("\n") - - q.seplist(lines, -> { q.breakable(indent: false) }) do |line| - q.text(line) - end - end - - # Visit a Misc node. - def visit_misc(node) - visit(node.value) - end - - private - - # Format a text by splitting nicely at newlines and spaces. - def format_text(q, value) - sep_line = -> { q.breakable(force: true, indent: false) } - sep_word = -> { q.group { q.breakable } } - - q.seplist(value.strip.split("\n"), sep_line) do |line| - q.seplist(line.split(/\b(?: +)\b/), sep_word) { |word| q.text(word) } - end - end - end - end -end diff --git a/lib/syntax_tree/xml/parser.rb b/lib/syntax_tree/xml/parser.rb deleted file mode 100644 index cfc0d72..0000000 --- a/lib/syntax_tree/xml/parser.rb +++ /dev/null @@ -1,384 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module XML - class Parser - NAME_START = - "[:a-zA-Z_\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFFD}]" - - NAME_CHAR = - "[#{NAME_START}-\\.\\d\u{00B7}\u{0300}-\u{036F}\u{203F}-\u{2040}]" - - NAME = "#{NAME_START}(?:#{NAME_CHAR})*" - - # This is the parent class of any kind of errors that will be raised by - # the parser. - class ParseError < StandardError - end - - # This error occurs when a certain token is expected in a certain place - # but is not found. Sometimes this is handled internally because some - # elements are optional. Other times it is not and it is raised to end the - # parsing process. - class MissingTokenError < ParseError - end - - attr_reader :source, :tokens - - def initialize(source) - @source = source - @tokens = make_tokens - end - - def parse - parse_document - end - - private - - def make_tokens - Enumerator.new do |enum| - index = 0 - line = 1 - state = %i[outside] - - while index < source.length - case state.last - in :outside - case source[index..] - when /\A(?: |\t|\n|\r\n)+/m - # whitespace - enum.yield :whitespace, $&, index, line - line += $&.count("\n") - when /\A/m - # comments - # - enum.yield :comment, $&, index, line - line += $&.count("\n") - when /\A/m - # character data tags - # Welcome!]]> - enum.yield :cdata, $&, index, line - line += $&.count("\n") - when /\A/ - # document type definition tags - # - enum.yield :dtd, $&, index, line - when /\A<\?xml[ \t\r\n]/ - # xml declaration opening - # / - # a processing instruction - # - enum.yield :processing_instruction, $&, index, line - when /\A/ - # the end of a tag - # > - enum.yield :close, $&, index, line - state.pop - when /\A\?>/ - # the end of a tag - # ?> - enum.yield :special_close, $&, index, line - state.pop - when %r{\A/>} - # the end of a self-closing tag - enum.yield :slash_close, $&, index, line - state.pop - when %r{\A/} - # a forward slash - # / - enum.yield :slash, $&, index, line - when /\A=/ - # an equals sign - # = - enum.yield :equals, $&, index, line - when /\A(?:"[^<"]*"|'[<^']*')/ - # a quoted string - # "abc" - enum.yield :string, $&, index, line - when /\A#{NAME}/ - # a name - # abc - enum.yield :name, $&, index, line - else - raise ParseError, - "Unexpected character at #{index}: #{source[index]}" - end - end - - index += $&.length - end - - enum.yield :EOF, nil, index, line - end - end - - # If the next token in the list of tokens matches the expected type, then - # we're going to create a new Token, advance the token enumerator, and - # return the new Token. Otherwise we're going to raise a - # MissingTokenError. - def consume(expected) - type, value, index, line = tokens.peek - - if expected != type - raise MissingTokenError, "expected #{expected} got #{type}" - end - - tokens.next - - Token.new( - type: type, - value: value, - location: - Location.new( - start_char: index, - end_char: index + value.length, - start_line: line, - end_line: line + value.count("\n") - ) - ) - end - - # We're going to yield to the block which should attempt to consume some - # number of tokens. If any of them are missing, then we're going to return - # nil from this block. - def maybe - yield - rescue MissingTokenError - end - - # We're going to attempt to parse everything by yielding to the block. If - # nothing is returned by the block, then we're going to raise an error. - # Otherwise we'll return the value returned by the block. - def atleast - result = yield - raise MissingTokenError if result.nil? - result - end - - # We're going to attempt to parse with the block many times. We'll stop - # parsing once we get an error back from the block. - def many - items = [] - - loop do - begin - items << yield - rescue MissingTokenError - break - end - end - - items - end - - def parse_document - prolog = maybe { parse_prolog } - miscs = many { parse_misc } - - doctype = maybe { parse_doctype } - miscs += many { parse_misc } - - element = parse_element - miscs += many { parse_misc } - - parts = [prolog, *miscs, doctype, element].compact - - Document.new( - prolog: prolog, - miscs: miscs, - doctype: doctype, - element: element, - location: parts.first.location.to(parts.last.location) - ) - end - - def parse_prolog - opening = consume(:xml_decl) - attributes = many { parse_attribute } - closing = consume(:special_close) - - Prolog.new( - opening: opening, - attributes: attributes, - closing: closing, - location: opening.location.to(closing.location) - ) - end - - def parse_doctype - opening = consume(:doctype) - name = consume(:name) - external_id = maybe { parse_external_id } - closing = consume(:close) - - DocType.new( - opening: opening, - name: name, - external_id: external_id, - closing: closing, - location: opening.location.to(closing.location) - ) - end - - def parse_external_id - type = consume(:name) - public_id = consume(:string) if type.value == "PUBLIC" - system_id = consume(:string) - - ExternalID.new( - type: type, - public_id: public_id, - system_id: system_id, - location: type.location.to(system_id.location) - ) - end - - def parse_content - many do - atleast do - maybe { parse_element } || maybe { parse_chardata } || - maybe { parse_reference } || maybe { consume(:cdata) } || - maybe { consume(:processing_instruction) } || - maybe { consume(:comment) } - end - end - end - - def parse_opening_tag - opening = consume(:open) - name = consume(:name) - attributes = many { parse_attribute } - closing = - atleast do - maybe { consume(:close) } || maybe { consume(:slash_close) } - end - - Element::OpeningTag.new( - opening: opening, - name: name, - attributes: attributes, - closing: closing, - location: opening.location.to(closing.location) - ) - end - - def parse_closing_tag - opening = consume(:slash_open) - name = consume(:name) - closing = consume(:close) - - Element::ClosingTag.new( - opening: opening, - name: name, - closing: closing, - location: opening.location.to(closing.location) - ) - end - - def parse_element - opening_tag = parse_opening_tag - - if opening_tag.closing.value == ">" - content = parse_content - closing_tag = parse_closing_tag - - Element.new( - opening_tag: opening_tag, - content: content, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) - ) - else - Element.new( - opening_tag: opening_tag, - content: nil, - closing_tag: nil, - location: opening_tag.location - ) - end - end - - def parse_reference - value = - atleast do - maybe { consume(:entity_reference) } || - maybe { consume(:character_reference) } - end - - Reference.new(value: value, location: value.location) - end - - def parse_attribute - key = consume(:name) - equals = consume(:equals) - value = consume(:string) - - Attribute.new( - key: key, - equals: equals, - value: value, - location: key.location.to(value.location) - ) - end - - def parse_chardata - value = - atleast { maybe { consume(:text) } || maybe { consume(:whitespace) } } - - CharData.new(value: value, location: value.location) - end - - def parse_misc - value = - atleast do - maybe { consume(:comment) } || - maybe { consume(:processing_instruction) } || - maybe { consume(:whitespace) } - end - - Misc.new(value: value, location: value.location) - end - end - end -end diff --git a/lib/syntax_tree/xml/pretty_print.rb b/lib/syntax_tree/xml/pretty_print.rb deleted file mode 100644 index 22428f1..0000000 --- a/lib/syntax_tree/xml/pretty_print.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module XML - class PrettyPrint < Visitor - attr_reader :q - - def initialize(q) - @q = q - end - - # Visit a Token node. - def visit_token(node) - q.pp(node.value) - end - - # Visit a Document node. - def visit_document(node) - visit_node("document", node) - end - - # Visit a Prolog node. - def visit_prolog(node) - visit_node("prolog", node) - end - - # Visit a Doctype node. - def visit_doctype(node) - visit_node("doctype", node) - end - - # Visit an ExternalID node. - def visit_external_id(node) - visit_node("external_id", node) - end - - # Visit an Element node. - def visit_element(node) - visit_node("element", node) - end - - # Visit an Element::OpeningTag node. - def visit_opening_tag(node) - visit_node("opening_tag", node) - end - - # Visit an Element::ClosingTag node. - def visit_closing_tag(node) - visit_node("closing_tag", node) - end - - # Visit a Reference node. - def visit_reference(node) - visit_node("reference", node) - end - - # Visit an Attribute node. - def visit_attribute(node) - visit_node("attribute", node) - end - - # Visit a CharData node. - def visit_char_data(node) - visit_node("char_data", node) - end - - # Visit a Misc node. - def visit_misc(node) - visit_node("misc", node) - end - - private - - # A generic visit node function for how we pretty print nodes. - def visit_node(type, node) - q.group do - q.text("(#{type}") - q.nest(2) do - q.breakable - q.seplist(node.child_nodes) { |child_node| visit(child_node) } - end - q.breakable("") - q.text(")") - end - end - end - end -end diff --git a/syntax_tree-erb.gemspec b/syntax_tree-erb.gemspec new file mode 100644 index 0000000..3165e61 --- /dev/null +++ b/syntax_tree-erb.gemspec @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require_relative "lib/syntax_tree/erb/version" + +Gem::Specification.new do |spec| + spec.name = "syntax_tree-erb" + spec.version = SyntaxTree::ERB::VERSION + spec.authors = ["Kevin Newton", "David Wessman"] + spec.email = %w[kddnewton@gmail.com david@wessman.co] + + spec.summary = "Syntax Tree support for ERB" + spec.homepage = "https://github.com/davidwessman/syntax_tree-erb" + spec.license = "MIT" + spec.metadata = { "rubygems_mfa_required" => "true" } + + spec.files = + Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0") + .reject { |f| f.match(%r{^(test|spec|features)/}) } + end + + spec.bindir = "exe" + spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.require_paths = %w[lib] + + spec.add_dependency "prettier_print", ">= 1.0.2" + spec.add_dependency "syntax_tree", ">= 6.1.1" + + spec.add_development_dependency "bundler" + spec.add_development_dependency "minitest" + spec.add_development_dependency "rake" + spec.add_development_dependency "simplecov" +end diff --git a/syntax_tree-xml.gemspec b/syntax_tree-xml.gemspec deleted file mode 100644 index 41adc46..0000000 --- a/syntax_tree-xml.gemspec +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require_relative "lib/syntax_tree/xml/version" - -Gem::Specification.new do |spec| - spec.name = "syntax_tree-xml" - spec.version = SyntaxTree::XML::VERSION - spec.authors = ["Kevin Newton"] - spec.email = ["kddnewton@gmail.com"] - - spec.summary = "Syntax Tree support for XML" - spec.homepage = "https://github.com/ruby-syntax-tree/syntax_tree-xml" - spec.license = "MIT" - spec.metadata = { "rubygems_mfa_required" => "true" } - - spec.files = Dir.chdir(__dir__) do - `git ls-files -z`.split("\x0").reject do |f| - f.match(%r{^(test|spec|features)/}) - end - end - - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = %w[lib] - - spec.add_dependency "prettier_print", ">= 1.0.2" - spec.add_dependency "syntax_tree", ">= 4.0.1" - - spec.add_development_dependency "bundler" - spec.add_development_dependency "minitest" - spec.add_development_dependency "rake" - spec.add_development_dependency "simplecov" -end diff --git a/test/erb_test.rb b/test/erb_test.rb new file mode 100644 index 0000000..bb20892 --- /dev/null +++ b/test/erb_test.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "test_helper" + +module SyntaxTree + class ERBTest < Minitest::Test + def test_example1 + directory = File.expand_path("fixture", __dir__) + unformatted_file = File.join(directory, "example1_unformatted.html.erb") + formatted_file = File.join(directory, "example1_formatted.html.erb") + + expected = ERB.read(formatted_file) + actual = ERB.format(ERB.read(unformatted_file)) + + assert_equal(actual, expected) + end + end +end diff --git a/test/fixture/example1_formatted.html.erb b/test/fixture/example1_formatted.html.erb new file mode 100644 index 0000000..550c350 --- /dev/null +++ b/test/fixture/example1_formatted.html.erb @@ -0,0 +1,20 @@ +<%= "this" %> +<% if david %> + Hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej + what? +<% elsif that %> +

This

+
+

+ + This looks amazing + +

+
+<% else %> + Cool! +<% end %> diff --git a/test/fixture/example1_unformatted.html.erb b/test/fixture/example1_unformatted.html.erb new file mode 100644 index 0000000..7164893 --- /dev/null +++ b/test/fixture/example1_unformatted.html.erb @@ -0,0 +1,17 @@ +<%="this"%> + +<%if david%> + Hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej + what? +<% elsif that%> +

This

+
+

+ + This looks amazing + +

+
+<%else%> + Cool! +<%end%> diff --git a/test/fixture/formatted.xml b/test/fixture/formatted.xml deleted file mode 100644 index 3e68413..0000000 --- a/test/fixture/formatted.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - Style inheritance and the use element - - & - 〹 - - foo - - bar - - - - - - - - - - 1 - - 2 - - 3 - - - - - - - - - - - - - - -

- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed at est eget - enim consectetur accumsan. Aliquam pretium sodales ipsum quis dignissim. Sed - id sem vel diam luctus fringilla. Aliquam quis egestas magna. Curabitur - molestie lorem et odio porta, et molestie libero laoreet. Morbi rhoncus - sagittis cursus. Nullam vehicula pretium consequat. Praesent porta ante at - posuere sollicitudin. Nullam commodo tempor arcu, at condimentum neque - elementum ut. -

- content - -
- even more - -
- - diff --git a/test/fixture/unformatted.xml b/test/fixture/unformatted.xml deleted file mode 100644 index 596d7df..0000000 --- a/test/fixture/unformatted.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - Style inheritance and the use element - - & 〹 - - foo - - bar - - - - - - - - - - 1 - - 2 - - 3 - - - - - - - - - - - - < ignored /> - - -

- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed at est eget enim consectetur accumsan. Aliquam pretium sodales ipsum quis dignissim. Sed id sem vel diam luctus fringilla. Aliquam quis egestas magna. Curabitur molestie lorem et odio porta, et molestie libero laoreet. Morbi rhoncus sagittis cursus. Nullam vehicula pretium consequat. Praesent porta ante at posuere sollicitudin. Nullam commodo tempor arcu, at condimentum neque elementum ut. -

- - content - - - -
- even more - -
- - diff --git a/test/test_helper.rb b/test/test_helper.rb index c3ea122..365ca3c 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,6 +4,6 @@ SimpleCov.start $:.unshift File.expand_path("../lib", __dir__) -require "syntax_tree/xml" +require "syntax_tree/erb" require "minitest/autorun" diff --git a/test/xml_test.rb b/test/xml_test.rb deleted file mode 100644 index a25166c..0000000 --- a/test/xml_test.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" - -module SyntaxTree - class XMLTest < Minitest::Test - def test_formatting - directory = File.expand_path("fixture", __dir__) - - expected = XML.read(File.join(directory, "formatted.xml")) - actual = XML.format(XML.read(File.join(directory, "unformatted.xml"))) - - assert_equal(expected, actual) - end - end -end From 2cc26ac56f6c82793e814362217f24a413922e76 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 14 May 2023 18:10:03 +0200 Subject: [PATCH 02/73] Format all ruby files --- lib/syntax_tree/erb.rb | 2 +- lib/syntax_tree/erb/format.rb | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/erb.rb b/lib/syntax_tree/erb.rb index ac6bb4c..ae726fa 100644 --- a/lib/syntax_tree/erb.rb +++ b/lib/syntax_tree/erb.rb @@ -12,7 +12,7 @@ module SyntaxTree module ERB - MAX_WIDTH=80 + MAX_WIDTH = 80 def self.format(source, maxwidth = MAX_WIDTH) PrettierPrint.format(+"", maxwidth) { |q| parse(source).format(q) } end diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index c4bc4ff..b7f78e9 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -32,9 +32,10 @@ def visit_html(node) if node.content q.indent do q.breakable("") - q.seplist(node.content, -> { q.breakable(force: true) }) do |child_node| - visit(child_node) - end + q.seplist( + node.content, + -> { q.breakable(force: true) } + ) { |child_node| visit(child_node) } end end @@ -60,9 +61,10 @@ def visit_erb_if(node, key: "if") if node.elements.any? q.indent do q.breakable - q.seplist(node.elements, -> { q.breakable(force: true) }) do |child_node| - visit(child_node) - end + q.seplist( + node.elements, + -> { q.breakable(force: true) } + ) { |child_node| visit(child_node) } end end From 45d7f45980057db9e7912a195cb8246c92fcb97e Mon Sep 17 00:00:00 2001 From: David Wessman Date: Thu, 18 May 2023 21:33:22 +0200 Subject: [PATCH 03/73] First implementation of blocks - Added some more specific test cases to easily track regressions --- lib/syntax_tree/erb.rb | 2 +- lib/syntax_tree/erb/format.rb | 46 ++++++++-- lib/syntax_tree/erb/nodes.rb | 48 ++++++++--- lib/syntax_tree/erb/parser.rb | 83 +++++++++++++++---- lib/syntax_tree/erb/pretty_print.rb | 18 +++- test/erb_test.rb | 30 +++++-- test/fixture/block_formatted.html.erb | 10 +++ test/fixture/block_unformatted.html.erb | 19 +++++ test/fixture/example1_formatted.html.erb | 20 ----- test/fixture/example1_unformatted.html.erb | 17 ---- test/fixture/if_statements_formatted.html.erb | 11 +++ .../if_statements_unformatted.html.erb | 3 + test/fixture/nested_html_formatted.html.erb | 5 ++ test/fixture/nested_html_unformatted.html.erb | 1 + 14 files changed, 234 insertions(+), 79 deletions(-) create mode 100644 test/fixture/block_formatted.html.erb create mode 100644 test/fixture/block_unformatted.html.erb delete mode 100644 test/fixture/example1_formatted.html.erb delete mode 100644 test/fixture/example1_unformatted.html.erb create mode 100644 test/fixture/if_statements_formatted.html.erb create mode 100644 test/fixture/if_statements_unformatted.html.erb create mode 100644 test/fixture/nested_html_formatted.html.erb create mode 100644 test/fixture/nested_html_unformatted.html.erb diff --git a/lib/syntax_tree/erb.rb b/lib/syntax_tree/erb.rb index ae726fa..6fe8098 100644 --- a/lib/syntax_tree/erb.rb +++ b/lib/syntax_tree/erb.rb @@ -26,5 +26,5 @@ def self.read(filepath) end end - register_handler(".erb", ERB) + register_handler(".html.erb", ERB) end diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index b7f78e9..49c00c8 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -47,20 +47,48 @@ def visit_html(node) # Visit an ErbNode node. def visit_erb(node) visit(node.opening_tag) + + q.text(" ") visit(node.content) + q.text(" ") + visit(node.closing_tag) end + def visit_erb_block(node) + visit(node.erb_node) + + if node.elements.any? + q.group do + q.indent do + q.breakable(force: true) + q.seplist( + node.elements, + -> { q.breakable(force: true) } + ) { |child_node| visit(child_node) } + end + end + end + + q.breakable("") + visit(node.consequent) + end + + def visit_erb_do_close(node) + q.text(node.value.rstrip) + q.text(" %>") + end + # Visit an ErbIf node. def visit_erb_if(node, key: "if") q.group do - q.text("<% #{key}") + q.text("<% #{key} ") visit(node.erb_node.content) - q.text("%>") + q.text(" %>") if node.elements.any? q.indent do - q.breakable + q.breakable(force: true) q.seplist( node.elements, -> { q.breakable(force: true) } @@ -106,8 +134,16 @@ def visit_erb_content(node) SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) formatter.format(node.value.statements) formatter.flush - formatted = [" ", *formatter.output, " "].join - q.text(formatted) + + rows = formatter.output.join.split("\n") + + if rows.size > 1 + q.group do + q.seplist(rows, -> { q.breakable(" ") }) { |row| q.text(row) } + end + else + q.text(rows.first) + end end end diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 3c18417..14a4e67 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -230,35 +230,58 @@ def deconstruct_keys(keys) end class ErbBlock < Node - attr_reader :opening_tag, :content, :closing_tag, :location + attr_reader :erb_node, :elements, :consequent, :location - def initialize(opening_tag:, content:, closing_tag:, location:) - @opening_tag = opening_tag - @content = content - @closing_tag = closing_tag - @location = location + def initialize(erb_node:, elements:, consequent:) + @erb_node = erb_node + @elements = elements + @consequent = consequent + @location = erb_node.location.to(consequent.location) end def accept(visitor) - visitor.visit_erb(self) + visitor.visit_erb_block(self) end def child_nodes - [opening_tag, content, closing_tag].compact + [*elements].compact end alias deconstruct child_nodes def deconstruct_keys(keys) { - opening_tag: opening_tag, - content: content, - closing_tag: closing_tag, + erb_node: erb_node, + elements: elements, + consequent: consequent, location: location } end end + class ErbDoClose < Node + attr_reader :location, :value + + def initialize(location:, value:) + @location = location + @value = value + end + + def accept(visitor) + visitor.visit_erb_do_close(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { location: location, value: value } + end + end + class ErbControl < Node attr_reader :erb_node @@ -348,7 +371,8 @@ def initialize(value:) begin @value = SyntaxTree.parse(@value) @parsed = true - rescue SyntaxTree::Parser::ParseError + rescue SyntaxTree::Parser::ParseError => error + puts error.message @parsed = false end end diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 4e4e8f6..2e18c6e 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -62,7 +62,7 @@ def debug_tokens def parse_any_tag atleast do - maybe { parse_erb_if } || maybe { parse_erb } || + maybe { parse_erb_if } || maybe { parse_erb_tag } || maybe { parse_html_element } || maybe { parse_chardata } end end @@ -146,6 +146,9 @@ def make_tokens when /\A[\n]+/ # whitespace line += $&.count("\n") + when /\Ado\s*.*?\s*%>/ + enum.yield :erb_do_close, $&, index, line + state.pop when /\A%>/ enum.yield :erb_close, $&, index, line state.pop @@ -310,7 +313,24 @@ def parse_until_erb_end items end - def parse_any_tag_until_erb_keyword + def parse_any_tag_after_erb_elsif + items = [] + + loop do + result = + maybe { parse_erb_elsif } || maybe { parse_erb_else } || + maybe { parse_erb_end } || maybe { parse_any_tag } + items << result + if result.is_a?(ErbElsif) || result.is_a?(ErbElse) || + result.is_a?(ErbEnd) + break + end + end + + items + end + + def parse_any_tag_after_erb_if items = [] loop do @@ -318,7 +338,10 @@ def parse_any_tag_until_erb_keyword maybe { parse_erb_elsif } || maybe { parse_erb_else } || maybe { parse_erb_end } || maybe { parse_any_tag } items << result - break if result.is_a?(ErbControl) || result.is_a?(ErbEnd) + if result.is_a?(ErbElsif) || result.is_a?(ErbElse) || + result.is_a?(ErbEnd) + break + end end items @@ -328,7 +351,7 @@ def parse_content many do atleast do maybe { parse_html_element } || maybe { parse_chardata } || - maybe { parse_erb } || maybe { consume(:comment) } + maybe { parse_erb_tag } || maybe { consume(:comment) } end end end @@ -388,16 +411,12 @@ def parse_html_element end end - def parse_erb - parse_erb_tag - end - def parse_erb_if opening_tag = consume(:erb_if_open) statement = many { consume(:erb_code) } closing_tag = consume(:erb_close) - contents = parse_any_tag_until_erb_keyword + contents = parse_any_tag_after_erb_if erb_tag = contents.pop @@ -423,7 +442,7 @@ def parse_erb_elsif statement = many { consume(:erb_code) } closing_tag = consume(:erb_close) - contents = parse_any_tag_until_erb_keyword + contents = parse_any_tag_after_erb_elsif erb_tag = contents.pop @@ -471,13 +490,43 @@ def parse_ruby_or_string(content) def parse_erb_tag opening_tag = consume(:erb_open) content = many { consume(:erb_code) } - closing_tag = consume(:erb_close) + closing_tag = + atleast do + maybe { consume(:erb_close) } || maybe { parse_erb_do_close } + end + + erb_node = + ErbNode.new( + opening_tag: opening_tag, + content: content.map(&:value).join, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ) + + if closing_tag.is_a?(ErbDoClose) + elements = parse_until_erb_end + erb_end = elements.pop + + unless erb_end.is_a?(ErbEnd) + raise(ErbKeywordError, "Found no matching end-tag for the do-tag") + end + + ErbBlock.new( + erb_node: erb_node, + elements: elements, + consequent: erb_end + ) + else + erb_node + end + end + + def parse_erb_do_close + token = consume(:erb_do_close) - ErbNode.new( - opening_tag: opening_tag, - content: content.map(&:value).join, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) + ErbDoClose.new( + location: token.location, + value: token.value.gsub(/%>/, "") ) end @@ -487,7 +536,7 @@ def parse_string many do atleast do maybe { consume(:text) } || maybe { consume(:whitespace) } || - maybe { parse_erb } + maybe { parse_erb_tag } end end closing = consume(:string_close) diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index 42256da..8897ce4 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -51,11 +51,25 @@ def visit_erb(node) end end + def visit_erb_block(node) + q.group do + q.text("(erb_block") + q.nest(2) do + q.breakable + q.seplist(node.child_nodes) { |child_node| visit(child_node) } + end + q.breakable + visit(node.consequent) + q.breakable("") + q.text(")") + end + end + def visit_erb_if(node, key = "erb_if") q.group do q.text("(#{key}") q.nest(2) do - q.breakable + q.breakable() q.seplist(node.child_nodes) { |child_node| visit(child_node) } end q.breakable @@ -74,7 +88,7 @@ def visit_erb_else(node) end def visit_erb_end(node) - q.text("(erb_end)") + q.text("erb_end") end # Visit an ErbContent node. diff --git a/test/erb_test.rb b/test/erb_test.rb index bb20892..504f72b 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -4,15 +4,35 @@ module SyntaxTree class ERBTest < Minitest::Test - def test_example1 + def test_block + assert_parsing("block") + end + + def test_nested_html + assert_parsing("nested_html") + end + + def test_if_statements + assert_parsing("if_statements") + end + + private + + def assert_parsing(name) directory = File.expand_path("fixture", __dir__) - unformatted_file = File.join(directory, "example1_unformatted.html.erb") - formatted_file = File.join(directory, "example1_formatted.html.erb") + unformatted_file = File.join(directory, "#{name}_unformatted.html.erb") + formatted_file = File.join(directory, "#{name}_formatted.html.erb") expected = ERB.read(formatted_file) - actual = ERB.format(ERB.read(unformatted_file)) + formatted = ERB.format(ERB.read(unformatted_file)) + + if (expected != formatted) + puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") + Dir.mkdir("./tmp") unless Dir.exist?("./tmp") + File.write("./tmp/#{name}_failed.html.erb", formatted) + end - assert_equal(actual, expected) + assert_equal(formatted, expected) end end end diff --git a/test/fixture/block_formatted.html.erb b/test/fixture/block_formatted.html.erb new file mode 100644 index 0000000..10a6ee6 --- /dev/null +++ b/test/fixture/block_formatted.html.erb @@ -0,0 +1,10 @@ +<%= form_with(url: format_path) do |form| %> +
+ <%= form.label(:name, "Name") %> + <%= form.text_field(:name, class: "form-control") %> +
+ <%= form.submit( + "Very very very very very long text", + class: "btn btn-primary btn btn-primary btn btn-primary btn btn-primary" + ) %> +<% end %> diff --git a/test/fixture/block_unformatted.html.erb b/test/fixture/block_unformatted.html.erb new file mode 100644 index 0000000..739a2f2 --- /dev/null +++ b/test/fixture/block_unformatted.html.erb @@ -0,0 +1,19 @@ +<%= form_with(url: format_path) do |form| %> +
+ <%=form.label(:name, "Name")%> + <%=form.text_field(:name, class: "form-control")%> +
+ + + + + + + + + + + + + <%= form.submit("Very very very very very long text", class: "btn btn-primary btn btn-primary btn btn-primary btn btn-primary") %> +<% end %> diff --git a/test/fixture/example1_formatted.html.erb b/test/fixture/example1_formatted.html.erb deleted file mode 100644 index 550c350..0000000 --- a/test/fixture/example1_formatted.html.erb +++ /dev/null @@ -1,20 +0,0 @@ -<%= "this" %> -<% if david %> - Hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej - what? -<% elsif that %> -

This

-
-

- - This looks amazing - -

-
-<% else %> - Cool! -<% end %> diff --git a/test/fixture/example1_unformatted.html.erb b/test/fixture/example1_unformatted.html.erb deleted file mode 100644 index 7164893..0000000 --- a/test/fixture/example1_unformatted.html.erb +++ /dev/null @@ -1,17 +0,0 @@ -<%="this"%> - -<%if david%> - Hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej hej - what? -<% elsif that%> -

This

-
-

- - This looks amazing - -

-
-<%else%> - Cool! -<%end%> diff --git a/test/fixture/if_statements_formatted.html.erb b/test/fixture/if_statements_formatted.html.erb new file mode 100644 index 0000000..1362486 --- /dev/null +++ b/test/fixture/if_statements_formatted.html.erb @@ -0,0 +1,11 @@ +<% if this %> +

that

+<% elsif that %> +

this

+ <% if nested_this %> +

this

+ <% end %> +<% else %> +

else

+<% end %> +<%= what if this %> diff --git a/test/fixture/if_statements_unformatted.html.erb b/test/fixture/if_statements_unformatted.html.erb new file mode 100644 index 0000000..4ccb5f2 --- /dev/null +++ b/test/fixture/if_statements_unformatted.html.erb @@ -0,0 +1,3 @@ +<%if this%>

that

<%elsif that %>

this

+<% if nested_this %>

this

<%end%><%else%>

else

<%end %> +<%= what if this %> diff --git a/test/fixture/nested_html_formatted.html.erb b/test/fixture/nested_html_formatted.html.erb new file mode 100644 index 0000000..724038c --- /dev/null +++ b/test/fixture/nested_html_formatted.html.erb @@ -0,0 +1,5 @@ +
+ + <%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %> + +
diff --git a/test/fixture/nested_html_unformatted.html.erb b/test/fixture/nested_html_unformatted.html.erb new file mode 100644 index 0000000..55c0e03 --- /dev/null +++ b/test/fixture/nested_html_unformatted.html.erb @@ -0,0 +1 @@ +
<%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %>
From a51e673742222845430dde911058ab9cc2491735 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 19 May 2023 09:29:38 +0200 Subject: [PATCH 04/73] Handles basic vue-components (#2) --- lib/syntax_tree/erb/format.rb | 2 +- lib/syntax_tree/erb/parser.rb | 26 +++++++++++++------ test/erb_test.rb | 4 +++ .../fixture/vue_components_formatted.html.erb | 10 +++++++ .../vue_components_unformatted.html.erb | 3 +++ 5 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 test/fixture/vue_components_formatted.html.erb create mode 100644 test/fixture/vue_components_unformatted.html.erb diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 49c00c8..eadca01 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -29,7 +29,7 @@ def visit_html(node) q.group do visit(node.opening_tag) - if node.content + if node.content.any? q.indent do q.breakable("") q.seplist( diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 2e18c6e..cfa49c9 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -551,15 +551,25 @@ def parse_string def parse_html_attribute key = consume(:name) - equals = consume(:equals) - value = parse_string + equals = maybe { consume(:equals) } + + if equals.nil? + HtmlAttribute.new( + key: key, + equals: nil, + value: nil, + location: key.location + ) + else + value = parse_string - HtmlAttribute.new( - key: key, - equals: equals, - value: value, - location: key.location.to(value.location) - ) + HtmlAttribute.new( + key: key, + equals: equals, + value: value, + location: key.location.to(value.location) + ) + end end def parse_chardata diff --git a/test/erb_test.rb b/test/erb_test.rb index 504f72b..9fbea02 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -16,6 +16,10 @@ def test_if_statements assert_parsing("if_statements") end + def test_vue_components + assert_parsing("vue_components") + end + private def assert_parsing(name) diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/vue_components_formatted.html.erb new file mode 100644 index 0000000..1d46280 --- /dev/null +++ b/test/fixture/vue_components_formatted.html.erb @@ -0,0 +1,10 @@ +
+ " + boolean + :value="['a', 'b']" + :long-variable-name="data.item.javascript.code" + > + + +
diff --git a/test/fixture/vue_components_unformatted.html.erb b/test/fixture/vue_components_unformatted.html.erb new file mode 100644 index 0000000..478a3f6 --- /dev/null +++ b/test/fixture/vue_components_unformatted.html.erb @@ -0,0 +1,3 @@ +
" boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code"> + +
From 4deaeb12e317bf900ab41387f012f889c4cf34f9 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 19 May 2023 09:35:28 +0200 Subject: [PATCH 05/73] Changes register_handler --- lib/syntax_tree/erb.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/erb.rb b/lib/syntax_tree/erb.rb index 6fe8098..ae726fa 100644 --- a/lib/syntax_tree/erb.rb +++ b/lib/syntax_tree/erb.rb @@ -26,5 +26,5 @@ def self.read(filepath) end end - register_handler(".html.erb", ERB) + register_handler(".erb", ERB) end From 7320b93bbb5eead2d9955614afa9edf60f3f30cc Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 19 May 2023 18:52:50 +0200 Subject: [PATCH 06/73] Handles empty and invalid files (#3) --- lib/syntax_tree/erb.rb | 2 +- lib/syntax_tree/erb/format.rb | 2 +- lib/syntax_tree/erb/parser.rb | 55 ++++++++++++++++++++--------------- test/erb_test.rb | 13 +++++++++ 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/lib/syntax_tree/erb.rb b/lib/syntax_tree/erb.rb index ae726fa..8bba3ad 100644 --- a/lib/syntax_tree/erb.rb +++ b/lib/syntax_tree/erb.rb @@ -13,7 +13,7 @@ module SyntaxTree module ERB MAX_WIDTH = 80 - def self.format(source, maxwidth = MAX_WIDTH) + def self.format(source, maxwidth = MAX_WIDTH, options: nil) PrettierPrint.format(+"", maxwidth) { |q| parse(source).format(q) } end diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index eadca01..2b0e73a 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -29,7 +29,7 @@ def visit_html(node) q.group do visit(node.opening_tag) - if node.content.any? + if node.content&.any? q.indent do q.breakable("") q.seplist( diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index cfa49c9..72b1f91 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -46,10 +46,10 @@ def initialize(source) def parse elements = many { parse_any_tag } - Document.new( - elements: elements, - location: elements.first.location.to(elements.last.location) - ) + location = + elements.first.location.to(elements.last.location) if elements.any? + + Document.new(elements: elements, location: location) end def debug_tokens @@ -103,11 +103,17 @@ def make_tokens state << :erb line += $&.count("\n") when /\A<%\s*if/ - # ERB elsif statements - # <% elsif + # ERB if statements + # <% if enum.yield :erb_if_open, $&, index, line state << :erb line += $&.count("\n") + when /\A<%\s*unless/ + # ERB unless statements + # <% unless + enum.yield :erb_unless_open, $&, index, line + state << :erb + line += $&.count("\n") when /\A<%\s*end\s*%>/ # ERB end statements # <% end %> @@ -302,7 +308,8 @@ def parse_until_erb_end loop do begin - result = maybe { parse_erb_end } || maybe { parse_any_tag } + result = + atleast { maybe { parse_erb_end } || maybe { parse_any_tag } } items << result break if result.is_a?(ErbEnd) rescue ErbKeywordError @@ -318,8 +325,10 @@ def parse_any_tag_after_erb_elsif loop do result = - maybe { parse_erb_elsif } || maybe { parse_erb_else } || - maybe { parse_erb_end } || maybe { parse_any_tag } + atleast do + maybe { parse_erb_elsif } || maybe { parse_erb_else } || + maybe { parse_erb_end } || maybe { parse_any_tag } + end items << result if result.is_a?(ErbElsif) || result.is_a?(ErbElse) || result.is_a?(ErbEnd) @@ -335,9 +344,13 @@ def parse_any_tag_after_erb_if loop do result = - maybe { parse_erb_elsif } || maybe { parse_erb_else } || - maybe { parse_erb_end } || maybe { parse_any_tag } + atleast do + maybe { parse_erb_elsif } || maybe { parse_erb_else } || + maybe { parse_erb_end } || maybe { parse_any_tag } + end + items << result + if result.is_a?(ErbElsif) || result.is_a?(ErbElse) || result.is_a?(ErbEnd) break @@ -347,15 +360,6 @@ def parse_any_tag_after_erb_if items end - def parse_content - many do - atleast do - maybe { parse_html_element } || maybe { parse_chardata } || - maybe { parse_erb_tag } || maybe { consume(:comment) } - end - end - end - def parse_html_opening_tag opening = consume(:open) name = consume(:name) @@ -392,7 +396,7 @@ def parse_html_element opening_tag = parse_html_opening_tag if opening_tag.closing.value == ">" - content = parse_content + content = many { parse_any_tag } closing_tag = parse_html_closing_tag HtmlNode.new( @@ -412,11 +416,16 @@ def parse_html_element end def parse_erb_if - opening_tag = consume(:erb_if_open) + opening_tag = + atleast do + maybe { consume(:erb_if_open) } || + maybe { consume(:erb_unless_open) } + end + statement = many { consume(:erb_code) } closing_tag = consume(:erb_close) - contents = parse_any_tag_after_erb_if + contents = maybe { parse_any_tag_after_erb_if } || [] erb_tag = contents.pop diff --git a/test/erb_test.rb b/test/erb_test.rb index 9fbea02..a7c4981 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -20,6 +20,19 @@ def test_vue_components assert_parsing("vue_components") end + def test_empty_file + parsed = ERB.parse("") + assert_instance_of(SyntaxTree::ERB::Document, parsed) + assert_empty(parsed.elements) + assert_nil(parsed.location) + end + + def test_invalid_file + assert_raises(SyntaxTree::ERB::Parser::ErbKeywordError) do + ERB.parse("<% if no_end_tag %>") + end + end + private def assert_parsing(name) From c87ffc467ff1378eb05461df14d4c779f1fd0dd6 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 19 May 2023 18:54:46 +0200 Subject: [PATCH 07/73] Handles attribute names like vue events `@click` (#4) --- lib/syntax_tree/erb/parser.rb | 2 +- test/fixture/vue_components_formatted.html.erb | 2 +- test/fixture/vue_components_unformatted.html.erb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 72b1f91..f116c02 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -4,7 +4,7 @@ module SyntaxTree module ERB class Parser NAME_START = - "[:a-zA-Z_\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFFD}]" + "[@:a-zA-Z_\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFFD}]" NAME_CHAR = "[#{NAME_START}-\\.\\d\u{00B7}\u{0300}-\u{036F}\u{203F}-\u{2040}]" diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/vue_components_formatted.html.erb index 1d46280..97ffbbf 100644 --- a/test/fixture/vue_components_formatted.html.erb +++ b/test/fixture/vue_components_formatted.html.erb @@ -6,5 +6,5 @@ :long-variable-name="data.item.javascript.code" > - + diff --git a/test/fixture/vue_components_unformatted.html.erb b/test/fixture/vue_components_unformatted.html.erb index 478a3f6..9764266 100644 --- a/test/fixture/vue_components_unformatted.html.erb +++ b/test/fixture/vue_components_unformatted.html.erb @@ -1,3 +1,3 @@
" boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code"> -
+ From e2f8307a2f9aeb95f7f1be85e2eb8394e5ceb476 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sat, 20 May 2023 10:42:31 +0200 Subject: [PATCH 08/73] Adds ErbUnless to handle `unless` syntax (#5) --- lib/syntax_tree/erb/format.rb | 5 +++ lib/syntax_tree/erb/nodes.rb | 6 ++++ lib/syntax_tree/erb/parser.rb | 36 +++++++++++++------ lib/syntax_tree/erb/pretty_print.rb | 5 +++ test/fixture/if_statements_formatted.html.erb | 9 +++++ .../if_statements_unformatted.html.erb | 3 ++ 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 2b0e73a..e87f961 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -101,6 +101,11 @@ def visit_erb_if(node, key: "if") end end + # Visit an ErbUnless node. + def visit_erb_unless(node) + visit_erb_if(node, key: "unless") + end + # Visit an ErbElsIf node. def visit_erb_elsif(node) visit_erb_if(node, key: "elsif") diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 14a4e67..1d920a6 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -329,6 +329,12 @@ def deconstruct_keys(keys) end end + class ErbUnless < ErbIf + def accept(visitor) + visitor.visit_erb_unless(self) + end + end + class ErbElsif < ErbIf def accept(visitor) visitor.visit_erb_elsif(self) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index f116c02..8068b2a 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -433,17 +433,31 @@ def parse_erb_if raise(ErbKeywordError, "Found no matching tag to the if-tag") end - ErbIf.new( - erb_node: - ErbNode.new( - opening_tag: opening_tag, - content: statement.map(&:value).join, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) - ), - elements: contents, - consequent: erb_tag - ) + if opening_tag.type == :erb_if_open + ErbIf.new( + erb_node: + ErbNode.new( + opening_tag: opening_tag, + content: statement.map(&:value).join, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ), + elements: contents, + consequent: erb_tag + ) + else + ErbUnless.new( + erb_node: + ErbNode.new( + opening_tag: opening_tag, + content: statement.map(&:value).join, + closing_tag: closing_tag, + location: opening_tag.location.to(closing_tag.location) + ), + elements: contents, + consequent: erb_tag + ) + end end def parse_erb_elsif diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index 8897ce4..fb7d7e5 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -79,6 +79,11 @@ def visit_erb_if(node, key = "erb_if") end end + # Visit an ErbUnless node. + def visit_erb_unless(node) + visit_erb_if(node, key: "unless") + end + def visit_erb_elsif(node) visit_erb_if(node, "erb_elsif") end diff --git a/test/fixture/if_statements_formatted.html.erb b/test/fixture/if_statements_formatted.html.erb index 1362486..c918cd1 100644 --- a/test/fixture/if_statements_formatted.html.erb +++ b/test/fixture/if_statements_formatted.html.erb @@ -9,3 +9,12 @@

else

<% end %> <%= what if this %> +

+ <% unless what %> + Ja + <% elsif allowed? %> + Nej + <% else %> + Kanske + <% end %> +

diff --git a/test/fixture/if_statements_unformatted.html.erb b/test/fixture/if_statements_unformatted.html.erb index 4ccb5f2..41610a7 100644 --- a/test/fixture/if_statements_unformatted.html.erb +++ b/test/fixture/if_statements_unformatted.html.erb @@ -1,3 +1,6 @@ <%if this%>

that

<%elsif that %>

this

<% if nested_this %>

this

<%end%><%else%>

else

<%end %> <%= what if this %> +

<% unless what %> Ja<% elsif allowed? %> + Nej<% else %>Kanske + <% end %>

From c26791e5551b8b0166851041026d1d46304459fe Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 00:25:14 +0200 Subject: [PATCH 09/73] Handles more formats for ERB syntax (#6) --- lib/syntax_tree/erb/format.rb | 77 +++--- lib/syntax_tree/erb/nodes.rb | 44 +-- lib/syntax_tree/erb/parser.rb | 270 ++++++++----------- lib/syntax_tree/erb/pretty_print.rb | 31 ++- test/erb_test.rb | 4 + test/fixture/erb_syntax_formatted.html.erb | 21 ++ test/fixture/erb_syntax_unformatted.html.erb | 24 ++ 7 files changed, 220 insertions(+), 251 deletions(-) create mode 100644 test/fixture/erb_syntax_formatted.html.erb create mode 100644 test/fixture/erb_syntax_unformatted.html.erb diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index e87f961..30e3f57 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -48,9 +48,12 @@ def visit_html(node) def visit_erb(node) visit(node.opening_tag) - q.text(" ") + if node.keyword + q.text(" ") + visit(node.keyword) + end + visit(node.content) - q.text(" ") visit(node.closing_tag) end @@ -76,15 +79,14 @@ def visit_erb_block(node) def visit_erb_do_close(node) q.text(node.value.rstrip) - q.text(" %>") + q.text(" ") + q.text(node.closing) end # Visit an ErbIf node. - def visit_erb_if(node, key: "if") + def visit_erb_if(node) q.group do - q.text("<% #{key} ") - visit(node.erb_node.content) - q.text(" %>") + visit(node.erb_node) if node.elements.any? q.indent do @@ -101,54 +103,35 @@ def visit_erb_if(node, key: "if") end end - # Visit an ErbUnless node. - def visit_erb_unless(node) - visit_erb_if(node, key: "unless") - end - - # Visit an ErbElsIf node. - def visit_erb_elsif(node) - visit_erb_if(node, key: "elsif") - end - - # Visit an ErbElse node. - def visit_erb_else(node) - q.group do - q.text("<% else %>") - - q.indent do - q.breakable - visit_all(node.elements) - end - - q.breakable_force - visit(node.consequent) - end - end - # Visit an ErbEnd node. def visit_erb_end(node) - q.text("<% end %>") + visit(node.opening_tag) + q.text(" ") + visit(node.keyword) + q.text(" ") + visit(node.closing_tag) end def visit_erb_content(node) - if (node.value.is_a?(String)) - q.text(node.value) - else - formatter = - SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) - formatter.format(node.value.statements) - formatter.flush + formatter = + SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) + formatter.format(node.value.statements) + formatter.flush - rows = formatter.output.join.split("\n") + rows = formatter.output.join.split("\n") - if rows.size > 1 - q.group do - q.seplist(rows, -> { q.breakable(" ") }) { |row| q.text(row) } - end - else - q.text(rows.first) + if rows.size > 1 + q.group do + q.text(" ") + q.seplist(rows, -> { q.breakable(" ") }) { |row| q.text(row) } + q.text(" ") end + elsif rows.size == 1 + q.text(" ") + q.text(rows.first) + q.text(" ") + else + q.text(" ") end end diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 1d920a6..5e134cc 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -199,12 +199,12 @@ def deconstruct_keys(keys) end class ErbNode < Node - attr_reader :opening_tag, :content, :closing_tag, :location + attr_reader :opening_tag, :keyword, :content, :closing_tag, :location - def initialize(opening_tag:, content:, closing_tag:, location:) + def initialize(opening_tag:, keyword:, content:, closing_tag:, location:) @opening_tag = opening_tag - @content = ErbContent.new(value: content) - + @keyword = keyword + @content = ErbContent.new(value: content) if content @closing_tag = closing_tag @location = location end @@ -214,7 +214,7 @@ def accept(visitor) end def child_nodes - [opening_tag, content, closing_tag].compact + [opening_tag, keyword, content, closing_tag].compact end alias deconstruct child_nodes @@ -222,6 +222,7 @@ def child_nodes def deconstruct_keys(keys) { opening_tag: opening_tag, + keyword: keyword, content: content, closing_tag: closing_tag, location: location @@ -260,11 +261,12 @@ def deconstruct_keys(keys) end class ErbDoClose < Node - attr_reader :location, :value + attr_reader :location, :value, :closing - def initialize(location:, value:) + def initialize(location:, value:, closing:) @location = location @value = value + @closing = closing end def accept(visitor) @@ -278,7 +280,7 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(keys) - { location: location, value: value } + { location: location, value: value, closing: closing } end end @@ -330,30 +332,15 @@ def deconstruct_keys(keys) end class ErbUnless < ErbIf - def accept(visitor) - visitor.visit_erb_unless(self) - end end class ErbElsif < ErbIf - def accept(visitor) - visitor.visit_erb_elsif(self) - end end class ErbElse < ErbIf - def accept(visitor) - visitor.visit_erb_else(self) - end end - class ErbEnd < Node - attr_reader :location - - def initialize(location:) - @location = location - end - + class ErbEnd < ErbNode def accept(visitor) visitor.visit_erb_end(self) end @@ -363,10 +350,6 @@ def child_nodes end alias deconstruct child_nodes - - def deconstruct_keys(keys) - { location: location } - end end class ErbContent < Node @@ -377,8 +360,9 @@ def initialize(value:) begin @value = SyntaxTree.parse(@value) @parsed = true - rescue SyntaxTree::Parser::ParseError => error - puts error.message + rescue SyntaxTree::Parser::ParseError + # Removes leading and trailing whitespace + @value = @value&.lstrip&.rstrip @parsed = false end end diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 8068b2a..5b23f12 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -62,7 +62,7 @@ def debug_tokens def parse_any_tag atleast do - maybe { parse_erb_if } || maybe { parse_erb_tag } || + maybe { parse_erb_tag } || maybe { consume(:erb_comment) } || maybe { parse_html_element } || maybe { parse_chardata } end end @@ -91,39 +91,16 @@ def make_tokens # / - # ERB else statements - # <% else %> - enum.yield :erb_else, $&, index, line - line += $&.count("\n") - when /\A<%\s*elsif/ - # ERB elsif statements - # <% elsif - enum.yield :erb_elsif_open, $&, index, line - state << :erb - line += $&.count("\n") - when /\A<%\s*if/ - # ERB if statements - # <% if - enum.yield :erb_if_open, $&, index, line - state << :erb - line += $&.count("\n") - when /\A<%\s*unless/ - # ERB unless statements - # <% unless - enum.yield :erb_unless_open, $&, index, line - state << :erb - line += $&.count("\n") - when /\A<%\s*end\s*%>/ - # ERB end statements - # <% end %> - enum.yield :erb_end, $&, index, line - line += $&.count("\n") - when /\A<%[=]?/ + when /\A<%#.*%>/ + # An ERB-comment + # <%# this is an ERB comment %> + enum.yield :erb_comment, $&, index, line + when /\A<%={1,2}/, /\A<%-/, /\A<%/ # the beginning of an ERB tag # <% + # <%=, <%== enum.yield :erb_open, $&, index, line - state << :erb + state << :erb_start line += $&.count("\n") when %r{\A/ enum.yield :erb_do_close, $&, index, line state.pop - when /\A%>/ + when /\A-?%>/ enum.yield :erb_close, $&, index, line state.pop else @@ -191,9 +208,9 @@ def make_tokens when /\A[ \t\r\n]+/ # whitespace line += $&.count("\n") - when /\A%>/ + when /\A-?%>/ # the end of an ERB tag - # %> + # -%> or %> enum.yield :erb_close, $&, index, line state.pop when /\A>/ @@ -303,15 +320,14 @@ def many items end - def parse_until_erb_end + def parse_until_erb(classes:) items = [] loop do begin - result = - atleast { maybe { parse_erb_end } || maybe { parse_any_tag } } + result = parse_any_tag items << result - break if result.is_a?(ErbEnd) + break if classes.any? { |cls| result.is_a?(cls) } rescue ErbKeywordError break end @@ -320,46 +336,6 @@ def parse_until_erb_end items end - def parse_any_tag_after_erb_elsif - items = [] - - loop do - result = - atleast do - maybe { parse_erb_elsif } || maybe { parse_erb_else } || - maybe { parse_erb_end } || maybe { parse_any_tag } - end - items << result - if result.is_a?(ErbElsif) || result.is_a?(ErbElse) || - result.is_a?(ErbEnd) - break - end - end - - items - end - - def parse_any_tag_after_erb_if - items = [] - - loop do - result = - atleast do - maybe { parse_erb_elsif } || maybe { parse_erb_else } || - maybe { parse_erb_end } || maybe { parse_any_tag } - end - - items << result - - if result.is_a?(ErbElsif) || result.is_a?(ErbElse) || - result.is_a?(ErbEnd) - break - end - end - - items - end - def parse_html_opening_tag opening = consume(:open) name = consume(:name) @@ -415,93 +391,54 @@ def parse_html_element end end - def parse_erb_if - opening_tag = - atleast do - maybe { consume(:erb_if_open) } || - maybe { consume(:erb_unless_open) } - end - - statement = many { consume(:erb_code) } - closing_tag = consume(:erb_close) - - contents = maybe { parse_any_tag_after_erb_if } || [] + def parse_erb_if(erb_node) + elements = + maybe { parse_until_erb(classes: [ErbElsif, ErbElse, ErbEnd]) } || [] - erb_tag = contents.pop + erb_tag = elements.pop unless erb_tag.is_a?(ErbControl) || erb_tag.is_a?(ErbEnd) raise(ErbKeywordError, "Found no matching tag to the if-tag") end - if opening_tag.type == :erb_if_open - ErbIf.new( - erb_node: - ErbNode.new( - opening_tag: opening_tag, - content: statement.map(&:value).join, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) - ), - elements: contents, + case erb_node.keyword.type + when :erb_if + ErbIf.new(erb_node: erb_node, elements: elements, consequent: erb_tag) + when :erb_unless + ErbUnless.new( + erb_node: erb_node, + elements: elements, consequent: erb_tag ) - else - ErbUnless.new( - erb_node: - ErbNode.new( - opening_tag: opening_tag, - content: statement.map(&:value).join, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) - ), - elements: contents, + when :erb_elsif + ErbElsif.new( + erb_node: erb_node, + elements: elements, consequent: erb_tag ) end end - def parse_erb_elsif - opening_tag = consume(:erb_elsif_open) - statement = many { consume(:erb_code) } - closing_tag = consume(:erb_close) - - contents = parse_any_tag_after_erb_elsif - - erb_tag = contents.pop + def parse_erb_else(erb_node) + elements = maybe { parse_until_erb(classes: [ErbEnd]) } || [] - unless erb_tag.is_a?(ErbControl) - raise(ErbKeywordError, "Found no matching end-tag to the elsif-tag") - end - - ErbElsif.new( - erb_node: - ErbNode.new( - opening_tag: opening_tag, - content: statement.map(&:value).join, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) - ), - elements: contents, - consequent: erb_tag - ) - end - - def parse_erb_else - tag = consume(:erb_else) - child_nodes = parse_until_erb_end - - erb_end = child_nodes.pop + erb_end = elements.pop unless erb_end.is_a?(ErbEnd) raise(ErbKeywordError, "Found no matching end-tag for the else-tag") end - ErbElse.new(erb_node: tag, elements: child_nodes, consequent: erb_end) + ErbElse.new(erb_node: erb_node, elements: elements, consequent: erb_end) end - def parse_erb_end - tag = consume(:erb_end) - ErbEnd.new(location: tag.location) + def parse_erb_end(erb_node) + ErbEnd.new( + opening_tag: erb_node.opening_tag, + keyword: erb_node.keyword, + content: nil, + closing_tag: erb_node.closing_tag, + location: erb_node.location + ) end def parse_ruby_or_string(content) @@ -512,6 +449,10 @@ def parse_ruby_or_string(content) def parse_erb_tag opening_tag = consume(:erb_open) + keyword = + maybe { consume(:erb_if) } || maybe { consume(:erb_unless) } || + maybe { consume(:erb_elsif) } || maybe { consume(:erb_else) } || + maybe { consume(:erb_end) } content = many { consume(:erb_code) } closing_tag = atleast do @@ -521,35 +462,48 @@ def parse_erb_tag erb_node = ErbNode.new( opening_tag: opening_tag, + keyword: keyword, content: content.map(&:value).join, closing_tag: closing_tag, location: opening_tag.location.to(closing_tag.location) ) - if closing_tag.is_a?(ErbDoClose) - elements = parse_until_erb_end - erb_end = elements.pop + case keyword&.type + when :erb_if, :erb_unless, :erb_elsif + parse_erb_if(erb_node) + when :erb_else + parse_erb_else(erb_node) + when :erb_end + parse_erb_end(erb_node) + else + if closing_tag.is_a?(ErbDoClose) + elements = parse_until_erb(classes: [ErbEnd]) + erb_end = elements.pop - unless erb_end.is_a?(ErbEnd) - raise(ErbKeywordError, "Found no matching end-tag for the do-tag") - end + unless erb_end.is_a?(ErbEnd) + raise(ErbKeywordError, "Found no matching end-tag for the do-tag") + end - ErbBlock.new( - erb_node: erb_node, - elements: elements, - consequent: erb_end - ) - else - erb_node + ErbBlock.new( + erb_node: erb_node, + elements: elements, + consequent: erb_end + ) + else + erb_node + end end end def parse_erb_do_close token = consume(:erb_do_close) + closing = token.value.match(/-?%>$/).to_s + ErbDoClose.new( location: token.location, - value: token.value.gsub(/%>/, "") + value: token.value.gsub(closing, ""), + closing: closing ) end diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index fb7d7e5..c0065b1 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -41,8 +41,15 @@ def visit_erb(node) q.nest(2) do q.breakable visit(node.opening_tag) - q.breakable - q.text("content") + if node.keyword + q.breakable + visit(node.keyword) + end + if node.content + q.breakable + q.text("content") + end + q.breakable visit(node.closing_tag) end @@ -67,7 +74,8 @@ def visit_erb_block(node) def visit_erb_if(node, key = "erb_if") q.group do - q.text("(#{key}") + q.text("(") + visit(node.keyword) if node.keyword q.nest(2) do q.breakable() q.seplist(node.child_nodes) { |child_node| visit(child_node) } @@ -79,19 +87,6 @@ def visit_erb_if(node, key = "erb_if") end end - # Visit an ErbUnless node. - def visit_erb_unless(node) - visit_erb_if(node, key: "unless") - end - - def visit_erb_elsif(node) - visit_erb_if(node, "erb_elsif") - end - - def visit_erb_else(node) - visit_erb_if(node, "erb_else") - end - def visit_erb_end(node) q.text("erb_end") end @@ -121,6 +116,10 @@ def visit_char_data(node) visit_node("char_data", node) end + def visit_erb_do_close(node) + visit_node("erb_do_close", node) + end + private # A generic visit node function for how we pretty print nodes. diff --git a/test/erb_test.rb b/test/erb_test.rb index a7c4981..e8cfddb 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -8,6 +8,10 @@ def test_block assert_parsing("block") end + def test_erb_syntax + assert_parsing("erb_syntax") + end + def test_nested_html assert_parsing("nested_html") end diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb new file mode 100644 index 0000000..8d552ce --- /dev/null +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -0,0 +1,21 @@ +<% this = "avoids line break after expression" -%> +<%# This is an ERB-comment https://stackoverflow.com/a/25626629 this answer describes ERB and erubis syntax%> +<%== rails_raw_output %> +<%- "this only works in ERB not erubis" %> +<% if this -%> + <%= form.submit -%> +<% elsif that -%> + <%= form.submit -%> +<% else -%> + <%= form.submit -%> +<% end -%> +<%- if this %> + <%= form.submit -%> +<%- elsif that %> + <%= form.submit -%> +<%- else %> + <%= form.submit -%> +<%- end %> +<%= link_to(link, text) do -%> +

Cool

+<%- end %> diff --git a/test/fixture/erb_syntax_unformatted.html.erb b/test/fixture/erb_syntax_unformatted.html.erb new file mode 100644 index 0000000..3ff4b26 --- /dev/null +++ b/test/fixture/erb_syntax_unformatted.html.erb @@ -0,0 +1,24 @@ +<% this = "avoids line break after expression"-%> +<%# This is an ERB-comment https://stackoverflow.com/a/25626629 this answer describes ERB and erubis syntax%> +<%== rails_raw_output%> +<%-"this only works in ERB not erubis"%> + +<% if this -%> + <%= form.submit -%> +<% elsif that -%> + <%= form.submit -%> +<% else -%> + <%= form.submit -%> +<% end -%> + +<%- if this %> + <%= form.submit -%> +<%- elsif that %> + <%= form.submit -%> +<%- else %> + <%= form.submit -%> +<%- end %> + +<%= link_to(link, text) do -%> +

Cool

+<%- end %> From 6d31d9826165c3997eb89f6c8990d37b5957ded0 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 00:50:50 +0200 Subject: [PATCH 10/73] Keep blank lines (#7) --- lib/syntax_tree/erb/parser.rb | 13 ++++++++++++- test/fixture/block_formatted.html.erb | 1 + test/fixture/erb_syntax_formatted.html.erb | 3 +++ test/fixture/vue_components_formatted.html.erb | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 5b23f12..066b85b 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -63,7 +63,8 @@ def debug_tokens def parse_any_tag atleast do maybe { parse_erb_tag } || maybe { consume(:erb_comment) } || - maybe { parse_html_element } || maybe { parse_chardata } + maybe { parse_html_element } || maybe { parse_blank_line } || + maybe { parse_chardata } end end @@ -77,6 +78,10 @@ def make_tokens case state.last in :outside case source[index..] + when /\A\n{2,}/ + # two or more newlines should be ONE blank line + enum.yield :blank_line, $&, index, line + line += $&.count("\n") when /\A(?: |\t|\n|\r\n)+/m # whitespace # enum.yield :whitespace, $&, index, line @@ -495,6 +500,12 @@ def parse_erb_tag end end + def parse_blank_line + blank_line = consume(:blank_line) + + CharData.new(value: blank_line, location: blank_line.location) + end + def parse_erb_do_close token = consume(:erb_do_close) diff --git a/test/fixture/block_formatted.html.erb b/test/fixture/block_formatted.html.erb index 10a6ee6..8bc99af 100644 --- a/test/fixture/block_formatted.html.erb +++ b/test/fixture/block_formatted.html.erb @@ -3,6 +3,7 @@ <%= form.label(:name, "Name") %> <%= form.text_field(:name, class: "form-control") %> + <%= form.submit( "Very very very very very long text", class: "btn btn-primary btn btn-primary btn btn-primary btn btn-primary" diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb index 8d552ce..1c6866f 100644 --- a/test/fixture/erb_syntax_formatted.html.erb +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -2,6 +2,7 @@ <%# This is an ERB-comment https://stackoverflow.com/a/25626629 this answer describes ERB and erubis syntax%> <%== rails_raw_output %> <%- "this only works in ERB not erubis" %> + <% if this -%> <%= form.submit -%> <% elsif that -%> @@ -9,6 +10,7 @@ <% else -%> <%= form.submit -%> <% end -%> + <%- if this %> <%= form.submit -%> <%- elsif that %> @@ -16,6 +18,7 @@ <%- else %> <%= form.submit -%> <%- end %> + <%= link_to(link, text) do -%>

Cool

<%- end %> diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/vue_components_formatted.html.erb index 97ffbbf..e8a7ac4 100644 --- a/test/fixture/vue_components_formatted.html.erb +++ b/test/fixture/vue_components_formatted.html.erb @@ -6,5 +6,6 @@ :long-variable-name="data.item.javascript.code" > + From d56cf874912a6864a83e6d3ebd2511fdff4c3107 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 01:23:18 +0200 Subject: [PATCH 11/73] Support full layouts and doctype (#8) --- lib/syntax_tree/erb/format.rb | 11 ++++++ lib/syntax_tree/erb/nodes.rb | 29 +++++++++++++++ lib/syntax_tree/erb/parser.rb | 20 ++++++++-- lib/syntax_tree/erb/pretty_print.rb | 5 +++ test/erb_test.rb | 4 ++ test/fixture/layout_formatted.html.erb | 47 ++++++++++++++++++++++++ test/fixture/layout_unformatted.html.erb | 33 +++++++++++++++++ 7 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 test/fixture/layout_formatted.html.erb create mode 100644 test/fixture/layout_unformatted.html.erb diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 30e3f57..76c4cc5 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -198,6 +198,17 @@ def visit_char_data(node) end end + # Visit a Doctype node. + def visit_doctype(node) + q.group do + visit(node.opening) + q.text(" ") + visit(node.name) + + visit(node.closing) + end + end + private # Format a text by splitting nicely at newlines and spaces. diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 5e134cc..965f5f1 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -464,5 +464,34 @@ def deconstruct_keys(keys) { value: value, location: location } end end + + # A document type declaration is a special kind of tag that specifies the + # type of the document. It contains an opening declaration, the name of + # the document type, an optional external identifier, and a closing of the + # tag. + class DocType < Node + attr_reader :opening, :name, :closing, :location + + def initialize(opening:, name:, closing:, location:) + @opening = opening + @name = name + @closing = closing + @location = location + end + + def accept(visitor) + visitor.visit_doctype(self) + end + + def child_nodes + [opening, name, closing].compact + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { opening: opening, name: name, closing: closing, location: location } + end + end end end diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 066b85b..d3f44e7 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -44,12 +44,13 @@ def initialize(source) end def parse + doctype = maybe { parse_doctype } elements = many { parse_any_tag } location = elements.first.location.to(elements.last.location) if elements.any? - Document.new(elements: elements, location: location) + Document.new(elements: [doctype].compact + elements, location: location) end def debug_tokens @@ -121,7 +122,7 @@ def make_tokens # the beginning of a string enum.yield :string_open, $&, index, line state << :string - when /\A[^<&]+/ + when /\A[^<]+/ # plain text content # abc enum.yield :text, $&, index, line @@ -200,7 +201,7 @@ def make_tokens # <% enum.yield :erb_open, $&, index, line state << :erb - when /\A[^<&"]+/ + when /\A[^<"]+/ # plain text content # abc enum.yield :text, $&, index, line @@ -581,6 +582,19 @@ def parse_chardata CharData.new(value: token, location: token.location) if token end + + def parse_doctype + opening = consume(:doctype) + name = consume(:name) + closing = consume(:close) + + DocType.new( + opening: opening, + name: name, + closing: closing, + location: opening.location.to(closing.location) + ) + end end end end diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index c0065b1..e63c570 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -120,6 +120,11 @@ def visit_erb_do_close(node) visit_node("erb_do_close", node) end + # Visit a Doctype node. + def visit_doctype(node) + visit_node("doctype", node) + end + private # A generic visit node function for how we pretty print nodes. diff --git a/test/erb_test.rb b/test/erb_test.rb index e8cfddb..6052989 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -24,6 +24,10 @@ def test_vue_components assert_parsing("vue_components") end + def test_layout + assert_parsing("layout") + end + def test_empty_file parsed = ERB.parse("") assert_instance_of(SyntaxTree::ERB::Document, parsed) diff --git a/test/fixture/layout_formatted.html.erb b/test/fixture/layout_formatted.html.erb new file mode 100644 index 0000000..c5ac740 --- /dev/null +++ b/test/fixture/layout_formatted.html.erb @@ -0,0 +1,47 @@ + + + + + + + + + <%= full_title(t("general.title"), yield(:title)) %> + <%= stylesheet_link_tag( + "application", + media: "all", + "data-turbolinks-track": "reload" + ) %> + <%= javascript_include_tag( + "application", + "data-turbolinks-track": "reload", + defer: true + ) %> + <%= csrf_meta_tags %> + <%= csp_meta_tag %> + <%= render("application/favicon") %> + + + +
+ <%= render("header") %> + +
+ <%= yield(:sidebar) %> +
+ <%= render("application/heading") %> + <%= render("flashes") %> + <%= yield %> +
+
+ +
<%= render("footer") %>
+
+ + diff --git a/test/fixture/layout_unformatted.html.erb b/test/fixture/layout_unformatted.html.erb new file mode 100644 index 0000000..3176d5a --- /dev/null +++ b/test/fixture/layout_unformatted.html.erb @@ -0,0 +1,33 @@ + + + + + + + <%= full_title(t('general.title'), yield(:title)) %> + <%= stylesheet_link_tag('application', media: 'all', 'data-turbolinks-track': 'reload') %> + <%= javascript_include_tag('application', 'data-turbolinks-track': 'reload', defer: true) %> + <%= csrf_meta_tags %> + <%= csp_meta_tag %> + <%= render('application/favicon') %> + + + +
+ <%= render('header') %> + +
+ <%= yield(:sidebar) %> +
+ <%= render('application/heading') %> + <%= render('flashes') %> + <%= yield %> +
+
+ +
+ <%= render('footer') %> +
+
+ + From 995ed0fc8cb50fd1fbdfa8908518134d14576e4d Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 13:34:28 +0200 Subject: [PATCH 12/73] Changes error handling (#9) --- lib/syntax_tree/erb/nodes.rb | 8 ++++++ lib/syntax_tree/erb/parser.rb | 54 +++++++++++++++++++---------------- test/erb_test.rb | 16 +++++++++-- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 965f5f1..202a7cf 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -34,6 +34,14 @@ def to(other) def <=>(other) start_char <=> other.start_char end + + def to_s + if start_line == end_line + "line #{start_line}, char #{start_char}..#{end_char}" + else + "line #{start_line},char #{start_char} to line #{end_line}, char #{end_char}" + end + end end # A parent node that contains a bit of shared functionality. diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index d3f44e7..13e8c6c 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -23,19 +23,6 @@ class ParseError < StandardError class MissingTokenError < ParseError end - class ErbKeywordError < ParseError - end - - # This error is thrown when an erb-tag with a do-statement is parsed. - # It is used to control the flow of the parser. - class ErbDoTokenError < ParseError - attr_reader(:tag) - - def initialize(tag:) - @tag = tag - end - end - attr_reader :source, :tokens def initialize(source) @@ -330,13 +317,9 @@ def parse_until_erb(classes:) items = [] loop do - begin - result = parse_any_tag - items << result - break if classes.any? { |cls| result.is_a?(cls) } - rescue ErbKeywordError - break - end + result = parse_any_tag + items << result + break if classes.any? { |cls| result.is_a?(cls) } end items @@ -379,7 +362,21 @@ def parse_html_element if opening_tag.closing.value == ">" content = many { parse_any_tag } - closing_tag = parse_html_closing_tag + closing_tag = maybe { parse_html_closing_tag } + + if closing_tag.nil? + raise( + ParseError, + "Missing closing tag for <#{opening_tag.name.value}> at #{opening_tag.location}" + ) + end + + if closing_tag.name.value != opening_tag.name.value + raise( + ParseError, + "Expected closing tag for <#{opening_tag.name.value}> but got <#{closing_tag.name.value}> at #{closing_tag.location}" + ) + end HtmlNode.new( opening_tag: opening_tag, @@ -404,7 +401,10 @@ def parse_erb_if(erb_node) erb_tag = elements.pop unless erb_tag.is_a?(ErbControl) || erb_tag.is_a?(ErbEnd) - raise(ErbKeywordError, "Found no matching tag to the if-tag") + raise( + ParseError, + "Found no matching tag to the if-tag at #{erb_node.location}" + ) end case erb_node.keyword.type @@ -431,7 +431,10 @@ def parse_erb_else(erb_node) erb_end = elements.pop unless erb_end.is_a?(ErbEnd) - raise(ErbKeywordError, "Found no matching end-tag for the else-tag") + raise( + ParseError, + "Found no matching end-tag for the else-tag at #{erb_node.location}" + ) end ErbElse.new(erb_node: erb_node, elements: elements, consequent: erb_end) @@ -487,7 +490,10 @@ def parse_erb_tag erb_end = elements.pop unless erb_end.is_a?(ErbEnd) - raise(ErbKeywordError, "Found no matching end-tag for the do-tag") + raise( + ParseError, + "Found no matching end-tag for the do-tag at #{erb_node.location}" + ) end ErbBlock.new( diff --git a/test/erb_test.rb b/test/erb_test.rb index 6052989..6481a12 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -35,12 +35,24 @@ def test_empty_file assert_nil(parsed.location) end - def test_invalid_file - assert_raises(SyntaxTree::ERB::Parser::ErbKeywordError) do + def test_missing_erb_end_tag + assert_raises(SyntaxTree::ERB::Parser::ParseError) do ERB.parse("<% if no_end_tag %>") end end + def test_missing_html_end_tag + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("

Hello World") + end + end + + def test_incorrect_html_end_tag + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("

Hello World

") + end + end + private def assert_parsing(name) From ac9b4d367a8b956a637aeec68060addafc4f9567 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 16:02:17 +0200 Subject: [PATCH 13/73] Handles edge cases where blocks failed to parse (#10) - A block containing the word `do` generated some errors. In ```ruby <% what_to_do do |form| %> ``` the `do` at the end of `what_to_do` was interpreted as the start of a block. To avoid this all erb-code is now parsed between word boundaries. So `what_to_do` will become one Token with type `erb_code`. --- lib/syntax_tree/erb/format.rb | 16 ++- lib/syntax_tree/erb/nodes.rb | 25 ++-- lib/syntax_tree/erb/parser.rb | 127 ++++++++++++++---- lib/syntax_tree/erb/pretty_print.rb | 10 +- lib/syntax_tree/erb/visitor.rb | 4 +- test/erb_test.rb | 6 + test/fixture/block_formatted.html.erb | 6 + test/fixture/block_unformatted.html.erb | 5 + test/fixture/nested_html_formatted.html.erb | 5 +- test/fixture/nested_html_unformatted.html.erb | 2 +- 10 files changed, 152 insertions(+), 54 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 76c4cc5..c8fdbf4 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -78,9 +78,11 @@ def visit_erb_block(node) end def visit_erb_do_close(node) - q.text(node.value.rstrip) - q.text(" ") - q.text(node.closing) + visit(node.closing) + end + + def visit_erb_close(node) + visit(node.closing) end # Visit an ErbIf node. @@ -178,12 +180,12 @@ def visit_attribute(node) end end - # Visit an ErbString node. - def visit_erb_string(node) + # Visit a HtmlString node. + def visit_html_string(node) q.group do - visit(node.opening) + q.text("\"") q.seplist(node.contents, -> { "" }) { |child_node| visit(child_node) } - visit(node.closing) + q.text("\"") end end diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 202a7cf..5bbd5ce 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -212,7 +212,7 @@ class ErbNode < Node def initialize(opening_tag:, keyword:, content:, closing_tag:, location:) @opening_tag = opening_tag @keyword = keyword - @content = ErbContent.new(value: content) if content + @content = ErbContent.new(value: content.map(&:value).join) if content @closing_tag = closing_tag @location = location end @@ -268,17 +268,16 @@ def deconstruct_keys(keys) end end - class ErbDoClose < Node - attr_reader :location, :value, :closing + class ErbClose < Node + attr_reader :location, :closing - def initialize(location:, value:, closing:) + def initialize(location:, closing:) @location = location - @value = value @closing = closing end def accept(visitor) - visitor.visit_erb_do_close(self) + visitor.visit_erb_close(self) end def child_nodes @@ -288,7 +287,13 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(keys) - { location: location, value: value, closing: closing } + { location: location, closing: closing } + end + end + + class ErbDoClose < ErbClose + def accept(visitor) + visitor.visit_erb_do_close(self) end end @@ -417,8 +422,8 @@ def deconstruct_keys(keys) end end - # An ErbString can include ERB-tags - class ErbString < Node + # A HtmlString can include ERB-tags + class HtmlString < Node attr_reader :opening, :contents, :closing, :location def initialize(opening:, contents:, closing:, location:) @@ -429,7 +434,7 @@ def initialize(opening:, contents:, closing:, location:) end def accept(visitor) - visitor.visit_erb_string(self) + visitor.visit_html_string(self) end def child_nodes diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 13e8c6c..c23d852 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -106,9 +106,13 @@ def make_tokens enum.yield :open, $&, index, line state << :inside when /\A"/ - # the beginning of a string - enum.yield :string_open, $&, index, line - state << :string + # the beginning of a double quoted string + enum.yield :string_open_double_quote, $&, index, line + state << :string_double_quote + when /\A'/ + # the beginning of a double quoted string + enum.yield :string_open_single_quote, $&, index, line + state << :string_single_quote when /\A[^<]+/ # plain text content # abc @@ -162,18 +166,45 @@ def make_tokens when /\A[\n]+/ # whitespace line += $&.count("\n") - when /\Ado\s*.*?\s*%>/ + when /\Ado\b(\s*\|[\w\s,]+\|)?\s*-?%>/ enum.yield :erb_do_close, $&, index, line state.pop when /\A-?%>/ enum.yield :erb_close, $&, index, line state.pop + when /\A\w*\b/ + # Split by word boundary while parsing the code + # This allows us to separate what_to_do vs do + enum.yield :erb_code, $&, index, line else enum.yield :erb_code, source[index], index, line index += 1 next end - in :string + in :string_single_quote + case source[index..] + when /\A(?: |\t|\n|\r\n)+/m + # whitespace + enum.yield :whitespace, $&, index, line + line += $&.count("\n") + when /\A\'/ + # the end of a quoted string + enum.yield :string_close_single_quote, $&, index, line + state.pop + when /\A<%[=]?/ + # the beginning of an ERB tag + # <% + enum.yield :erb_open, $&, index, line + state << :erb + when /\A[^<']+/ + # plain text content + # abc + enum.yield :text, $&, index, line + else + raise ParseError, + "Unexpected character in string at #{index}: #{source[index]}" + end + in :string_double_quote case source[index..] when /\A(?: |\t|\n|\r\n)+/m # whitespace @@ -181,7 +212,7 @@ def make_tokens line += $&.count("\n") when /\A\"/ # the end of a quoted string - enum.yield :string_close, $&, index, line + enum.yield :string_close_double_quote, $&, index, line state.pop when /\A<%[=]?/ # the beginning of an ERB tag @@ -239,8 +270,12 @@ def make_tokens state << :erb when /\A"/ # the beginning of a string - enum.yield :string_open, $&, index, line - state << :string + enum.yield :string_open_double_quote, $&, index, line + state << :string_double_quote + when /\A'/ + # the beginning of a string + enum.yield :string_open_single_quote, $&, index, line + state << :string_single_quote else raise ParseError, "Unexpected character at #{index}: #{source[index]}" @@ -462,17 +497,22 @@ def parse_erb_tag maybe { consume(:erb_if) } || maybe { consume(:erb_unless) } || maybe { consume(:erb_elsif) } || maybe { consume(:erb_else) } || maybe { consume(:erb_end) } - content = many { consume(:erb_code) } - closing_tag = - atleast do - maybe { consume(:erb_close) } || maybe { parse_erb_do_close } - end + + content = parse_until_erb_close + closing_tag = content.pop + + if !closing_tag.is_a?(ErbClose) + raise( + ParseError, + "Found no matching closing tag for the erb-tag at #{opening_tag.location}" + ) + end erb_node = ErbNode.new( opening_tag: opening_tag, keyword: keyword, - content: content.map(&:value).join, + content: content, closing_tag: closing_tag, location: opening_tag.location.to(closing_tag.location) ) @@ -486,7 +526,7 @@ def parse_erb_tag parse_erb_end(erb_node) else if closing_tag.is_a?(ErbDoClose) - elements = parse_until_erb(classes: [ErbEnd]) + elements = maybe { parse_until_erb(classes: [ErbEnd]) } || [] erb_end = elements.pop unless erb_end.is_a?(ErbEnd) @@ -507,26 +547,45 @@ def parse_erb_tag end end + def parse_until_erb_close + items = [] + + loop do + result = + maybe { parse_erb_do_close } || maybe { parse_erb_close } || + maybe { consume(:erb_code) } + items << result + + break if result.is_a?(ErbClose) + end + + items + end + def parse_blank_line blank_line = consume(:blank_line) CharData.new(value: blank_line, location: blank_line.location) end - def parse_erb_do_close - token = consume(:erb_do_close) + def parse_erb_close + closing = consume(:erb_close) + + ErbClose.new(location: closing.location, closing: closing) + end - closing = token.value.match(/-?%>$/).to_s + def parse_erb_do_close + closing = consume(:erb_do_close) - ErbDoClose.new( - location: token.location, - value: token.value.gsub(closing, ""), - closing: closing - ) + ErbDoClose.new(location: closing.location, closing: closing) end - def parse_string - opening = consume(:string_open) + def parse_html_string + opening = + atleast do + maybe { consume(:string_open_double_quote) } || + maybe { consume(:string_open_single_quote) } + end contents = many do atleast do @@ -534,9 +593,15 @@ def parse_string maybe { parse_erb_tag } end end - closing = consume(:string_close) - ErbString.new( + closing = + if opening.type == :string_open_double_quote + consume(:string_close_double_quote) + else + consume(:string_close_single_quote) + end + + HtmlString.new( opening: opening, contents: contents, closing: closing, @@ -556,7 +621,7 @@ def parse_html_attribute location: key.location ) else - value = parse_string + value = parse_html_string HtmlAttribute.new( key: key, @@ -571,7 +636,11 @@ def parse_chardata values = many do atleast do - maybe { consume(:text) } || maybe { consume(:whitespace) } + maybe { consume(:string_open_double_quote) } || + maybe { consume(:string_open_single_quote) } || + maybe { consume(:string_close_double_quote) } || + maybe { consume(:string_close_single_quote) } || + maybe { consume(:text) } || maybe { consume(:whitespace) } end end diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index e63c570..f060d15 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -106,9 +106,9 @@ def visit_attribute(node) visit_node("attribute", node) end - # Visit an ErbString node. - def visit_erb_string(node) - visit_node("erb_string", node) + # Visit a HtmlString node. + def visit_html_string(node) + visit_node("html_string", node) end # Visit a CharData node. @@ -116,6 +116,10 @@ def visit_char_data(node) visit_node("char_data", node) end + def visit_erb_close(node) + visit_node("erb_close", node) + end + def visit_erb_do_close(node) visit_node("erb_do_close", node) end diff --git a/lib/syntax_tree/erb/visitor.rb b/lib/syntax_tree/erb/visitor.rb index fbc9350..0652b4e 100644 --- a/lib/syntax_tree/erb/visitor.rb +++ b/lib/syntax_tree/erb/visitor.rb @@ -50,8 +50,8 @@ def visit_child_nodes(node) # Visit an ErbNode node. alias visit_erb visit_child_nodes - # Visit an ErbString node. - alias visit_erb_string visit_child_nodes + # Visit a HtmlString node. + alias visit_html_string visit_child_nodes end end end diff --git a/test/erb_test.rb b/test/erb_test.rb index 6481a12..c0b44e7 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -41,6 +41,12 @@ def test_missing_erb_end_tag end end + def test_missing_erb_block_end_tag + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("<% no_end_tag do %>") + end + end + def test_missing_html_end_tag assert_raises(SyntaxTree::ERB::Parser::ParseError) do ERB.parse("

Hello World") diff --git a/test/fixture/block_formatted.html.erb b/test/fixture/block_formatted.html.erb index 8bc99af..bed1d7f 100644 --- a/test/fixture/block_formatted.html.erb +++ b/test/fixture/block_formatted.html.erb @@ -9,3 +9,9 @@ class: "btn btn-primary btn btn-primary btn btn-primary btn btn-primary" ) %> <% end %> + +<%= link_to(dont_replace, what_to_do, class: "do |what,bad|") do |hello| %> + Should allow to use the word do in the code +<% end %> + +<%= this_is_not_a_do_block_do %> diff --git a/test/fixture/block_unformatted.html.erb b/test/fixture/block_unformatted.html.erb index 739a2f2..bfb0796 100644 --- a/test/fixture/block_unformatted.html.erb +++ b/test/fixture/block_unformatted.html.erb @@ -17,3 +17,8 @@ <%= form.submit("Very very very very very long text", class: "btn btn-primary btn btn-primary btn btn-primary btn btn-primary") %> <% end %> + +<%= link_to(dont_replace, what_to_do, class: 'do |what,bad|') do |hello| %>Should allow to use the word do in the code +<% end %> + +<%= this_is_not_a_do_block_do %> diff --git a/test/fixture/nested_html_formatted.html.erb b/test/fixture/nested_html_formatted.html.erb index 724038c..a263dc9 100644 --- a/test/fixture/nested_html_formatted.html.erb +++ b/test/fixture/nested_html_formatted.html.erb @@ -1,5 +1,6 @@ -
- +
+ <%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %> + '"This is a quote within a quote"'
diff --git a/test/fixture/nested_html_unformatted.html.erb b/test/fixture/nested_html_unformatted.html.erb index 55c0e03..4cdbf73 100644 --- a/test/fixture/nested_html_unformatted.html.erb +++ b/test/fixture/nested_html_unformatted.html.erb @@ -1 +1 @@ -
<%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %>
+
<%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %>'"This is a quote within a quote"'
From e4d42a1e5d9421c518676aee2db0833e574f3831 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 16:37:44 +0200 Subject: [PATCH 14/73] Adds test for parsing unmatched quotes (#11) --- test/erb_test.rb | 57 +------------------ test/fixture/nested_html_formatted.html.erb | 3 +- test/fixture/nested_html_unformatted.html.erb | 2 +- test/formatting_test.rb | 31 ++++++++++ test/html_test.rb | 30 ++++++++++ test/test_helper.rb | 19 +++++++ 6 files changed, 84 insertions(+), 58 deletions(-) create mode 100644 test/formatting_test.rb create mode 100644 test/html_test.rb diff --git a/test/erb_test.rb b/test/erb_test.rb index c0b44e7..060269f 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -3,31 +3,7 @@ require "test_helper" module SyntaxTree - class ERBTest < Minitest::Test - def test_block - assert_parsing("block") - end - - def test_erb_syntax - assert_parsing("erb_syntax") - end - - def test_nested_html - assert_parsing("nested_html") - end - - def test_if_statements - assert_parsing("if_statements") - end - - def test_vue_components - assert_parsing("vue_components") - end - - def test_layout - assert_parsing("layout") - end - + class ErbTest < TestCase def test_empty_file parsed = ERB.parse("") assert_instance_of(SyntaxTree::ERB::Document, parsed) @@ -46,36 +22,5 @@ def test_missing_erb_block_end_tag ERB.parse("<% no_end_tag do %>") end end - - def test_missing_html_end_tag - assert_raises(SyntaxTree::ERB::Parser::ParseError) do - ERB.parse("

Hello World") - end - end - - def test_incorrect_html_end_tag - assert_raises(SyntaxTree::ERB::Parser::ParseError) do - ERB.parse("

Hello World

") - end - end - - private - - def assert_parsing(name) - directory = File.expand_path("fixture", __dir__) - unformatted_file = File.join(directory, "#{name}_unformatted.html.erb") - formatted_file = File.join(directory, "#{name}_formatted.html.erb") - - expected = ERB.read(formatted_file) - formatted = ERB.format(ERB.read(unformatted_file)) - - if (expected != formatted) - puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") - Dir.mkdir("./tmp") unless Dir.exist?("./tmp") - File.write("./tmp/#{name}_failed.html.erb", formatted) - end - - assert_equal(formatted, expected) - end end end diff --git a/test/fixture/nested_html_formatted.html.erb b/test/fixture/nested_html_formatted.html.erb index a263dc9..e35a4b2 100644 --- a/test/fixture/nested_html_formatted.html.erb +++ b/test/fixture/nested_html_formatted.html.erb @@ -1,6 +1,7 @@
<%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %> - '"This is a quote within a quote"' + 'This is a single quote' + "This is a double quote"
diff --git a/test/fixture/nested_html_unformatted.html.erb b/test/fixture/nested_html_unformatted.html.erb index 4cdbf73..ce02151 100644 --- a/test/fixture/nested_html_unformatted.html.erb +++ b/test/fixture/nested_html_unformatted.html.erb @@ -1 +1 @@ -
<%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %>'"This is a quote within a quote"'
+
<%= t(".title") + " " + t(".description") + " " + t(".pretty_long") %>'This is a single quote'"This is a double quote"
diff --git a/test/formatting_test.rb b/test/formatting_test.rb new file mode 100644 index 0000000..e87f612 --- /dev/null +++ b/test/formatting_test.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "test_helper" + +module SyntaxTree + class FormattingTest < TestCase + def test_block + assert_formatting("block") + end + + def test_erb_syntax + assert_formatting("erb_syntax") + end + + def test_nested_html + assert_formatting("nested_html") + end + + def test_if_statements + assert_formatting("if_statements") + end + + def test_vue_components + assert_formatting("vue_components") + end + + def test_layout + assert_formatting("layout") + end + end +end diff --git a/test/html_test.rb b/test/html_test.rb new file mode 100644 index 0000000..f392bb8 --- /dev/null +++ b/test/html_test.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require "test_helper" + +module SyntaxTree + class HtmlTest < Minitest::Test + def test_html_missing_end_tag + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("

Hello World") + end + end + + def test_html_incorrect_end_tag + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("

Hello World

") + end + end + + def test_html_unmatched_double_quote + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("
Hello World
") + end + end + def test_html_unmatched_single_quote + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("
Hello World
") + end + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 365ca3c..0b24995 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -7,3 +7,22 @@ require "syntax_tree/erb" require "minitest/autorun" + +class TestCase < Minitest::Test + def assert_formatting(name) + directory = File.expand_path("fixture", __dir__) + unformatted_file = File.join(directory, "#{name}_unformatted.html.erb") + formatted_file = File.join(directory, "#{name}_formatted.html.erb") + + expected = SyntaxTree::ERB.read(formatted_file) + formatted = SyntaxTree::ERB.format(SyntaxTree::ERB.read(unformatted_file)) + + if (expected != formatted) + puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") + Dir.mkdir("./tmp") unless Dir.exist?("./tmp") + File.write("./tmp/#{name}_failed.html.erb", formatted) + end + + assert_equal(formatted, expected) + end +end From 23a305673b18c7ff8e6e1cdb6149d27c6a28652a Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 16:45:28 +0200 Subject: [PATCH 15/73] Doctype: Handles casing (#12) --- lib/syntax_tree/erb/nodes.rb | 2 +- lib/syntax_tree/erb/parser.rb | 4 ++-- test/html_test.rb | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 5bbd5ce..8489a01 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -482,7 +482,7 @@ def deconstruct_keys(keys) # type of the document. It contains an opening declaration, the name of # the document type, an optional external identifier, and a closing of the # tag. - class DocType < Node + class Doctype < Node attr_reader :opening, :name, :closing, :location def initialize(opening:, name:, closing:, location:) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index c23d852..d197c2a 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -79,7 +79,7 @@ def make_tokens # enum.yield :comment, $&, index, line line += $&.count("\n") - when /\AHello World
") end end + def test_html_unmatched_single_quote assert_raises(SyntaxTree::ERB::Parser::ParseError) do ERB.parse("
Hello World
") end end + + def test_html_doctype + parsed = ERB.parse("") + assert_instance_of(SyntaxTree::ERB::Doctype, parsed.elements.first) + + parsed = ERB.parse("") + assert_instance_of(SyntaxTree::ERB::Doctype, parsed.elements.first) + end end end From b8e0e0c0bcd786304113cf0c9f4360ff80534b33 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 17:04:22 +0200 Subject: [PATCH 16/73] ERB: Consider newlines before parsing the code (#13) --- lib/syntax_tree/erb/parser.rb | 5 +++-- test/fixture/erb_syntax_formatted.html.erb | 12 ++++++++++++ test/fixture/erb_syntax_unformatted.html.erb | 13 +++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index d197c2a..14ca13d 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -164,7 +164,8 @@ def make_tokens in :erb case source[index..] when /\A[\n]+/ - # whitespace + # newlines + enum.yield :erb_code, $&, index, line line += $&.count("\n") when /\Ado\b(\s*\|[\w\s,]+\|)?\s*-?%>/ enum.yield :erb_do_close, $&, index, line @@ -438,7 +439,7 @@ def parse_erb_if(erb_node) unless erb_tag.is_a?(ErbControl) || erb_tag.is_a?(ErbEnd) raise( ParseError, - "Found no matching tag to the if-tag at #{erb_node.location}" + "Found no matching erb-tag to the if-tag at #{erb_node.location}" ) end diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb index 1c6866f..c1fcec6 100644 --- a/test/fixture/erb_syntax_formatted.html.erb +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -22,3 +22,15 @@ <%= link_to(link, text) do -%>

Cool

<%- end %> + +<%= t( + ".verified_at", + at: + ( + if @repository.github_status_at + l(@repository.github_status_at, format: :long) + else + "?" + end + ) +) %> diff --git a/test/fixture/erb_syntax_unformatted.html.erb b/test/fixture/erb_syntax_unformatted.html.erb index 3ff4b26..c5b0fb9 100644 --- a/test/fixture/erb_syntax_unformatted.html.erb +++ b/test/fixture/erb_syntax_unformatted.html.erb @@ -22,3 +22,16 @@ <%= link_to(link, text) do -%>

Cool

<%- end %> + + +<%= t( + ".verified_at", + at: + ( + if @repository.github_status_at + l(@repository.github_status_at, format: :long) + else + "?" + end + ) +) %> From 10c73d0120e1d92779a9c8b0173e0aeae145efac Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 21 May 2023 22:15:50 +0200 Subject: [PATCH 17/73] Handles HTML-comments (#14) --- lib/syntax_tree/erb/format.rb | 4 ++++ lib/syntax_tree/erb/nodes.rb | 23 +++++++++++++++++++++++ lib/syntax_tree/erb/parser.rb | 14 ++++++++++---- lib/syntax_tree/erb/pretty_print.rb | 4 ++++ test/html_test.rb | 11 +++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index c8fdbf4..4ef9a7e 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -189,6 +189,10 @@ def visit_html_string(node) end end + def visit_html_comment(node) + visit(node.token) + end + # Visit a CharData node. def visit_char_data(node) lines = node.value.value.strip.split("\n") diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 8489a01..4b57246 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -453,6 +453,29 @@ def deconstruct_keys(keys) end end + class HtmlComment < Node + attr_reader :token, :location + + def initialize(token:, location:) + @token = token + @location = location + end + + def accept(visitor) + visitor.visit_html_comment(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { token: token, location: location } + end + end + # A CharData contains either plain text or whitespace within an element. # It wraps a single token value. class CharData < Node diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 14ca13d..89e93c0 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -50,9 +50,9 @@ def debug_tokens def parse_any_tag atleast do - maybe { parse_erb_tag } || maybe { consume(:erb_comment) } || - maybe { parse_html_element } || maybe { parse_blank_line } || - maybe { parse_chardata } + maybe { parse_html_comment } || maybe { parse_erb_tag } || + maybe { consume(:erb_comment) } || maybe { parse_html_element } || + maybe { parse_blank_line } || maybe { parse_chardata } end end @@ -77,7 +77,7 @@ def make_tokens when /\A/m # comments # - enum.yield :comment, $&, index, line + enum.yield :html_comment, $&, index, line line += $&.count("\n") when /\A") assert_instance_of(SyntaxTree::ERB::Doctype, parsed.elements.first) end + + def test_html_comment + source = "\n" + parsed = ERB.parse(source) + elements = parsed.elements + assert_equal(1, elements.size) + assert_instance_of(SyntaxTree::ERB::HtmlComment, elements.first) + + formatted = ERB.format(source) + assert_equal(source, formatted) + end end end From dc3ab8d47518dce5559f3f32239c745098e1f1e5 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Thu, 25 May 2023 21:52:20 +0200 Subject: [PATCH 18/73] Handles non-ascii ERB-code (#15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following code caused errors: ``` <%= link_to("Düsseldorf", "http://duesseldorf.de") %> ``` --- lib/syntax_tree/erb/parser.rb | 2 +- test/erb_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 89e93c0..fa309c1 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -173,7 +173,7 @@ def make_tokens when /\A-?%>/ enum.yield :erb_close, $&, index, line state.pop - when /\A\w*\b/ + when /\A[\p{L}\w]*\b/ # Split by word boundary while parsing the code # This allows us to separate what_to_do vs do enum.yield :erb_code, $&, index, line diff --git a/test/erb_test.rb b/test/erb_test.rb index 060269f..87148ba 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -22,5 +22,11 @@ def test_missing_erb_block_end_tag ERB.parse("<% no_end_tag do %>") end end + + def test_erb_code_with_non_ascii + parsed = ERB.parse("<% \"Påäööööö\" %>") + assert_equal(1, parsed.elements.size) + assert_instance_of(SyntaxTree::ERB::ErbNode, parsed.elements.first) + end end end From df4acbfc3db027f284d513b5642eb6696c13b3c3 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 26 May 2023 06:55:09 +0200 Subject: [PATCH 19/73] Support quotes in HTML-text (#16) ```html

Here is a "<%= quote %>" in HTML

``` --- lib/syntax_tree/erb/parser.rb | 8 -------- test/html_test.rb | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index fa309c1..6cf5a03 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -105,14 +105,6 @@ def make_tokens # < enum.yield :open, $&, index, line state << :inside - when /\A"/ - # the beginning of a double quoted string - enum.yield :string_open_double_quote, $&, index, line - state << :string_double_quote - when /\A'/ - # the beginning of a double quoted string - enum.yield :string_open_single_quote, $&, index, line - state << :string_single_quote when /\A[^<]+/ # plain text content # abc diff --git a/test/html_test.rb b/test/html_test.rb index 9252e15..d2a494a 100644 --- a/test/html_test.rb +++ b/test/html_test.rb @@ -46,5 +46,19 @@ def test_html_comment formatted = ERB.format(source) assert_equal(source, formatted) end + + def test_html_within_quotes + source = + "

This is our text \"<%= @object.quote %>\"

" + parsed = ERB.parse(source) + elements = parsed.elements + + assert_equal(1, elements.size) + assert_instance_of(SyntaxTree::ERB::HtmlNode, elements.first) + content = elements.first.content + + assert_equal("This is our text \"", content.first.value.value) + assert_equal("\"", content.last.value.value) + end end end From 526b80a92c5bae8a4783ae2cac8c1c0299a4a1a7 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 26 May 2023 07:32:29 +0200 Subject: [PATCH 20/73] Allow html attribute to start with @:# (#17) In Vue we often use ``` ``` --- lib/syntax_tree/erb/parser.rb | 18 +++++++----------- test/fixture/vue_components_formatted.html.erb | 4 +++- .../vue_components_unformatted.html.erb | 2 +- test/html_test.rb | 12 ++++++++++++ 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 6cf5a03..4628ac6 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -3,14 +3,6 @@ module SyntaxTree module ERB class Parser - NAME_START = - "[@:a-zA-Z_\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFFD}]" - - NAME_CHAR = - "[#{NAME_START}-\\.\\d\u{00B7}\u{0300}-\u{036F}\u{203F}-\u{2040}]" - - NAME = "#{NAME_START}(?:#{NAME_CHAR})*" - # This is the parent class of any kind of errors that will be raised by # the parser. class ParseError < StandardError @@ -252,9 +244,10 @@ def make_tokens # an equals sign # = enum.yield :equals, $&, index, line - when /\A#{NAME}/ - # a name - # abc + when /\A[@:#]*[\w-]+\b/ + # a name for an element or an attribute + # strong, vue-component-kebab, VueComponentPascal + # abc, #abc, @abc, :abc enum.yield :name, $&, index, line when /\A<%/ # the beginning of an ERB tag @@ -356,6 +349,9 @@ def parse_until_erb(classes:) def parse_html_opening_tag opening = consume(:open) name = consume(:name) + if name.value =~ /\A[@:#]/ + raise ParseError, "Invalid html-tag name #{name}" + end attributes = many { parse_html_attribute } closing = diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/vue_components_formatted.html.erb index e8a7ac4..623d4da 100644 --- a/test/fixture/vue_components_formatted.html.erb +++ b/test/fixture/vue_components_formatted.html.erb @@ -7,5 +7,7 @@ > - + + + diff --git a/test/fixture/vue_components_unformatted.html.erb b/test/fixture/vue_components_unformatted.html.erb index 9764266..e7a6511 100644 --- a/test/fixture/vue_components_unformatted.html.erb +++ b/test/fixture/vue_components_unformatted.html.erb @@ -1,3 +1,3 @@
" boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code"> -
+ diff --git a/test/html_test.rb b/test/html_test.rb index d2a494a..fc22506 100644 --- a/test/html_test.rb +++ b/test/html_test.rb @@ -60,5 +60,17 @@ def test_html_within_quotes assert_equal("This is our text \"", content.first.value.value) assert_equal("\"", content.last.value.value) end + + def test_html_tag_names + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("<@br />") + end + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("<:br />") + end + assert_raises(SyntaxTree::ERB::Parser::ParseError) do + ERB.parse("<#br />") + end + end end end From 189471485c5258a4e0d535f427c9d1b73a658e4d Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 26 May 2023 07:36:16 +0200 Subject: [PATCH 21/73] Handles dot, dash and underscore in attribute names (#18) Vue uses syntax like `v-b-modal.name_of_the_modal` for attribute names. --- lib/syntax_tree/erb/parser.rb | 2 +- test/fixture/vue_components_formatted.html.erb | 1 + test/fixture/vue_components_unformatted.html.erb | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 4628ac6..d02231a 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -244,7 +244,7 @@ def make_tokens # an equals sign # = enum.yield :equals, $&, index, line - when /\A[@:#]*[\w-]+\b/ + when /\A[@:#]*[\w\.\-\_]+\b/ # a name for an element or an attribute # strong, vue-component-kebab, VueComponentPascal # abc, #abc, @abc, :abc diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/vue_components_formatted.html.erb index 623d4da..e0516b2 100644 --- a/test/fixture/vue_components_formatted.html.erb +++ b/test/fixture/vue_components_formatted.html.erb @@ -1,5 +1,6 @@
" boolean :value="['a', 'b']" diff --git a/test/fixture/vue_components_unformatted.html.erb b/test/fixture/vue_components_unformatted.html.erb index e7a6511..363ccf0 100644 --- a/test/fixture/vue_components_unformatted.html.erb +++ b/test/fixture/vue_components_unformatted.html.erb @@ -1,3 +1,3 @@ -
" boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code"> +
" boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code">
From ad3647e1aea47625176a35a2cd6c7bd4bdf8f44b Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 26 May 2023 08:36:15 +0200 Subject: [PATCH 22/73] HTML: Handles attribute values without quotes (#19) ```
``` can now be parsed and formatted as ```
``` --- lib/syntax_tree/erb/parser.rb | 20 ++++++++++++++++---- test/html_test.rb | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index d02231a..352426e 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -571,10 +571,22 @@ def parse_erb_do_close def parse_html_string opening = - atleast do - maybe { consume(:string_open_double_quote) } || - maybe { consume(:string_open_single_quote) } - end + maybe { consume(:string_open_double_quote) } || + maybe { consume(:string_open_single_quote) } + + if opening.nil? + value = consume(:name) + + return( + HtmlString.new( + opening: nil, + contents: [value], + closing: nil, + location: value.location + ) + ) + end + contents = many do atleast do diff --git a/test/html_test.rb b/test/html_test.rb index fc22506..46762ad 100644 --- a/test/html_test.rb +++ b/test/html_test.rb @@ -72,5 +72,22 @@ def test_html_tag_names ERB.parse("<#br />") end end + + def test_html_attribute_without_quotes + source = "
Hello World
" + parsed = ERB.parse(source) + elements = parsed.elements + + assert_equal(1, elements.size) + assert_instance_of(SyntaxTree::ERB::HtmlNode, elements.first) + assert_equal(1, elements.first.opening_tag.attributes.size) + + attribute = elements.first.opening_tag.attributes.first + assert_equal("class", attribute.key.value) + assert_equal("card", attribute.value.contents.first.value) + + formatted = ERB.format(source) + assert_equal("
Hello World
\n", formatted) + end end end From 412b54f983ff068c5cc8774d2135020887b3854a Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 28 May 2023 11:22:06 +0200 Subject: [PATCH 23/73] Adds script for parsing all .html.erb and output (#20) which failed to parse --- check_erb_parse.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 check_erb_parse.rb diff --git a/check_erb_parse.rb b/check_erb_parse.rb new file mode 100644 index 0000000..13cefe4 --- /dev/null +++ b/check_erb_parse.rb @@ -0,0 +1,18 @@ +#!/bin/ruby + +require "syntax_tree/erb" + +failures = [] + +Dir + .glob("./app/**/*.html.erb") + .each do |file| + puts("Processing #{file}") + begin + SyntaxTree::ERB.parse(SyntaxTree::ERB.read(file)) + rescue => exception + failures << {file: file, message: exception.message} + end + end + +puts failures From 357e8e540532973f5cd59058b8128c412162f2a5 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 28 May 2023 12:58:58 +0200 Subject: [PATCH 24/73] Failing to format long if-statements multiple times (#21) --- lib/syntax_tree/erb/format.rb | 16 ++++++++++------ lib/syntax_tree/erb/nodes.rb | 16 ++++++++++------ test/erb_test.rb | 11 +++++++++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 4ef9a7e..2b38a25 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -115,12 +115,16 @@ def visit_erb_end(node) end def visit_erb_content(node) - formatter = - SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) - formatter.format(node.value.statements) - formatter.flush - - rows = formatter.output.join.split("\n") + rows = + if node.value.is_a?(String) + node.value.split("\n") + else + formatter = + SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) + formatter.format(node.value.statements) + formatter.flush + formatter.output.join.split("\n") + end if rows.size > 1 q.group do diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 4b57246..976040a 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -366,17 +366,21 @@ def child_nodes end class ErbContent < Node - attr_reader(:value, :parsed) + attr_reader(:value, :unparsed_value) def initialize(value:) - @value = value + @unparsed_value = value begin - @value = SyntaxTree.parse(@value) - @parsed = true + # We cannot handle IfNode inside a ErbContent + @value = + if SyntaxTree.search(value, "IfNode").any? + value&.lstrip&.rstrip + else + SyntaxTree.parse(value) + end rescue SyntaxTree::Parser::ParseError # Removes leading and trailing whitespace - @value = @value&.lstrip&.rstrip - @parsed = false + @value = value&.lstrip&.rstrip end end diff --git a/test/erb_test.rb b/test/erb_test.rb index 87148ba..576341d 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -28,5 +28,16 @@ def test_erb_code_with_non_ascii assert_equal(1, parsed.elements.size) assert_instance_of(SyntaxTree::ERB::ErbNode, parsed.elements.first) end + + def test_long_if_statement + source = + "<%= number_to_percentage(@reports&.first&.stability * 100, precision: 1) if @reports&.first %>\n" + + formatted = ERB.format(source) + formatted_again = ERB.format(formatted) + + assert_equal(source, formatted) + assert_equal(source, formatted_again) + end end end From b366e80d5db94dbef8832fbded71e97d8b9b00d1 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 28 May 2023 22:01:31 +0200 Subject: [PATCH 25/73] Changed how blocks are formatted (#22) --- lib/syntax_tree/erb/format.rb | 100 ++++--------- lib/syntax_tree/erb/nodes.rb | 140 +++++++----------- lib/syntax_tree/erb/parser.rb | 68 ++++----- lib/syntax_tree/erb/pretty_print.rb | 17 +-- lib/syntax_tree/erb/visitor.rb | 3 - test/erb_test.rb | 11 +- test/fixture/block_formatted.html.erb | 4 +- test/fixture/erb_syntax_formatted.html.erb | 4 +- test/fixture/if_statements_formatted.html.erb | 16 +- test/fixture/layout_formatted.html.erb | 10 +- .../fixture/vue_components_formatted.html.erb | 4 +- test/formatting_test.rb | 25 +++- test/html_test.rb | 12 +- test/test_helper.rb | 19 --- 14 files changed, 193 insertions(+), 240 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 2b38a25..fb6cc54 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -25,25 +25,45 @@ def visit_document(node) q.breakable(force: true) end - def visit_html(node) - q.group do - visit(node.opening_tag) + def visit_block(node) + visit(node.opening) - if node.content&.any? - q.indent do - q.breakable("") - q.seplist( - node.content, - -> { q.breakable(force: true) } - ) { |child_node| visit(child_node) } - end + if node.elements.any? + q.indent do + q.breakable("") + q.seplist( + node.elements, + -> { q.breakable(force: true) } + ) { |child_node| visit(child_node) } end + end + if node.closing q.breakable("") - visit(node.closing_tag) + visit(node.closing) end end + def visit_html(node) + visit_block(node) + end + + def visit_erb_block(node) + visit_block(node) + end + + def visit_erb_if(node) + visit_block(node) + end + + def visit_erb_elsif(node) + visit_block(node) + end + + def visit_erb_else(node) + visit_block(node) + end + # Visit an ErbNode node. def visit_erb(node) visit(node.opening_tag) @@ -58,25 +78,6 @@ def visit_erb(node) visit(node.closing_tag) end - def visit_erb_block(node) - visit(node.erb_node) - - if node.elements.any? - q.group do - q.indent do - q.breakable(force: true) - q.seplist( - node.elements, - -> { q.breakable(force: true) } - ) { |child_node| visit(child_node) } - end - end - end - - q.breakable("") - visit(node.consequent) - end - def visit_erb_do_close(node) visit(node.closing) end @@ -85,26 +86,6 @@ def visit_erb_close(node) visit(node.closing) end - # Visit an ErbIf node. - def visit_erb_if(node) - q.group do - visit(node.erb_node) - - if node.elements.any? - q.indent do - q.breakable(force: true) - q.seplist( - node.elements, - -> { q.breakable(force: true) } - ) { |child_node| visit(child_node) } - end - end - - q.breakable("") - visit(node.consequent) - end - end - # Visit an ErbEnd node. def visit_erb_end(node) visit(node.opening_tag) @@ -170,11 +151,6 @@ def visit_closing_tag(node) end end - # Visit a Reference node. - def visit_reference(node) - visit(node.value) - end - # Visit an Attribute node. def visit_attribute(node) q.group do @@ -218,18 +194,6 @@ def visit_doctype(node) visit(node.closing) end end - - private - - # Format a text by splitting nicely at newlines and spaces. - def format_text(q, value) - sep_line = -> { q.breakable(force: true, indent: false) } - sep_word = -> { q.group { q.breakable } } - - q.seplist(value.strip.split("\n"), sep_line) do |line| - q.seplist(line.split(/\b(?: +)\b/), sep_word) { |word| q.text(word) } - end - end end end end diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 976040a..7b8a621 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -110,11 +110,44 @@ def deconstruct_keys(keys) end end + # This is a base class for a block that contains: + # - an opening + # - optional elements + # - optional closing + class Block < Node + attr_reader(:opening, :elements, :closing, :location) + def initialize(opening:, location:, elements: nil, closing: nil) + @opening = opening + @elements = elements || [] + @closing = closing + @location = location + end + + def accept(visitor) + visitor.visit_block(self) + end + + def child_nodes + [opening, *elements, closing].compact + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { + opening: opening, + content: content, + closing: closing, + location: location + } + end + end + # An element is a child of the document. It contains an opening tag, any # optional content within the tag, and a closing tag. It can also # potentially contain an opening tag that self-closes, in which case the # content and closing tag will be nil. - class HtmlNode < Node + class HtmlNode < Block # The opening tag of an element. It contains the opening character (<), # the name of the element, any optional attributes, and the closing # token (either > or />). @@ -177,33 +210,9 @@ def deconstruct_keys(keys) end end - attr_reader :opening_tag, :content, :closing_tag, :location - - def initialize(opening_tag:, content:, closing_tag:, location:) - @opening_tag = opening_tag - @content = content - @closing_tag = closing_tag - @location = location - end - def accept(visitor) visitor.visit_html(self) end - - def child_nodes - [opening_tag, *content, closing_tag].compact - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { - opening_tag: opening_tag, - content: content, - closing_tag: closing_tag, - location: location - } - end end class ErbNode < Node @@ -238,34 +247,10 @@ def deconstruct_keys(keys) end end - class ErbBlock < Node - attr_reader :erb_node, :elements, :consequent, :location - - def initialize(erb_node:, elements:, consequent:) - @erb_node = erb_node - @elements = elements - @consequent = consequent - @location = erb_node.location.to(consequent.location) - end - + class ErbBlock < Block def accept(visitor) visitor.visit_erb_block(self) end - - def child_nodes - [*elements].compact - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { - erb_node: erb_node, - elements: elements, - consequent: consequent, - location: location - } - end end class ErbClose < Node @@ -297,60 +282,37 @@ def accept(visitor) end end - class ErbControl < Node - attr_reader :erb_node - - def initialize(erb_node:) - @erb_node = erb_node - end - - def location - erb_node.location - end + class ErbControl < Block end class ErbIf < ErbControl - attr_reader :erb_node - - # [[HtmlNode | ErbNode | CharDataNode]] the child elements - attr_reader :elements - - # [nil | ErbElsif | ErbElse] the next clause in the chain - attr_reader :consequent - - def initialize(erb_node:, elements:, consequent:) - super(erb_node: erb_node) - @elements = elements - @consequent = consequent - end - + # opening: ErbNode + # elements: [[HtmlNode | ErbNode | CharDataNode]] + # closing: [nil | ErbElsif | ErbElse] def accept(visitor) visitor.visit_erb_if(self) end - - def child_nodes - elements - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { - erb_node: erb_node, - elements: elements, - consequent: consequent, - location: location - } - end end class ErbUnless < ErbIf + # opening: ErbNode + # elements: [[HtmlNode | ErbNode | CharDataNode]] + # closing: [nil | ErbElsif | ErbElse] + def accept(visitor) + visitor.visit_erb_if(self) + end end class ErbElsif < ErbIf + def accept(visitor) + visitor.visit_erb_if(self) + end end class ErbElse < ErbIf + def accept(visitor) + visitor.visit_erb_if(self) + end end class ErbEnd < ErbNode diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 352426e..13f5169 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -368,7 +368,7 @@ def parse_html_opening_tag ) end - def parse_html_closing_tag + def parse_html_closing opening = consume(:slash_open) name = consume(:name) closing = consume(:close) @@ -382,39 +382,34 @@ def parse_html_closing_tag end def parse_html_element - opening_tag = parse_html_opening_tag + opening = parse_html_opening_tag - if opening_tag.closing.value == ">" - content = many { parse_any_tag } - closing_tag = maybe { parse_html_closing_tag } + if opening.closing.value == ">" + elements = many { parse_any_tag } + closing = maybe { parse_html_closing } - if closing_tag.nil? + if closing.nil? raise( ParseError, - "Missing closing tag for <#{opening_tag.name.value}> at #{opening_tag.location}" + "Missing closing tag for <#{opening.name.value}> at #{opening.location}" ) end - if closing_tag.name.value != opening_tag.name.value + if closing.name.value != opening.name.value raise( ParseError, - "Expected closing tag for <#{opening_tag.name.value}> but got <#{closing_tag.name.value}> at #{closing_tag.location}" + "Expected closing tag for <#{opening.name.value}> but got <#{closing.name.value}> at #{closing.location}" ) end HtmlNode.new( - opening_tag: opening_tag, - content: content, - closing_tag: closing_tag, - location: opening_tag.location.to(closing_tag.location) + opening: opening, + elements: elements, + closing: closing, + location: opening.location.to(closing.location) ) else - HtmlNode.new( - opening_tag: opening_tag, - content: nil, - closing_tag: nil, - location: opening_tag.location - ) + HtmlNode.new(opening: opening, location: opening.location) end end @@ -433,18 +428,25 @@ def parse_erb_if(erb_node) case erb_node.keyword.type when :erb_if - ErbIf.new(erb_node: erb_node, elements: elements, consequent: erb_tag) + ErbIf.new( + opening: erb_node, + elements: elements, + closing: erb_tag, + location: erb_node.location.to(erb_tag.location) + ) when :erb_unless ErbUnless.new( - erb_node: erb_node, + opening: erb_node, elements: elements, - consequent: erb_tag + closing: erb_tag, + location: erb_node.location.to(erb_tag.location) ) when :erb_elsif ErbElsif.new( - erb_node: erb_node, + opening: erb_node, elements: elements, - consequent: erb_tag + closing: erb_tag, + location: erb_node.location.to(erb_tag.location) ) end end @@ -461,7 +463,12 @@ def parse_erb_else(erb_node) ) end - ErbElse.new(erb_node: erb_node, elements: elements, consequent: erb_end) + ErbElse.new( + opening: erb_node, + elements: elements, + closing: erb_end, + location: erb_node.location.to(erb_end.location) + ) end def parse_erb_end(erb_node) @@ -474,12 +481,6 @@ def parse_erb_end(erb_node) ) end - def parse_ruby_or_string(content) - SyntaxTree.parse(content).statements - rescue SyntaxTree::Parser::ParseError - content - end - def parse_erb_tag opening_tag = consume(:erb_open) keyword = @@ -526,9 +527,10 @@ def parse_erb_tag end ErbBlock.new( - erb_node: erb_node, + opening: erb_node, elements: elements, - consequent: erb_end + closing: erb_end, + location: erb_node.location.to(erb_end.location) ) else erb_node diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index 336c469..e7c2f12 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -19,6 +19,10 @@ def visit_document(node) visit_node("document", node) end + def visit_block(node) + visit_node(node.class.to_s, node) + end + # Visit an HtmlNode. def visit_html(node) visit_node("html", node) @@ -63,10 +67,10 @@ def visit_erb_block(node) q.text("(erb_block") q.nest(2) do q.breakable - q.seplist(node.child_nodes) { |child_node| visit(child_node) } + q.seplist(node.elements) { |child_node| visit(child_node) } end q.breakable - visit(node.consequent) + visit(node.closing) q.breakable("") q.text(")") end @@ -75,13 +79,13 @@ def visit_erb_block(node) def visit_erb_if(node, key = "erb_if") q.group do q.text("(") - visit(node.keyword) if node.keyword + visit(node.opening) q.nest(2) do q.breakable() q.seplist(node.child_nodes) { |child_node| visit(child_node) } end q.breakable - visit(node.consequent) + visit(node.closing) q.breakable("") q.text(")") end @@ -96,11 +100,6 @@ def visit_erb_content(node) q.text(node.value) end - # Visit a Reference node. - def visit_reference(node) - visit_node("reference", node) - end - # Visit an Attribute node. def visit_attribute(node) visit_node("attribute", node) diff --git a/lib/syntax_tree/erb/visitor.rb b/lib/syntax_tree/erb/visitor.rb index 0652b4e..6603f91 100644 --- a/lib/syntax_tree/erb/visitor.rb +++ b/lib/syntax_tree/erb/visitor.rb @@ -38,9 +38,6 @@ def visit_child_nodes(node) # Visit an HtmlNode::ClosingTag node. alias visit_closing_tag visit_child_nodes - # Visit a Reference node. - alias visit_reference visit_child_nodes - # Visit an Attribute node. alias visit_attribute visit_child_nodes diff --git a/test/erb_test.rb b/test/erb_test.rb index 576341d..bfe01c3 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -3,7 +3,7 @@ require "test_helper" module SyntaxTree - class ErbTest < TestCase + class ErbTest < Minitest::Test def test_empty_file parsed = ERB.parse("") assert_instance_of(SyntaxTree::ERB::Document, parsed) @@ -39,5 +39,14 @@ def test_long_if_statement assert_equal(source, formatted) assert_equal(source, formatted_again) end + + def test_text_erb_text + assert_equal( + ERB.format( + "
This is some text <%= variable %> and the special value after
" + ), + "
\n This is some text\n <%= variable %>\n and the special value after\n
\n" + ) + end end end diff --git a/test/fixture/block_formatted.html.erb b/test/fixture/block_formatted.html.erb index bed1d7f..0a434af 100644 --- a/test/fixture/block_formatted.html.erb +++ b/test/fixture/block_formatted.html.erb @@ -11,7 +11,9 @@ <% end %> <%= link_to(dont_replace, what_to_do, class: "do |what,bad|") do |hello| %> - Should allow to use the word do in the code + + Should allow to use the word do in the code + <% end %> <%= this_is_not_a_do_block_do %> diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb index c1fcec6..23c8ba0 100644 --- a/test/fixture/erb_syntax_formatted.html.erb +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -20,7 +20,9 @@ <%- end %> <%= link_to(link, text) do -%> -

Cool

+

+ Cool +

<%- end %> <%= t( diff --git a/test/fixture/if_statements_formatted.html.erb b/test/fixture/if_statements_formatted.html.erb index c918cd1..aa63505 100644 --- a/test/fixture/if_statements_formatted.html.erb +++ b/test/fixture/if_statements_formatted.html.erb @@ -1,12 +1,20 @@ <% if this %> -

that

+

+ that +

<% elsif that %> -

this

+

+ this +

<% if nested_this %> -

this

+

+ this +

<% end %> <% else %> -

else

+

+ else +

<% end %> <%= what if this %>

diff --git a/test/fixture/layout_formatted.html.erb b/test/fixture/layout_formatted.html.erb index c5ac740..660aefc 100644 --- a/test/fixture/layout_formatted.html.erb +++ b/test/fixture/layout_formatted.html.erb @@ -6,13 +6,13 @@ name="viewport" content="width=device-width, initial-scale=1 maximum-scale=1.0, user-scalable=no" /> - - - <%= full_title(t("general.title"), yield(:title)) %> + + <%= full_title(t("general.title"), yield(:title)) %> + <%= stylesheet_link_tag( "application", media: "all", @@ -41,7 +41,9 @@

-
<%= render("footer") %>
+
+ <%= render("footer") %> +
diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/vue_components_formatted.html.erb index e0516b2..91b3039 100644 --- a/test/fixture/vue_components_formatted.html.erb +++ b/test/fixture/vue_components_formatted.html.erb @@ -9,6 +9,8 @@ - + diff --git a/test/formatting_test.rb b/test/formatting_test.rb index e87f612..5c41788 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -3,7 +3,7 @@ require "test_helper" module SyntaxTree - class FormattingTest < TestCase + class FormattingTest < Minitest::Test def test_block assert_formatting("block") end @@ -27,5 +27,28 @@ def test_vue_components def test_layout assert_formatting("layout") end + + private + + def assert_formatting(name) + directory = File.expand_path("fixture", __dir__) + unformatted_file = File.join(directory, "#{name}_unformatted.html.erb") + formatted_file = File.join(directory, "#{name}_formatted.html.erb") + + expected = SyntaxTree::ERB.read(formatted_file) + formatted = SyntaxTree::ERB.format(SyntaxTree::ERB.read(unformatted_file)) + + if (expected != formatted) + puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") + Dir.mkdir("./tmp") unless Dir.exist?("./tmp") + File.write("./tmp/#{name}_failed.html.erb", formatted) + end + + assert_equal(formatted, expected) + + # Check that pretty_print works + output = SyntaxTree::ERB.parse(expected).pretty_inspect + refute_predicate(output, :empty?) + end end end diff --git a/test/html_test.rb b/test/html_test.rb index 46762ad..3ec9a43 100644 --- a/test/html_test.rb +++ b/test/html_test.rb @@ -55,10 +55,10 @@ def test_html_within_quotes assert_equal(1, elements.size) assert_instance_of(SyntaxTree::ERB::HtmlNode, elements.first) - content = elements.first.content + elements = elements.first.elements - assert_equal("This is our text \"", content.first.value.value) - assert_equal("\"", content.last.value.value) + assert_equal("This is our text \"", elements.first.value.value) + assert_equal("\"", elements.last.value.value) end def test_html_tag_names @@ -80,14 +80,14 @@ def test_html_attribute_without_quotes assert_equal(1, elements.size) assert_instance_of(SyntaxTree::ERB::HtmlNode, elements.first) - assert_equal(1, elements.first.opening_tag.attributes.size) + assert_equal(1, elements.first.opening.attributes.size) - attribute = elements.first.opening_tag.attributes.first + attribute = elements.first.opening.attributes.first assert_equal("class", attribute.key.value) assert_equal("card", attribute.value.contents.first.value) formatted = ERB.format(source) - assert_equal("
Hello World
\n", formatted) + assert_equal("
\n Hello World\n
\n", formatted) end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 0b24995..365ca3c 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -7,22 +7,3 @@ require "syntax_tree/erb" require "minitest/autorun" - -class TestCase < Minitest::Test - def assert_formatting(name) - directory = File.expand_path("fixture", __dir__) - unformatted_file = File.join(directory, "#{name}_unformatted.html.erb") - formatted_file = File.join(directory, "#{name}_formatted.html.erb") - - expected = SyntaxTree::ERB.read(formatted_file) - formatted = SyntaxTree::ERB.format(SyntaxTree::ERB.read(unformatted_file)) - - if (expected != formatted) - puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") - Dir.mkdir("./tmp") unless Dir.exist?("./tmp") - File.write("./tmp/#{name}_failed.html.erb", formatted) - end - - assert_equal(formatted, expected) - end -end From 9c929ab08d220a77dd98ca65be25213a1c537d50 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 28 May 2023 22:34:48 +0200 Subject: [PATCH 26/73] Changes print width if erb tag has IfNode (#23) --- lib/syntax_tree/erb/format.rb | 16 +++++++++++++++- lib/syntax_tree/erb/nodes.rb | 8 +------- test/erb_test.rb | 7 +++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index fb6cc54..b1e3b66 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -101,7 +101,7 @@ def visit_erb_content(node) node.value.split("\n") else formatter = - SyntaxTree::Formatter.new("", [], SyntaxTree::ERB::MAX_WIDTH) + SyntaxTree::Formatter.new("", [], erb_print_width(node.value)) formatter.format(node.value.statements) formatter.flush formatter.output.join.split("\n") @@ -194,6 +194,20 @@ def visit_doctype(node) visit(node.closing) end end + + def erb_print_width(syntax_tree) + syntax_tree => SyntaxTree::Program[ + statements: SyntaxTree::Statements[ + body: [SyntaxTree::IfNode => if_node] + ] + ] + + # Set the width to maximum if we have an IfNode, + # we cannot format them purely with SyntaxTree because the ERB-syntax will be unparseable. + if_node.nil? ? SyntaxTree::ERB::MAX_WIDTH : 999_999 + rescue NoMatchingPatternError => error + SyntaxTree::ERB::MAX_WIDTH + end end end end diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 7b8a621..b6a1bb8 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -333,13 +333,7 @@ class ErbContent < Node def initialize(value:) @unparsed_value = value begin - # We cannot handle IfNode inside a ErbContent - @value = - if SyntaxTree.search(value, "IfNode").any? - value&.lstrip&.rstrip - else - SyntaxTree.parse(value) - end + @value = SyntaxTree.parse(value) rescue SyntaxTree::Parser::ParseError # Removes leading and trailing whitespace @value = value&.lstrip&.rstrip diff --git a/test/erb_test.rb b/test/erb_test.rb index bfe01c3..33dd87b 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -31,13 +31,16 @@ def test_erb_code_with_non_ascii def test_long_if_statement source = + "<%=number_to_percentage(@reports&.first&.stability*100,precision: 1) if @reports&.first %>\n" + expected = "<%= number_to_percentage(@reports&.first&.stability * 100, precision: 1) if @reports&.first %>\n" + # With bad formatting, it is not parseable twice formatted = ERB.format(source) formatted_again = ERB.format(formatted) - assert_equal(source, formatted) - assert_equal(source, formatted_again) + assert_equal(expected, formatted) + assert_equal(expected, formatted_again) end def test_text_erb_text From 3be32029f56f58d51f62ba002fab71f5a55f2977 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Thu, 1 Jun 2023 08:18:22 +0200 Subject: [PATCH 27/73] Fixes handling of comments in ERB (#24) - Every time comments are formatted, another space was added to the end of it. --- lib/syntax_tree/erb/nodes.rb | 2 +- test/erb_test.rb | 10 ++++++++++ test/formatting_test.rb | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index b6a1bb8..c85ce5a 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -333,7 +333,7 @@ class ErbContent < Node def initialize(value:) @unparsed_value = value begin - @value = SyntaxTree.parse(value) + @value = SyntaxTree.parse(value.strip) rescue SyntaxTree::Parser::ParseError # Removes leading and trailing whitespace @value = value&.lstrip&.rstrip diff --git a/test/erb_test.rb b/test/erb_test.rb index 33dd87b..2162044 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -51,5 +51,15 @@ def test_text_erb_text "
\n This is some text\n <%= variable %>\n and the special value after\n
\n" ) end + + def test_erb_with_comment + source = "<%= what # This is a comment %>\n" + + formatted_once = ERB.format(source) + formatted_twice = ERB.format(formatted_once) + + assert_equal(source, formatted_once) + assert_equal(source, formatted_twice) + end end end diff --git a/test/formatting_test.rb b/test/formatting_test.rb index 5c41788..ca90615 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -37,6 +37,7 @@ def assert_formatting(name) expected = SyntaxTree::ERB.read(formatted_file) formatted = SyntaxTree::ERB.format(SyntaxTree::ERB.read(unformatted_file)) + formatted_twice = SyntaxTree::ERB.format(formatted) if (expected != formatted) puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") @@ -45,6 +46,7 @@ def assert_formatting(name) end assert_equal(formatted, expected) + assert_equal(formatted_twice, expected) # Check that pretty_print works output = SyntaxTree::ERB.parse(expected).pretty_inspect From 12b5e01772780eb05a2e8a0bcc54a38f813fe2b9 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sat, 3 Jun 2023 21:37:08 +0200 Subject: [PATCH 28/73] Handles formatting of IfOp (#25) - Changes the maximum width for IfOp (ternary) as well as IfNode. --- check_erb_parse.rb | 6 ++++-- lib/syntax_tree/erb/format.rb | 6 +++--- test/erb_test.rb | 4 ++-- test/fixture/if_statements_formatted.html.erb | 2 ++ test/fixture/if_statements_unformatted.html.erb | 2 ++ test/formatting_test.rb | 3 ++- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/check_erb_parse.rb b/check_erb_parse.rb index 13cefe4..36a7901 100644 --- a/check_erb_parse.rb +++ b/check_erb_parse.rb @@ -9,9 +9,11 @@ .each do |file| puts("Processing #{file}") begin - SyntaxTree::ERB.parse(SyntaxTree::ERB.read(file)) + source = SyntaxTree::ERB.read(file) + SyntaxTree::ERB.parse(source) + SyntaxTree::ERB.format(source) rescue => exception - failures << {file: file, message: exception.message} + failures << { file: file, message: exception.message } end end diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index b1e3b66..3487dbd 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -198,14 +198,14 @@ def visit_doctype(node) def erb_print_width(syntax_tree) syntax_tree => SyntaxTree::Program[ statements: SyntaxTree::Statements[ - body: [SyntaxTree::IfNode => if_node] + body: [SyntaxTree::IfNode | SyntaxTree::IfOp => if_node] ] ] - # Set the width to maximum if we have an IfNode, + # Set the width to maximum if we have an IfNode or IfOp, # we cannot format them purely with SyntaxTree because the ERB-syntax will be unparseable. if_node.nil? ? SyntaxTree::ERB::MAX_WIDTH : 999_999 - rescue NoMatchingPatternError => error + rescue NoMatchingPatternError SyntaxTree::ERB::MAX_WIDTH end end diff --git a/test/erb_test.rb b/test/erb_test.rb index 2162044..f7da269 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -31,9 +31,9 @@ def test_erb_code_with_non_ascii def test_long_if_statement source = - "<%=number_to_percentage(@reports&.first&.stability*100,precision: 1) if @reports&.first %>\n" + "<%=number_to_percentage(@reports&.first&.stability*100,precision: 1) if @reports&.first&.other&.stronger&.longer %>\n" expected = - "<%= number_to_percentage(@reports&.first&.stability * 100, precision: 1) if @reports&.first %>\n" + "<%= number_to_percentage(@reports&.first&.stability * 100, precision: 1) if @reports&.first&.other&.stronger&.longer %>\n" # With bad formatting, it is not parseable twice formatted = ERB.format(source) diff --git a/test/fixture/if_statements_formatted.html.erb b/test/fixture/if_statements_formatted.html.erb index aa63505..44d8573 100644 --- a/test/fixture/if_statements_formatted.html.erb +++ b/test/fixture/if_statements_formatted.html.erb @@ -26,3 +26,5 @@ Kanske <% end %>

+ +<%= @model ? @model.name : t("views.more_than_80_characters_long_row.categories.shared.version.default") %> diff --git a/test/fixture/if_statements_unformatted.html.erb b/test/fixture/if_statements_unformatted.html.erb index 41610a7..e0d28b4 100644 --- a/test/fixture/if_statements_unformatted.html.erb +++ b/test/fixture/if_statements_unformatted.html.erb @@ -4,3 +4,5 @@

<% unless what %> Ja<% elsif allowed? %> Nej<% else %>Kanske <% end %>

+ +<%= @model ? @model.name : t("views.more_than_80_characters_long_row.categories.shared.version.default") %> diff --git a/test/formatting_test.rb b/test/formatting_test.rb index ca90615..0eae118 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -37,7 +37,6 @@ def assert_formatting(name) expected = SyntaxTree::ERB.read(formatted_file) formatted = SyntaxTree::ERB.format(SyntaxTree::ERB.read(unformatted_file)) - formatted_twice = SyntaxTree::ERB.format(formatted) if (expected != formatted) puts("Failed to format #{name}, see ./tmp/#{name}_failed.html.erb") @@ -46,6 +45,8 @@ def assert_formatting(name) end assert_equal(formatted, expected) + + formatted_twice = SyntaxTree::ERB.format(formatted) assert_equal(formatted_twice, expected) # Check that pretty_print works From f310a7b6fa5b4f5a39d3c0998b6c7beb8480af48 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 4 Jun 2023 11:48:18 +0200 Subject: [PATCH 29/73] Updates README (#26) --- README.md | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 54ed91b..c813084 100644 --- a/README.md +++ b/README.md @@ -4,35 +4,40 @@ [Syntax Tree](https://github.com/ruby-syntax-tree/syntax_tree) support for ERB. -## Work in progress! - -This is not ready for production use just yet, still need to work on: - -- Comments -- Blocks using `do` -- Blank lines -- Probably more - Currently handles -- ERB tags with and without output -- ERB tags inside strings -- HTML tags with attributes -- HTML tags with and without closing tags -- ERB `if`, `elsif` and `else` statements +- ERB + - Tags with and without output + - Tags inside strings + - `if`, `elsif`, `else` and `unless` statements + - blocks + - comments + - Formatting of the ruby-code is done by `syntax_tree` +- HTML + - Tags with attributes + - Tags with and without closing tags + - Comments - Text output -- Formatting the ruby code inside the ERB tags (using syntax_tree itself) + +## Unhandled cases + +- `case` statements +- Create issue if you find more with a minimal example ## Installation Add this line to your application's Gemfile: ```ruby -gem github: "davidwessman/syntax_tree-erb" +gem "syntax_tree-erb", github: "davidwessman/syntax_tree-erb", require: false ``` ## Usage +```sh +bundle exec stree --plugins=erb "./**/*.html.erb" +``` + From code: ```ruby From 743db1e1d4601046bd42dc68ed11b547c6b0260f Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 4 Jun 2023 11:54:42 +0200 Subject: [PATCH 30/73] Update issue templates (#27) --- .github/ISSUE_TEMPLATE/formatting-report.md | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/formatting-report.md diff --git a/.github/ISSUE_TEMPLATE/formatting-report.md b/.github/ISSUE_TEMPLATE/formatting-report.md new file mode 100644 index 0000000..c414a91 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/formatting-report.md @@ -0,0 +1,37 @@ +--- +name: Formatting report +about: Create a report to help us improve +title: "[Formatting] " +labels: bug +assignees: davidwessman + +--- + +
+ ERB-snippet + ``` +
+ +
+ Expected formatting + ``` +
+
+ Actual formatting + ``` +
+ +## Comment + +## Versions +syntax_tree: +syntax_tree-erb: From 4cad3d7485ee583b4938ce26a5cbe89b84414033 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 4 Jun 2023 14:40:25 +0200 Subject: [PATCH 31/73] Updates README and gemspec (#29) --- Gemfile.lock | 2 +- README.md | 34 +++++++++++++++++++++++++++++++--- syntax_tree-erb.gemspec | 2 +- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3963b8d..c032d65 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: syntax_tree-erb (0.0.1) - prettier_print (>= 1.0.2) + prettier_print (>= 1.2.0) syntax_tree (>= 6.1.1) GEM diff --git a/README.md b/README.md index c813084..4518e60 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# SyntaxTree::XML +# SyntaxTree::ERB [![Build Status](https://github.com/davidwessman/syntax_tree-erb/actions/workflows/main.yml/badge.svg)](https://github.com/davidwessman/syntax_tree-erb/actions/workflows/main.yml) @@ -17,12 +17,13 @@ Currently handles - Tags with attributes - Tags with and without closing tags - Comments +- Vue + - Attributes, events and slots using `:`, `@` and `#` respectively - Text output ## Unhandled cases -- `case` statements -- Create issue if you find more with a minimal example +- Please add to this pinned issue (https://github.com/davidwessman/syntax_tree-erb/issues/28) or create a separate issue if you encounter formatting or parsing errors. ## Installation @@ -47,6 +48,33 @@ pp SyntaxTree::ERB.parse(source) # print out the AST puts SyntaxTree::ERB.format(source) # format the AST ``` +## List all parsing errors + +In order to get a list of all parsing errors (which needs to be fixed before the formatting works), this script can be used: + +```ruby +#!/bin/ruby + +require "syntax_tree/erb" + +failures = [] + +Dir + .glob("./app/**/*.html.erb") + .each do |file| + puts("Processing #{file}") + begin + source = SyntaxTree::ERB.read(file) + SyntaxTree::ERB.parse(source) + SyntaxTree::ERB.format(source) + rescue => exception + failures << { file: file, message: exception.message } + end + end + +puts failures +``` + ## Contributing Bug reports and pull requests are welcome on GitHub at https://github.com/davidwessman/syntax_tree-erb. diff --git a/syntax_tree-erb.gemspec b/syntax_tree-erb.gemspec index 3165e61..e3b248d 100644 --- a/syntax_tree-erb.gemspec +++ b/syntax_tree-erb.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] - spec.add_dependency "prettier_print", ">= 1.0.2" + spec.add_dependency "prettier_print", ">= 1.2.0" spec.add_dependency "syntax_tree", ">= 6.1.1" spec.add_development_dependency "bundler" From b18b871dc317fc47de27b378fa53f06b2c9fd91e Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 4 Jun 2023 14:43:29 +0200 Subject: [PATCH 32/73] Update issue templates --- .github/ISSUE_TEMPLATE/general.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/general.md diff --git a/.github/ISSUE_TEMPLATE/general.md b/.github/ISSUE_TEMPLATE/general.md new file mode 100644 index 0000000..97111a4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.md @@ -0,0 +1,10 @@ +--- +name: General +about: General issues +title: '' +labels: '' +assignees: '' + +--- + + From 8028145d1b5817b0ea64bdff8cfc1be7db82c1f5 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Tue, 6 Jun 2023 16:54:24 +0200 Subject: [PATCH 33/73] Handles Alpine.js syntax (#31) - Could not parse `
` because of the colon --- lib/syntax_tree/erb/parser.rb | 2 +- ...> javascript_frameworks_formatted.html.erb} | 18 ++++++++++++++++++ ...javascript_frameworks_unformatted.html.erb} | 4 ++++ test/formatting_test.rb | 4 ++-- 4 files changed, 25 insertions(+), 3 deletions(-) rename test/fixture/{vue_components_formatted.html.erb => javascript_frameworks_formatted.html.erb} (53%) rename test/fixture/{vue_components_unformatted.html.erb => javascript_frameworks_unformatted.html.erb} (53%) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 13f5169..9d1b2fd 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -244,7 +244,7 @@ def make_tokens # an equals sign # = enum.yield :equals, $&, index, line - when /\A[@:#]*[\w\.\-\_]+\b/ + when /\A[@#]*[:\w\.\-\_]+\b/ # a name for an element or an attribute # strong, vue-component-kebab, VueComponentPascal # abc, #abc, @abc, :abc diff --git a/test/fixture/vue_components_formatted.html.erb b/test/fixture/javascript_frameworks_formatted.html.erb similarity index 53% rename from test/fixture/vue_components_formatted.html.erb rename to test/fixture/javascript_frameworks_formatted.html.erb index 91b3039..99dbd99 100644 --- a/test/fixture/vue_components_formatted.html.erb +++ b/test/fixture/javascript_frameworks_formatted.html.erb @@ -1,3 +1,4 @@ +
+ + +
+ + +

+ Hello +

+ +
diff --git a/test/fixture/vue_components_unformatted.html.erb b/test/fixture/javascript_frameworks_unformatted.html.erb similarity index 53% rename from test/fixture/vue_components_unformatted.html.erb rename to test/fixture/javascript_frameworks_unformatted.html.erb index 363ccf0..03d4528 100644 --- a/test/fixture/vue_components_unformatted.html.erb +++ b/test/fixture/javascript_frameworks_unformatted.html.erb @@ -1,3 +1,7 @@ +
" boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code">
+ +
+

Hello

diff --git a/test/formatting_test.rb b/test/formatting_test.rb index 0eae118..cabf326 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -20,8 +20,8 @@ def test_if_statements assert_formatting("if_statements") end - def test_vue_components - assert_formatting("vue_components") + def test_javascript_frameworks + assert_formatting("javascript_frameworks") end def test_layout From b67eb2b56082b945e0bca8f551ec6ffefa42d6bc Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 18 Jun 2023 20:42:47 +0200 Subject: [PATCH 34/73] Github Actions: Adds ruby matrix (#33) --- .github/workflows/main.yml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c8e2b32..ddcad93 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,20 +1,27 @@ name: Main on: -- push -- pull_request_target + - push + - pull_request_target jobs: ci: + strategy: + fail-fast: false + matrix: + ruby: + - "3.0" + - "3.1" + - "3.2" name: CI runs-on: ubuntu-latest env: CI: true steps: - - uses: actions/checkout@master - - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - ruby-version: '3.1' - - name: Test - run: | - bundle exec rake test - bundle exec rake stree:check + - uses: actions/checkout@master + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + ruby-version: ${{ matrix.ruby }} + - name: Test + run: | + bundle exec rake test + bundle exec rake stree:check From 15b11a563a2f64a309a88b0896255c3aed2d4ada Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 18 Jun 2023 20:46:03 +0200 Subject: [PATCH 35/73] Adds support for case-when-statements (#32) - Fixes #30 --- lib/syntax_tree/erb/format.rb | 8 +++ lib/syntax_tree/erb/nodes.rb | 18 ++++++ lib/syntax_tree/erb/parser.rb | 79 ++++++++++++++++++++++---- lib/syntax_tree/erb/pretty_print.rb | 16 ++++++ test/fixture/case_formatted.html.erb | 10 ++++ test/fixture/case_unformatted.html.erb | 7 +++ test/formatting_test.rb | 4 ++ 7 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 test/fixture/case_formatted.html.erb create mode 100644 test/fixture/case_unformatted.html.erb diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 3487dbd..f6c75c7 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -64,6 +64,14 @@ def visit_erb_else(node) visit_block(node) end + def visit_erb_case(node) + visit_block(node) + end + + def visit_erb_case_when(node) + visit_block(node) + end + # Visit an ErbNode node. def visit_erb(node) visit(node.opening_tag) diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index c85ce5a..909f8ff 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -327,6 +327,24 @@ def child_nodes alias deconstruct child_nodes end + class ErbCase < ErbControl + # opening: ErbNode + # elements: [[HtmlNode | ErbNode | CharDataNode]] + # closing: [nil | ErbCaseWhen | ErbElse | ErbEnd] + def accept(visitor) + visitor.visit_erb_case(self) + end + end + + class ErbCaseWhen < ErbControl + # opening: ErbNode + # elements: [[HtmlNode | ErbNode | CharDataNode]] + # closing: [nil | ErbCaseWhen | ErbElse | ErbEnd] + def accept(visitor) + visitor.visit_erb_case_when(self) + end + end + class ErbContent < Node attr_reader(:value, :unparsed_value) diff --git a/lib/syntax_tree/erb/parser.rb b/lib/syntax_tree/erb/parser.rb index 9d1b2fd..5259731 100644 --- a/lib/syntax_tree/erb/parser.rb +++ b/lib/syntax_tree/erb/parser.rb @@ -125,15 +125,13 @@ def make_tokens state.pop state << :erb when /\A\s*case/ - raise( - NotImplementedError, - "case statements are not implemented" - ) + enum.yield :erb_case, $&, index, line + state.pop + state << :erb when /\A\s*when/ - raise( - NotImplementedError, - "when statements are not implemented" - ) + enum.yield :erb_when, $&, index, line + state.pop + state << :erb when /\A\s*end/ enum.yield :erb_end, $&, index, line state.pop @@ -413,6 +411,44 @@ def parse_html_element end end + def parse_erb_case(erb_node) + elements = + maybe { parse_until_erb(classes: [ErbCaseWhen, ErbElse, ErbEnd]) } || + [] + + erb_tag = elements.pop + + unless erb_tag.is_a?(ErbCaseWhen) || erb_tag.is_a?(ErbElse) || + erb_tag.is_a?(ErbEnd) + raise( + ParseError, + "Found no matching erb-tag to the if-tag at #{erb_node.location}" + ) + end + + case erb_node.keyword.type + when :erb_case + ErbCase.new( + opening: erb_node, + elements: elements, + closing: erb_tag, + location: erb_node.location.to(erb_tag.location) + ) + when :erb_when + ErbCaseWhen.new( + opening: erb_node, + elements: elements, + closing: erb_tag, + location: erb_node.location.to(erb_tag.location) + ) + else + raise( + ParseError, + "Found no matching when- or else-tag to the case-tag at #{erb_node.location}" + ) + end + end + def parse_erb_if(erb_node) elements = maybe { parse_until_erb(classes: [ErbElsif, ErbElse, ErbEnd]) } || [] @@ -448,6 +484,11 @@ def parse_erb_if(erb_node) closing: erb_tag, location: erb_node.location.to(erb_tag.location) ) + else + raise( + ParseError, + "Found no matching elsif- or else-tag to the if-tag at #{erb_node.location}" + ) end end @@ -486,7 +527,8 @@ def parse_erb_tag keyword = maybe { consume(:erb_if) } || maybe { consume(:erb_unless) } || maybe { consume(:erb_elsif) } || maybe { consume(:erb_else) } || - maybe { consume(:erb_end) } + maybe { consume(:erb_end) } || maybe { consume(:erb_case) } || + maybe { consume(:erb_when) } content = parse_until_erb_close closing_tag = content.pop @@ -510,6 +552,8 @@ def parse_erb_tag case keyword&.type when :erb_if, :erb_unless, :erb_elsif parse_erb_if(erb_node) + when :erb_case, :erb_when + parse_erb_case(erb_node) when :erb_else parse_erb_else(erb_node) when :erb_end @@ -536,6 +580,17 @@ def parse_erb_tag erb_node end end + rescue MissingTokenError => error + # If we have parsed tokens that we cannot process after we parsed <%, we should throw a ParseError + # and not let it be handled by a `maybe`. + if opening_tag + raise( + ParseError, + "Could not parse ERB-tag at #{opening_tag.location}" + ) + else + raise(error) + end end def parse_until_erb_close @@ -543,8 +598,10 @@ def parse_until_erb_close loop do result = - maybe { parse_erb_do_close } || maybe { parse_erb_close } || - maybe { consume(:erb_code) } + atleast do + maybe { parse_erb_do_close } || maybe { parse_erb_close } || + maybe { consume(:erb_code) } + end items << result break if result.is_a?(ErbClose) diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index e7c2f12..2657c9a 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -91,6 +91,22 @@ def visit_erb_if(node, key = "erb_if") end end + def visit_erb_elsif(node) + visit_erb_if(node, "erb_elsif") + end + + def visit_erb_else(node) + visit_erb_if(node, "erb_else") + end + + def visit_erb_case(node) + visit_erb_if(node, "erb_case") + end + + def visit_erb_case_when(node) + visit_erb_if(node, "erb_when") + end + def visit_erb_end(node) q.text("erb_end") end diff --git a/test/fixture/case_formatted.html.erb b/test/fixture/case_formatted.html.erb new file mode 100644 index 0000000..3db1640 --- /dev/null +++ b/test/fixture/case_formatted.html.erb @@ -0,0 +1,10 @@ +<% case variable %> +<% when 1 %> + This is the first case. +<% when 2 %> + This is the second case. +<% when 3 %> + This is the third case. +<% else %> + This is the default case. +<% end %> diff --git a/test/fixture/case_unformatted.html.erb b/test/fixture/case_unformatted.html.erb new file mode 100644 index 0000000..cdb95c0 --- /dev/null +++ b/test/fixture/case_unformatted.html.erb @@ -0,0 +1,7 @@ +<%case variable%><%when 1 %>This is the first case.<% when 2 %> +This is the second case. +<% when 3%> + This is the third case. +<% else %> +This is the default case. +<% end%> diff --git a/test/formatting_test.rb b/test/formatting_test.rb index cabf326..17041a4 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -24,6 +24,10 @@ def test_javascript_frameworks assert_formatting("javascript_frameworks") end + def test_case_statements + assert_formatting("case") + end + def test_layout assert_formatting("layout") end From 927f3c45945a873464fa1394fd9a3192f7c4a5d6 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sun, 18 Jun 2023 20:48:16 +0200 Subject: [PATCH 36/73] Github Actions: Fixes triggers (#34) --- .github/workflows/main.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ddcad93..5268526 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,7 +1,12 @@ name: Main on: - - push - - pull_request_target + workflow_dispatch: + push: + branches: + - main + + pull_request: + types: [opened, synchronize, reopened] jobs: ci: strategy: From 1c2f6585278c7d4e7dad01ca50c4f2119e6b167f Mon Sep 17 00:00:00 2001 From: David Wessman Date: Thu, 22 Jun 2023 16:46:02 +0200 Subject: [PATCH 37/73] Handles ternaries as arguments (#36) - Loop recursively through all nodes and look for IfNode and IfOp to be able to format all ternaries with longer line length. - Avoids some more if-statements inside other ERB-calls. --- lib/syntax_tree/erb/format.rb | 27 ++++++++++++++++++--------- test/erb_test.rb | 10 ++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index f6c75c7..aa8c30a 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -204,17 +204,26 @@ def visit_doctype(node) end def erb_print_width(syntax_tree) - syntax_tree => SyntaxTree::Program[ - statements: SyntaxTree::Statements[ - body: [SyntaxTree::IfNode | SyntaxTree::IfOp => if_node] - ] - ] - + statements = syntax_tree.statements.body # Set the width to maximum if we have an IfNode or IfOp, # we cannot format them purely with SyntaxTree because the ERB-syntax will be unparseable. - if_node.nil? ? SyntaxTree::ERB::MAX_WIDTH : 999_999 - rescue NoMatchingPatternError - SyntaxTree::ERB::MAX_WIDTH + if statements.any? { |node| check_for_if_statement(node) } + 999_999 + else + SyntaxTree::ERB::MAX_WIDTH + end + end + + def check_for_if_statement(node) + return false if node.nil? + + if node.is_a?(SyntaxTree::IfNode) || node.is_a?(SyntaxTree::IfOp) + return true + end + + node.child_nodes.any? do |child_node| + check_for_if_statement(child_node) + end end end end diff --git a/test/erb_test.rb b/test/erb_test.rb index f7da269..dcc7ca6 100644 --- a/test/erb_test.rb +++ b/test/erb_test.rb @@ -61,5 +61,15 @@ def test_erb_with_comment assert_equal(source, formatted_once) assert_equal(source, formatted_twice) end + + def test_erb_ternary_as_argument_without_parentheses + source = + "<%= f.submit f.object.id.present? ? t('buttons.titles.save'):t('buttons.titles.create') %>" + expected = + "<%= f.submit f.object.id.present? ? t(\"buttons.titles.save\") : t(\"buttons.titles.create\") %>\n" + formatted = ERB.format(source) + + assert_equal(expected, formatted) + end end end From 36da1d2183d028ab84b01ac169b55eb963bba816 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:50:32 +0200 Subject: [PATCH 38/73] Bump minitest from 5.18.0 to 5.18.1 (#35) Bumps [minitest](https://github.com/minitest/minitest) from 5.18.0 to 5.18.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.18.0...v5.18.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c032d65..c5b7757 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,7 @@ GEM remote: https://rubygems.org/ specs: docile (1.4.0) - minitest (5.18.0) + minitest (5.18.1) prettier_print (1.2.1) rake (13.0.6) simplecov (0.22.0) From 582e694cb1e0df90b57a297ea25a0cb55f159e06 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Thu, 22 Jun 2023 16:52:13 +0200 Subject: [PATCH 39/73] Setup gem and tag v0.9.0 (#37) --- CHANGELOG.md | 10 ++++++---- Gemfile.lock | 4 ++-- README.md | 4 +++- lib/syntax_tree/erb/version.rb | 2 +- syntax_tree-erb.gemspec | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a128997..9afc318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] -## [0.0.1] - 2023-05-25 +## [0.9.0] - 2023-06-22 ### Added -- 🎉 First version based on syntax_tree-xml 🎉 +- 🎉 First version based on syntax_tree-xml 🎉. +- Can format a lot of .html.erb-syntax and works as a plugin to syntax_tree. +- This is still early and there are a lot of different weird syntaxes out there. -[unreleased]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.1.0...HEAD -[0.0.1]: https://github.com/davidwessman/syntax_tree-erb/compare/b280a...v0.1.0 +[unreleased]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.0...HEAD +[0.9.0]: https://github.com/davidwessman/syntax_tree-erb/compare/419727a73af94057ca0980733e69ac8b4d52fdf4...v0.9.0 diff --git a/Gemfile.lock b/Gemfile.lock index c5b7757..898330a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree-erb (0.0.1) + w_syntax_tree-erb (0.9.0) prettier_print (>= 1.2.0) syntax_tree (>= 6.1.1) @@ -32,7 +32,7 @@ DEPENDENCIES minitest rake simplecov - syntax_tree-erb! + w_syntax_tree-erb! BUNDLED WITH 2.4.1 diff --git a/README.md b/README.md index 4518e60..13bd369 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,11 @@ Currently handles Add this line to your application's Gemfile: ```ruby -gem "syntax_tree-erb", github: "davidwessman/syntax_tree-erb", require: false +gem "w_syntax_tree-erb", "~> 0.9", require: false ``` +> I added the `w_` prefix to avoid conflicts if there will ever be an official `syntax_tree-erb` gem. + ## Usage ```sh diff --git a/lib/syntax_tree/erb/version.rb b/lib/syntax_tree/erb/version.rb index 14e53a4..6808872 100644 --- a/lib/syntax_tree/erb/version.rb +++ b/lib/syntax_tree/erb/version.rb @@ -2,6 +2,6 @@ module SyntaxTree module ERB - VERSION = "0.0.1" + VERSION = "0.9.0" end end diff --git a/syntax_tree-erb.gemspec b/syntax_tree-erb.gemspec index e3b248d..4fb2826 100644 --- a/syntax_tree-erb.gemspec +++ b/syntax_tree-erb.gemspec @@ -3,7 +3,7 @@ require_relative "lib/syntax_tree/erb/version" Gem::Specification.new do |spec| - spec.name = "syntax_tree-erb" + spec.name = "w_syntax_tree-erb" spec.version = SyntaxTree::ERB::VERSION spec.authors = ["Kevin Newton", "David Wessman"] spec.email = %w[kddnewton@gmail.com david@wessman.co] From 8b5aca93940cab558531c9f98013d3434a625009 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Wed, 28 Jun 2023 08:12:49 +0200 Subject: [PATCH 40/73] Handles ERB-tags with multiple statements (#38) ``` <% a = 1 b = 5 %> ``` this would be concatenated to one line before. - Had to change all indentation of multiline ERB-tags to fix it. --- CHANGELOG.md | 2 + lib/syntax_tree/erb/format.rb | 61 +++++++++++--------- lib/syntax_tree/erb/nodes.rb | 15 ++++- lib/syntax_tree/erb/pretty_print.rb | 6 +- test/fixture/erb_syntax_formatted.html.erb | 5 ++ test/fixture/erb_syntax_unformatted.html.erb | 5 ++ 6 files changed, 63 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9afc318..b4dce04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +- Handle formatting of multi-line ERB-tags with more than one statement. + ## [0.9.0] - 2023-06-22 ### Added diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index aa8c30a..55594d2 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -80,8 +80,7 @@ def visit_erb(node) q.text(" ") visit(node.keyword) end - - visit(node.content) + node.content.nil? ? q.text(" ") : visit(node.content) visit(node.closing_tag) end @@ -104,29 +103,44 @@ def visit_erb_end(node) end def visit_erb_content(node) - rows = - if node.value.is_a?(String) - node.value.split("\n") - else - formatter = - SyntaxTree::Formatter.new("", [], erb_print_width(node.value)) - formatter.format(node.value.statements) - formatter.flush - formatter.output.join.split("\n") - end + if node.value.is_a?(String) + output_rows(node.value.split("\n")) + else + child_nodes = node.value&.statements&.child_nodes || [] - if rows.size > 1 - q.group do + if child_nodes.size == 1 q.text(" ") - q.seplist(rows, -> { q.breakable(" ") }) { |row| q.text(row) } + q.seplist(child_nodes, -> { q.breakable("") }) do |child_node| + format_statement(child_node) + end q.text(" ") + elsif child_nodes.size > 1 + q.indent do + q.breakable("") + q.seplist(child_nodes, -> { q.breakable("") }) do |child_node| + format_statement(child_node) + end + end + q.breakable end + end + end + + def format_statement(statement) + formatter = + SyntaxTree::Formatter.new("", [], erb_print_width(statement)) + formatter.format(statement) + formatter.flush + rows = formatter.output.join.split("\n") + + output_rows(formatter.output.join.split("\n")) + end + + def output_rows(rows) + if rows.size > 1 + q.seplist(rows, -> { q.breakable("") }) { |row| q.text(row) } elsif rows.size == 1 - q.text(" ") q.text(rows.first) - q.text(" ") - else - q.text(" ") end end @@ -203,15 +217,10 @@ def visit_doctype(node) end end - def erb_print_width(syntax_tree) - statements = syntax_tree.statements.body + def erb_print_width(node) # Set the width to maximum if we have an IfNode or IfOp, # we cannot format them purely with SyntaxTree because the ERB-syntax will be unparseable. - if statements.any? { |node| check_for_if_statement(node) } - 999_999 - else - SyntaxTree::ERB::MAX_WIDTH - end + check_for_if_statement(node) ? 999_999 : SyntaxTree::ERB::MAX_WIDTH end def check_for_if_statement(node) diff --git a/lib/syntax_tree/erb/nodes.rb b/lib/syntax_tree/erb/nodes.rb index 909f8ff..fb6645e 100644 --- a/lib/syntax_tree/erb/nodes.rb +++ b/lib/syntax_tree/erb/nodes.rb @@ -220,8 +220,19 @@ class ErbNode < Node def initialize(opening_tag:, keyword:, content:, closing_tag:, location:) @opening_tag = opening_tag - @keyword = keyword - @content = ErbContent.new(value: content.map(&:value).join) if content + # prune whitespace from keyword + @keyword = + if keyword + Token.new( + type: keyword.type, + value: keyword.value.strip, + location: keyword.location + ) + end + # Set content to nil if it is empty + content ||= [] + content = content.map(&:value).join if content.is_a?(Array) + @content = ErbContent.new(value: content) unless content.strip.empty? @closing_tag = closing_tag @location = location end diff --git a/lib/syntax_tree/erb/pretty_print.rb b/lib/syntax_tree/erb/pretty_print.rb index 2657c9a..2f99b23 100644 --- a/lib/syntax_tree/erb/pretty_print.rb +++ b/lib/syntax_tree/erb/pretty_print.rb @@ -108,12 +108,12 @@ def visit_erb_case_when(node) end def visit_erb_end(node) - q.text("erb_end") + q.pp("erb_end") end # Visit an ErbContent node. def visit_erb_content(node) - q.text(node.value) + q.pp(node.value) end # Visit an Attribute node. @@ -132,7 +132,7 @@ def visit_char_data(node) end def visit_erb_close(node) - visit_node("erb_close", node) + visit(node.closing) end def visit_erb_do_close(node) diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb index 23c8ba0..c9f7ef8 100644 --- a/test/fixture/erb_syntax_formatted.html.erb +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -36,3 +36,8 @@ end ) ) %> + +<% + assign_b ||= "b" + assign_c ||= "c" +%> diff --git a/test/fixture/erb_syntax_unformatted.html.erb b/test/fixture/erb_syntax_unformatted.html.erb index c5b0fb9..95b00ef 100644 --- a/test/fixture/erb_syntax_unformatted.html.erb +++ b/test/fixture/erb_syntax_unformatted.html.erb @@ -35,3 +35,8 @@ end ) ) %> + +<% + assign_b ||= "b" + assign_c ||= "c" +%> From b43c4d6f4f97e18f0369bc5c14167350ce9725ee Mon Sep 17 00:00:00 2001 From: David Wessman Date: Wed, 28 Jun 2023 08:19:26 +0200 Subject: [PATCH 41/73] v0.9.1 (#39) --- CHANGELOG.md | 2 ++ Gemfile.lock | 2 +- lib/syntax_tree/erb/version.rb | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4dce04..346f3ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [0.9.1] - 2023-06-28 + - Handle formatting of multi-line ERB-tags with more than one statement. ## [0.9.0] - 2023-06-22 diff --git a/Gemfile.lock b/Gemfile.lock index 898330a..40c5bdc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - w_syntax_tree-erb (0.9.0) + w_syntax_tree-erb (0.9.1) prettier_print (>= 1.2.0) syntax_tree (>= 6.1.1) diff --git a/lib/syntax_tree/erb/version.rb b/lib/syntax_tree/erb/version.rb index 6808872..09b1029 100644 --- a/lib/syntax_tree/erb/version.rb +++ b/lib/syntax_tree/erb/version.rb @@ -2,6 +2,6 @@ module SyntaxTree module ERB - VERSION = "0.9.0" + VERSION = "0.9.1" end end From 7a4c3bd5ce2dbdeffdcc8ce9bb29bb0cacda7dc5 Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 30 Jun 2023 09:04:05 +0200 Subject: [PATCH 42/73] Handle whitespace in HTML-strings with ERB (#40) --- CHANGELOG.md | 2 ++ lib/syntax_tree/erb/format.rb | 6 +++++- test/fixture/erb_syntax_formatted.html.erb | 3 +++ test/fixture/erb_syntax_unformatted.html.erb | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 346f3ff..2cb1a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +- Handle whitespace in HTML-strings using ERB-tags + ## [0.9.1] - 2023-06-28 - Handle formatting of multi-line ERB-tags with more than one statement. diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 55594d2..9813e4f 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -11,7 +11,11 @@ def initialize(q) # Visit a Token node. def visit_token(node) - q.text(node.value.strip) + if %i[text whitespace].include?(node.type) + q.text(node.value) + else + q.text(node.value.strip) + end end # Visit a Document node. diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb index c9f7ef8..44e0002 100644 --- a/test/fixture/erb_syntax_formatted.html.erb +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -41,3 +41,6 @@ assign_b ||= "b" assign_c ||= "c" %> + +
mt-<%= 5 * 5 %>"> +
diff --git a/test/fixture/erb_syntax_unformatted.html.erb b/test/fixture/erb_syntax_unformatted.html.erb index 95b00ef..a0a25e4 100644 --- a/test/fixture/erb_syntax_unformatted.html.erb +++ b/test/fixture/erb_syntax_unformatted.html.erb @@ -40,3 +40,5 @@ assign_b ||= "b" assign_c ||= "c" %> + +
mt-<%=5 * 5%>">
From 3dbc3d9043dd23bc7ca9f2c0d11f7f133719957a Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 30 Jun 2023 09:07:56 +0200 Subject: [PATCH 43/73] v0.9.2 2023-06-30 (#41) --- CHANGELOG.md | 2 ++ Gemfile.lock | 2 +- lib/syntax_tree/erb/version.rb | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb1a5a..65fc853 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [0.9.2] - 2023-06-30 + - Handle whitespace in HTML-strings using ERB-tags ## [0.9.1] - 2023-06-28 diff --git a/Gemfile.lock b/Gemfile.lock index 40c5bdc..8f5535e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - w_syntax_tree-erb (0.9.1) + w_syntax_tree-erb (0.9.2) prettier_print (>= 1.2.0) syntax_tree (>= 6.1.1) diff --git a/lib/syntax_tree/erb/version.rb b/lib/syntax_tree/erb/version.rb index 09b1029..cb22d45 100644 --- a/lib/syntax_tree/erb/version.rb +++ b/lib/syntax_tree/erb/version.rb @@ -2,6 +2,6 @@ module SyntaxTree module ERB - VERSION = "0.9.1" + VERSION = "0.9.2" end end From 98227e3729427edbde20be047e0a11c1aede630c Mon Sep 17 00:00:00 2001 From: David Wessman Date: Fri, 30 Jun 2023 12:44:13 +0200 Subject: [PATCH 44/73] Format empty html-tags on one line if possible (#42) --- CHANGELOG.md | 4 ++++ Gemfile.lock | 2 +- lib/syntax_tree/erb/format.rb | 7 ++++++- lib/syntax_tree/erb/version.rb | 2 +- test/fixture/erb_syntax_formatted.html.erb | 3 +-- test/html_test.rb | 8 ++++++++ 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65fc853..36978f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [0.9.3] - 2023-06-30 + +- Print empty html-tags on one line if possible + ## [0.9.2] - 2023-06-30 - Handle whitespace in HTML-strings using ERB-tags diff --git a/Gemfile.lock b/Gemfile.lock index 8f5535e..a01beb1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - w_syntax_tree-erb (0.9.2) + w_syntax_tree-erb (0.9.3) prettier_print (>= 1.2.0) syntax_tree (>= 6.1.1) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 9813e4f..64d91f2 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -49,7 +49,12 @@ def visit_block(node) end def visit_html(node) - visit_block(node) + # Make sure to group the tags together if there is no child nodes. + if node.elements.size == 0 + q.group { visit_block(node) } + else + visit_block(node) + end end def visit_erb_block(node) diff --git a/lib/syntax_tree/erb/version.rb b/lib/syntax_tree/erb/version.rb index cb22d45..d9398fb 100644 --- a/lib/syntax_tree/erb/version.rb +++ b/lib/syntax_tree/erb/version.rb @@ -2,6 +2,6 @@ module SyntaxTree module ERB - VERSION = "0.9.2" + VERSION = "0.9.3" end end diff --git a/test/fixture/erb_syntax_formatted.html.erb b/test/fixture/erb_syntax_formatted.html.erb index 44e0002..14e9eca 100644 --- a/test/fixture/erb_syntax_formatted.html.erb +++ b/test/fixture/erb_syntax_formatted.html.erb @@ -42,5 +42,4 @@ assign_c ||= "c" %> -
mt-<%= 5 * 5 %>"> -
+
mt-<%= 5 * 5 %>">
diff --git a/test/html_test.rb b/test/html_test.rb index 3ec9a43..1010874 100644 --- a/test/html_test.rb +++ b/test/html_test.rb @@ -89,5 +89,13 @@ def test_html_attribute_without_quotes formatted = ERB.format(source) assert_equal("
\n Hello World\n
\n", formatted) end + + def test_html_attribute_without_content + source = "\n\n" + expected = "\n" + + formatted = ERB.format(source) + assert_equal(expected, formatted) + end end end From bf18cbea1f9f0f8653a67a856f1a1766466eab4b Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sat, 1 Jul 2023 15:46:03 +0200 Subject: [PATCH 45/73] 0.9.4: Format empty HTML-tags as a group (#43) ```diff - +> ``` --- CHANGELOG.md | 14 ++++++++++++++ Gemfile.lock | 2 +- lib/syntax_tree/erb/format.rb | 5 ++++- lib/syntax_tree/erb/version.rb | 2 +- .../javascript_frameworks_formatted.html.erb | 8 ++++++-- .../javascript_frameworks_unformatted.html.erb | 3 +++ 6 files changed, 29 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36978f5..50cffd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [0.9.4] - 2023-07-01 + +- Inline even more empty HTML-tags + +```diff + +- ++> +``` + ## [0.9.3] - 2023-06-30 - Print empty html-tags on one line if possible diff --git a/Gemfile.lock b/Gemfile.lock index a01beb1..75e3eae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - w_syntax_tree-erb (0.9.3) + w_syntax_tree-erb (0.9.4) prettier_print (>= 1.2.0) syntax_tree (>= 6.1.1) diff --git a/lib/syntax_tree/erb/format.rb b/lib/syntax_tree/erb/format.rb index 64d91f2..1af613a 100644 --- a/lib/syntax_tree/erb/format.rb +++ b/lib/syntax_tree/erb/format.rb @@ -51,7 +51,10 @@ def visit_block(node) def visit_html(node) # Make sure to group the tags together if there is no child nodes. if node.elements.size == 0 - q.group { visit_block(node) } + q.group do + visit(node.opening) + visit(node.closing) + end else visit_block(node) end diff --git a/lib/syntax_tree/erb/version.rb b/lib/syntax_tree/erb/version.rb index d9398fb..bf2155c 100644 --- a/lib/syntax_tree/erb/version.rb +++ b/lib/syntax_tree/erb/version.rb @@ -2,6 +2,6 @@ module SyntaxTree module ERB - VERSION = "0.9.3" + VERSION = "0.9.4" end end diff --git a/test/fixture/javascript_frameworks_formatted.html.erb b/test/fixture/javascript_frameworks_formatted.html.erb index 99dbd99..a8b0ff8 100644 --- a/test/fixture/javascript_frameworks_formatted.html.erb +++ b/test/fixture/javascript_frameworks_formatted.html.erb @@ -6,8 +6,7 @@ boolean :value="['a', 'b']" :long-variable-name="data.item.javascript.code" - > - + >