SWAMPy

✎ CARVE FLESH

EDITING: clean_document.rb

# frozen_string_literal: true

module SyntaxSuggest
  # Parses and sanitizes source into a lexically aware document
  #
  # Internally the document is represented by an array with each
  # index containing a CodeLine correlating to a line from the source code.
  #
  # There are three main phases in the algorithm:
  #
  # 1. Sanitize/format input source
  # 2. Search for invalid blocks
  # 3. Format invalid blocks into something meaninful
  #
  # This class handles the first part.
  #
  # The reason this class exists is to format input source
  # for better/easier/cleaner exploration.
  #
  # The CodeSearch class operates at the line level so
  # we must be careful to not introduce lines that look
  # valid by themselves, but when removed will trigger syntax errors
  # or strange behavior.
  #
  # ## Join Trailing slashes
  #
  # Code with a trailing slash is logically treated as a single line:
  #
  #     1 it "code can be split" \
  #     2    "across multiple lines" do
  #
  # In this case removing line 2 would add a syntax error. We get around
  # this by internally joining the two lines into a single "line" object
  #
  # ## Logically Consecutive lines
  #
  # Code that can be broken over multiple
  # lines such as method calls are on different lines:
  #
  #     1 User.
  #     2   where(name: "schneems").
  #     3   first
  #
  # Removing line 2 can introduce a syntax error. To fix this, all lines
  # are joined into one.
  #
  # ## Heredocs
  #
  # A heredoc is an way of defining a multi-line string. They can cause many
  # problems. If left as a single line, Ripper would try to parse the contents
  # as ruby code rather than as a string. Even without this problem, we still
  # hit an issue with indentation
  #
  #    1 foo = <<~HEREDOC
  #    2  "Be yourself; everyone else is already taken.""
  #    3    ― Oscar Wilde
  #    4      puts "I look like ruby code" # but i'm still a heredoc
  #    5 HEREDOC
  #
  # If we didn't join these lines then our algorithm would think that line 4
  # is separate from the rest, has a higher indentation, then look at it first
  # and remove it.
  #
  # If the code evaluates line 5 by itself it will think line 5 is a constant,
  # remove it, and introduce a syntax errror.
  #
  # All of these problems are fixed by joining the whole heredoc into a single
  # line.
  #
  # ## Comments and whitespace
  #
  # Comments can throw off the way the lexer tells us that the line
  # logically belongs with the next line. This is valid ruby but
  # results in a different lex output than before:
  #
  #     1 User.
  #     2   where(name: "schneems").
  #     3   # Comment here
  #     4   first
  #
  # To handle this we can replace comment lines with empty lines
  # and then re-lex the source. This removal and re-lexing preserves
  # line index and document size, but generates an easier to work with
  # document.
  #
  class CleanDocument
    def initialize(source:)
      lines = clean_sweep(source: source)
      @document = CodeLine.from_source(lines.join, lines: lines)
    end

# Call all of the document "cleaners"
    # and return self
    def call
      join_trailing_slash!
      join_consecutive!
      join_heredoc!

self
    end

# Return an array of CodeLines in the
    # document
    def lines
      @document
    end

# Renders the document back to a string
    def to_s
      @document.join
    end

# Remove comments
    #
    # replace with empty newlines
    #
    #     source = <<~'EOM'
    #       # Comment 1
    #       puts "hello"
    #       # Comment 2
    #       puts "world"
    #     EOM
    #
    #     lines = CleanDocument.new(source: source).lines
    #     expect(lines[0].to_s).to eq("\n")
    #     expect(lines[1].to_s).to eq("puts "hello")
    #     expect(lines[2].to_s).to eq("\n")
    #     expect(lines[3].to_s).to eq("puts "world")
    #
    # Important: This must be done before lexing.
    #
    # After this change is made, we lex the document because
    # removing comments can change how the doc is parsed.
    #
    # For example:
    #
    #     values = LexAll.new(source: <<~EOM))
    #       User.
    #         # comment
    #         where(name: 'schneems')
    #     EOM
    #     expect(
    #       values.count {|v| v.type == :on_ignored_nl}
    #     ).to eq(1)
    #
    # After the comment is removed:
    #
    #     values = LexAll.new(source: <<~EOM))
    #       User.
    #
    #         where(name: 'schneems')
    #     EOM
    #     expect(
    #      values.count {|v| v.type == :on_ignored_nl}
    #    ).to eq(2)
    #
    def clean_sweep(source:)
      # Match comments, but not HEREDOC strings with #{variable} interpolation
      # https://rubular.com/r/HPwtW9OYxKUHXQ
      source.lines.map do |line|
        if line.match?(/^\s*#([^{].*|)$/)
          $/
        else
          line
        end
      end
    end

# Smushes all heredoc lines into one line
    #
    #     source = <<~'EOM'
    #       foo = <<~HEREDOC
    #          lol
    #          hehehe
    #       HEREDOC
    #     EOM
    #
    #     lines = CleanDocument.new(source: source).join_heredoc!.lines
    #     expect(lines[0].to_s).to eq(source)
    #     expect(lines[1].to_s).to eq("")
    def join_heredoc!
      start_index_stack = []
      heredoc_beg_end_index = []
      lines.each do |line|
        line.lex.each do |lex_value|
          case lex_value.type
          when :on_heredoc_beg
            start_index_stack << line.index
          when :on_heredoc_end
            start_index = start_index_stack.pop
            end_index = line.index
            heredoc_beg_end_index << [start_index, end_index]
          end
        end
      end

heredoc_groups = heredoc_beg_end_index.map { |start_index, end_index| @document[start_index..end_index] }

join_groups(heredoc_groups)
      self
    end

# Smushes logically "consecutive" lines
    #
    #     source = <<~'EOM'
    #       User.
    #         where(name: 'schneems').
    #         first
    #     EOM
    #
    #     lines = CleanDocument.new(source: source).join_consecutive!.lines
    #     expect(lines[0].to_s).to eq(source)
    #     expect(lines[1].to_s).to eq("")
    #
    # The one known case this doesn't handle is:
    #
    #     Ripper.lex <<~EOM
    #       a &&
    #        b ||
    #        c
    #     EOM
    #
    # For some reason this introduces `on_ignore_newline` but with BEG type
    #
    def join_consecutive!
      consecutive_groups = @document.select(&:ignore_newline_not_beg?).map do |code_line|
        take_while_including(code_line.index..-1) do |line|
          line.ignore_newline_not_beg?
        end
      end

join_groups(consecutive_groups)
      self
    end

# Join lines with a trailing slash
    #
    #     source = <<~'EOM'
    #       it "code can be split" \
    #          "across multiple lines" do
    #     EOM
    #
    #     lines = CleanDocument.new(source: source).join_consecutive!.lines
    #     expect(lines[0].to_s).to eq(source)
    #     expect(lines[1].to_s).to eq("")
    def join_trailing_slash!
      trailing_groups = @document.select(&:trailing_slash?).map do |code_line|
        take_while_including(code_line.index..-1) { |x| x.trailing_slash? }
      end
      join_groups(trailing_groups)
      self
    end

# Helper method for joining "groups" of lines
    #
    # Input is expected to be type Array<Array<CodeLine>>
    #
    # The outer array holds the various "groups" while the
    # inner array holds code lines.
    #
    # All code lines are "joined" into the first line in
    # their group.
    #
    # To preserve document size, empty lines are placed
    # in the place of the lines that were "joined"
    def join_groups(groups)
      groups.each do |lines|
        line = lines.first

# Handle the case of multiple groups in a a row
        # if one is already replaced, move on
        next if @document[line.index].empty?

# Join group into the first line
        @document[line.index] = CodeLine.new(
          lex: lines.map(&:lex).flatten,
          line: lines.join,
          index: line.index
        )

# Hide the rest of the lines
        lines[1..-1].each do |line|
          # The above lines already have newlines in them, if add more
          # then there will be double newline, use an empty line instead
          @document[line.index] = CodeLine.new(line: "", index: line.index, lex: [])
        end
      end
      self
    end

# Helper method for grabbing elements from document
    #
    # Like `take_while` except when it stops
    # iterating, it also returns the line
    # that caused it to stop
    def take_while_including(range = 0..-1)
      take_next_and_stop = false
      @document[range].take_while do |line|
        next if take_next_and_stop

take_next_and_stop = !(yield line)
        true
      end
    end
  end
end

CANCEL

Name	Type	Size	Modified	Actions
↩ ..	DIR	—	—
📁 capture/	DIR	—	2026-07-28 14:00
📄 api.rb	RB	5.2 KB	2026-04-07 14:45	EDIT
📄 around_block_scan.rb	RB	6.9 KB	2026-04-07 14:45	EDIT
📄 block_expand.rb	RB	4.9 KB	2026-04-07 14:45	EDIT
📄 capture_code_context.rb	RB	6.7 KB	2026-04-07 14:45	EDIT
📄 clean_document.rb	RB	8.7 KB	2026-04-07 14:45	EDIT
📄 cli.rb	RB	3.1 KB	2026-04-07 14:45	EDIT
📄 code_block.rb	RB	2.1 KB	2026-04-07 14:45	EDIT
📄 code_frontier.rb	RB	5.6 KB	2026-04-07 14:45	EDIT
📄 code_line.rb	RB	6.5 KB	2026-04-07 14:45	EDIT
📄 code_search.rb	RB	3.8 KB	2026-04-07 14:45	EDIT
📄 core_ext.rb	RB	3 KB	2026-04-07 14:45	EDIT
📄 display_code_with_line_numbers.rb	RB	1.8 KB	2026-04-07 14:45	EDIT
📄 display_invalid_blocks.rb	RB	1.8 KB	2026-04-07 14:45	EDIT
📄 explain_syntax.rb	RB	2.4 KB	2026-04-07 14:45	EDIT
📄 left_right_lex_count.rb	RB	4.1 KB	2026-04-07 14:45	EDIT
📄 lex_all.rb	RB	1.1 KB	2026-04-07 14:45	EDIT
📄 lex_value.rb	RB	1.5 KB	2026-04-07 14:45	EDIT
📄 parse_blocks_from_indent_line.rb	RB	1.5 KB	2026-04-07 14:45	EDIT
📄 pathname_from_message.rb	RB	1.4 KB	2026-04-07 14:45	EDIT
📄 priority_engulf_queue.rb	RB	1.3 KB	2026-04-07 14:45	EDIT
📄 priority_queue.rb	RB	2 KB	2026-04-07 14:45	EDIT
📄 ripper_errors.rb	RB	837 B	2026-04-07 14:45	EDIT
📄 scan_history.rb	RB	2.9 KB	2026-04-07 14:45	EDIT
📄 unvisited_lines.rb	RB	705 B	2026-04-07 14:45	EDIT
📄 version.rb	RB	76 B	2026-04-07 14:45	EDIT