Class: RDoc::Markup::Parser

Inherits:
Object show all
Includes:
Text
Defined in:
lib/rdoc/markup/parser.rb

Overview

A recursive-descent parser for RDoc markup.

The parser tokenizes an input string then parses the tokens into a Document. Documents can be converted into output formats by writing a visitor like RDoc::Markup::ToHTML.

The parser only handles the block-level constructs Paragraph, List, ListItem, Heading, Verbatim, BlankLine and Rule. Inline markup such as +blah+ is handled separately by RDoc::Markup::AttributeManager.

To see what markup the Parser implements read RDoc. To see how to use RDoc markup to format text in your program read RDoc::Markup.

Defined Under Namespace

Classes: Error, ParseError

Constant Summary

LIST_TOKENS =

List token types

[
  :BULLET,
  :LABEL,
  :LALPHA,
  :NOTE,
  :NUMBER,
  :UALPHA,
]

Constants included from Text

Text::TO_HTML_CHARACTERS

Instance Attribute Summary (collapse)

Class Method Summary (collapse)

Instance Method Summary (collapse)

Methods included from Text

encode_fallback, #expand_tabs, #flush_left, #markup, #normalize_comment, #strip_hashes, #strip_newlines, #strip_stars, #to_html, #wrap

Constructor Details

- (Parser) initialize

Creates a new Parser. See also ::parse



76
77
78
79
80
81
82
83
 
# File 'lib/rdoc/markup/parser.rb', line 76

def initialize
  @tokens = []
  @current_token = nil
  @debug = false

  @line = 0
  @line_pos = 0
end

Instance Attribute Details

- (Object) debug

Enables display of debugging information



47
48
49
 
# File 'lib/rdoc/markup/parser.rb', line 47

def debug
  @debug
end

- (Object) tokens (readonly)

Token accessor



52
53
54
 
# File 'lib/rdoc/markup/parser.rb', line 52

def tokens
  @tokens
end

Class Method Details

+ (Object) parse(str)

Parses str into a Document



57
58
59
60
61
62
 
# File 'lib/rdoc/markup/parser.rb', line 57

def self.parse str
  parser = new
  parser.tokenize str
  doc = RDoc::Markup::Document.new
  parser.parse doc
end

+ (Object) tokenize(str)

Returns a token stream for str, for testing



67
68
69
70
71
 
# File 'lib/rdoc/markup/parser.rb', line 67

def self.tokenize str
  parser = new
  parser.tokenize str
  parser.tokens
end

Instance Method Details

- (Object) build_heading(level)

Builds a Heading of level



88
89
90
91
92
93
94
 
# File 'lib/rdoc/markup/parser.rb', line 88

def build_heading level
  _, text, = get  # TEXT
  heading = RDoc::Markup::Heading.new level, text
  skip :NEWLINE

  heading
end

- (Object) build_list(margin)

Builds a List flush to margin



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/rdoc/markup/parser.rb', line 99

def build_list margin
  p :list_start => margin if @debug

  list = RDoc::Markup::List.new

  until @tokens.empty? do
    type, data, column, = get

    case type
    when :BULLET, :LABEL, :LALPHA, :NOTE, :NUMBER, :UALPHA then

      if column < margin || (list.type && list.type != type) then
        unget
        break
      end

      list.type = type
      peek_type, _, column, = peek_token

      case type
      when :NOTE, :LABEL then
        if peek_type == :NEWLINE then
          # description not on the same line as LABEL/NOTE
          # skip the trailing newline & any blank lines below
          while peek_type == :NEWLINE
            get
            peek_type, _, column, = peek_token
          end

          # we may be:
          #   - at end of stream
          #   - at a column < margin:
          #         [text]
          #       blah blah blah
          #   - at the same column, but with a different type of list item
          #       [text]
          #       * blah blah
          #   - at the same column, with the same type of list item
          #       [one]
          #       [two]
          # In all cases, we have an empty description.
          # In the last case only, we continue.
          if peek_type.nil? || column < margin then
            empty = 1
          elsif column == margin then
            case peek_type
            when type
              empty = 2 # continue
            when *LIST_TOKENS
              empty = 1
            else
              empty = 0
            end
          else
            empty = 0
          end

          if empty > 0 then
            item = RDoc::Markup::ListItem.new(data)
            item << RDoc::Markup::BlankLine.new
            list << item
            break if empty == 1
            next
          end
        end
      else
        data = nil
      end

      list_item = RDoc::Markup::ListItem.new data
      parse list_item, column
      list << list_item

    else
      unget
      break
    end
  end

  p :list_end => margin if @debug

  return nil if list.empty?

  list
end

- (Object) build_paragraph(margin)

Builds a Paragraph that is flush to margin



188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/rdoc/markup/parser.rb', line 188

def build_paragraph margin
  p :paragraph_start => margin if @debug

  paragraph = RDoc::Markup::Paragraph.new

  until @tokens.empty? do
    type, data, column, = get

    if type == :TEXT && column == margin then
      paragraph << data
      skip :NEWLINE
    else
      unget
      break
    end
  end

  p :paragraph_end => margin if @debug

  paragraph
end

- (Object) build_verbatim(margin)

Builds a Verbatim that is indented from margin.

The verbatim block is shifted left (the least indented lines start in column 0). Each part of the verbatim is one line of text, always terminated by a newline. Blank lines always consist of a single newline character, and there is never a single newline at the end of the verbatim.



218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
# File 'lib/rdoc/markup/parser.rb', line 218

def build_verbatim margin
  p :verbatim_begin => margin if @debug
  verbatim = RDoc::Markup::Verbatim.new

  min_indent = nil
  generate_leading_spaces = true
  line = ''

  until @tokens.empty? do
    type, data, column, = get

    if type == :NEWLINE then
      line << data
      verbatim << line
      line = ''
      generate_leading_spaces = true
      next
    end

    if column <= margin
      unget
      break
    end

    if generate_leading_spaces then
      indent = column - margin
      line << ' ' * indent
      min_indent = indent if min_indent.nil? || indent < min_indent
      generate_leading_spaces = false
    end

    case type
    when :HEADER then
      line << '=' * data
      _, _, peek_column, = peek_token
      peek_column ||= column + data
      indent = peek_column - column - data
      line << ' ' * indent
    when :RULE then
      width = 2 + data
      line << '-' * width
      _, _, peek_column, = peek_token
      peek_column ||= column + width
      indent = peek_column - column - width
      line << ' ' * indent
    when :TEXT then
      line << data
    else # *LIST_TOKENS
      list_marker = case type
                    when :BULLET then data
                    when :LABEL  then "[#{data}]"
                    when :NOTE   then "#{data}::"
                    else # :LALPHA, :NUMBER, :UALPHA
                      "#{data}."
                    end
      line << list_marker
      peek_type, _, peek_column = peek_token
      unless peek_type == :NEWLINE then
        peek_column ||= column + list_marker.length
        indent = peek_column - column - list_marker.length
        line << ' ' * indent
      end
    end

  end

  verbatim << line << "\n" unless line.empty?
  verbatim.parts.each { |p| p.slice!(0, min_indent) unless p == "\n" } if min_indent > 0
  verbatim.normalize

  p :verbatim_end => margin if @debug

  verbatim
end

- (Object) get

Pulls the next token from the stream.



296
297
298
299
300
 
# File 'lib/rdoc/markup/parser.rb', line 296

def get
  @current_token = @tokens.shift
  p :get => @current_token if @debug
  @current_token
end

- (Object) parse(parent, indent = 0)

Parses the tokens into an array of RDoc::Markup::XXX objects, and appends them to the passed parent RDoc::Markup::YYY object.

Exits at the end of the token stream, or when it encounters a token in a column less than indent (unless it is a NEWLINE).

Returns parent.



311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
 
# File 'lib/rdoc/markup/parser.rb', line 311

def parse parent, indent = 0
  p :parse_start => indent if @debug

  until @tokens.empty? do
    type, data, column, = get

    if type == :NEWLINE then
      # trailing newlines are skipped below, so this is a blank line
      parent << RDoc::Markup::BlankLine.new
      skip :NEWLINE, false
      next
    end

    # indentation change: break or verbatim
    if column < indent then
      unget
      break
    elsif column > indent then
      unget
      parent << build_verbatim(indent)
      next
    end

    # indentation is the same
    case type
    when :HEADER then
      parent << build_heading(data)
    when :RULE then
      parent << RDoc::Markup::Rule.new(data)
      skip :NEWLINE
    when :TEXT then
      unget
      parent << build_paragraph(indent)
    when *LIST_TOKENS then
      unget
      parent << build_list(indent)
    else
      type, data, column, line = @current_token
      raise ParseError, "Unhandled token #{type} (#{data.inspect}) at #{line}:#{column}"
    end
  end

  p :parse_end => indent if @debug

  parent

end

- (Object) peek_token

Returns the next token on the stream without modifying the stream



362
363
364
365
366
 
# File 'lib/rdoc/markup/parser.rb', line 362

def peek_token
  token = @tokens.first || []
  p :peek => token if @debug
  token
end

- (Object) skip(token_type, error = true)

Skips the next token if its type is token_type.

Optionally raises an error if the next token is not of the expected type.

Raises:



373
374
375
376
377
378
379
 
# File 'lib/rdoc/markup/parser.rb', line 373

def skip token_type, error = true
  type, = get
  return unless type # end of stream
  return @current_token if token_type == type
  unget
  raise ParseError, "expected #{token_type} got #{@current_token.inspect}" if error
end

- (Object) token_pos(offset)

Calculates the column and line of the current token based on offset.



458
459
460
 
# File 'lib/rdoc/markup/parser.rb', line 458

def token_pos offset
  [offset - @line_pos, @line]
end

- (Object) tokenize(input)

Turns text input into a stream of tokens



384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
 
# File 'lib/rdoc/markup/parser.rb', line 384

def tokenize input
  s = StringScanner.new input

  @line = 0
  @line_pos = 0

  until s.eos? do
    pos = s.pos

    # leading spaces will be reflected by the column of the next token
    # the only thing we loose are trailing spaces at the end of the file
    next if s.scan(/ +/)

    # note: after BULLET, LABEL, etc.,
    # indent will be the column of the next non-newline token

    @tokens << case
               # [CR]LF => :NEWLINE
               when s.scan(/\r?\n/) then
                 token = [:NEWLINE, s.matched, *token_pos(pos)]
                 @line_pos = s.pos
                 @line += 1
                 token
               # === text => :HEADER then :TEXT
               when s.scan(/(=+)\s*/) then
                 level = s[1].length
                 level = 6 if level > 6
                 @tokens << [:HEADER, level, *token_pos(pos)]
                 pos = s.pos
                 s.scan(/.*/)
                 [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
               # --- (at least 3) and nothing else on the line => :RULE
               when s.scan(/(-{3,}) *$/) then
                 [:RULE, s[1].length - 2, *token_pos(pos)]
               # * or - followed by white space and text => :BULLET
               when s.scan(/([*-]) +(\S)/) then
                 s.pos -= s[2].bytesize # unget \S
                 [:BULLET, s[1], *token_pos(pos)]
               # A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
               when s.scan(/([a-z]|\d+)\. +(\S)/i) then
                 # FIXME if tab(s), the column will be wrong
                 # either support tabs everywhere by first expanding them to
                 # spaces, or assume that they will have been replaced
                 # before (and provide a check for that at least in debug
                 # mode)
                 list_label = s[1]
                 s.pos -= s[2].bytesize # unget \S
                 list_type =
                   case list_label
                   when /[a-z]/ then :LALPHA
                   when /[A-Z]/ then :UALPHA
                   when /\d/    then :NUMBER
                   else
                     raise ParseError, "BUG token #{list_label}"
                   end
                 [list_type, list_label, *token_pos(pos)]
               # [text] followed by spaces or end of line => :LABEL
               when s.scan(/\[(.*?)\]( +|$)/) then
                 [:LABEL, s[1], *token_pos(pos)]
               # text:: followed by spaces or end of line => :NOTE
               when s.scan(/(.*?)::( +|$)/) then
                 [:NOTE, s[1], *token_pos(pos)]
               # anything else: :TEXT
               else s.scan(/.*/)
                 [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
               end
  end

  self
end

- (Object) unget

Returns the current token to the token stream

Raises:



465
466
467
468
469
470
 
# File 'lib/rdoc/markup/parser.rb', line 465

def unget
  token = @current_token
  p :unget => token if @debug
  raise Error, 'too many #ungets' if token == @tokens.first
  @tokens.unshift token if token
end