Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change comment directive parsing #1149

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Change comment directive parsing #1149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions lib/rdoc/comment.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,12 @@ def normalize
self
end

# Change normalized, when creating already normalized comment.

def normalized=(value)
@normalized = value
end

##
# Was this text normalized?

Expand Down Expand Up @@ -226,4 +232,164 @@ def tomdoc?
@format == 'tomdoc'
end

MULTILINE_DIRECTIVES = %w[call-seq].freeze # :nodoc:
private_constant :MULTILINE_DIRECTIVES

class << self

# Parse comment, collect directives as an attribute and return [normalized_comment_text, directives_hash]
# This method expands include and removes everything not needed in the document text, such as
# private section, directive line, comment characters `# /* * */` and indent spaces.
#
# RDoc comment consists of include, directive, multiline directive, private section and comment text.
#
# Include
# # :include: filename
#
# Directive
# # :directive-without-value:
# # :directive-with-value: value
#
# Multiline directive (only :call-seq:)
# # :multiline-directive:
# # value1
# # value2
#
# Private section
# #--
# # private comment
# #++

def parse(text, filename, line_no, type)
case type
when :ruby
private_start_regexp = /^#?-{2,}$/
private_end_regexp = /^#?\+{2}$/
indent_regexp = /^#?\s*/
text = text.sub(/\A#\#$/, '')
when :c
private_start_regexp = /^(\s*\*)?-{2,}$/
private_end_regexp = /^(\s*\*)?\+{2}$/
indent_regexp = /^\s*(\/\*+|\*)?\s*/
text = text.sub(/\s*\*+\/\s*\z/, '')
# TODO: should not be here. Looks like another type of directive
text = text.gsub %r%Document-method:\s+[\w:.#=!?|^&<>~+\-/*\%@`\[\]]+%, ''
when :simple
# Unlike other types, this implementation only looks for two
# dashes at the beginning of the line. Three or more dashes are considered
# to be a rule and ignored.
private_start_regexp = /^-{2}$/
private_end_regexp = /^\+{2}$/
indent_regexp = /^\s*/
end

directives = {}
lines = text.split("\n")
in_private = false
comment_lines = []
until lines.empty?
line = lines.shift
read_lines = 1
if in_private
in_private = false if line.match?(private_end_regexp)
line_no += read_lines
next
elsif line.match?(private_start_regexp)
in_private = true
line_no += read_lines
next
end

prefix = line[indent_regexp]
prefix_indent = ' ' * prefix.size
line = line.byteslice(prefix.bytesize..)
/\A(?<colon>\\?:|:?)(?<directive>[\w-]+):(?<param>.*)/ =~ line
directive = directive&.downcase

if colon == '\\:'
# unescape if escaped
comment_lines << prefix_indent + line.sub('\\:', ':')
elsif !directive || param.start_with?(':') || (colon.empty? && directive != 'call-seq')
# Something like `:toto::` is not a directive
# directive without colon prefix is only allowed for call-seq
comment_lines << prefix_indent + line
elsif directive == 'include'
filename_to_include = param.strip
yield(filename_to_include, prefix_indent).lines.each { |l| comment_lines << l.chomp }
elsif MULTILINE_DIRECTIVES.include?(directive)
param = param.strip
value_lines = take_multiline_directive_value_lines(directive, filename, line_no, lines, prefix_indent, indent_regexp, private_start_regexp)
read_lines += value_lines.size
lines.shift(value_lines.size)
unless param.empty?
# Accept `:call-seq: first-line\n second-line` for now
value_lines.unshift(param)
end
value = value_lines.join("\n")
directives[directive] = [value.empty? ? nil : value, line_no]
else
value = param.strip
directives[directive] = [value.empty? ? nil : value, line_no]
end
line_no += read_lines
end
# normalize comment
min_spaces = nil
comment_lines.each do |l|
next if l.match?(/\A\s*\z/)
n = l[/\A */].size
min_spaces = n if !min_spaces || n < min_spaces
end
comment_lines.map! { |l| l[min_spaces..] || '' } if min_spaces
comment_lines.shift while comment_lines.first&.empty?

[String.new(encoding: text.encoding) << comment_lines.join("\n"), directives]
end

# Take value lines of multiline directive

private def take_multiline_directive_value_lines(directive, filename, line_no, lines, base_indent, indent_regexp, private_start_regexp)
return [] if lines.empty?

base_indent_size = base_indent.size

if lines.first[indent_regexp].size <= base_indent_size && !lines.first.sub(indent_regexp, '').chomp.empty?
# Unindented invalid block directive
warn "#{filename}:#{line_no} Multiline directive :#{directive}: should be indented."
# Take until next blank line, directive line or private section start to accept this invalid format
# # :multiline-directive:
# # line1
# # line2
# # :other-directive:
value_lines = lines.take_while do |l|
next false if l.match?(private_start_regexp)
l = l.sub(indent_regexp, '').chomp
!l.empty? && !l.start_with?(/:[\w-]+:/)
end.map do |l|
l.sub(indent_regexp, '').chomp
end
else
# Take indented lines accepting blank lines between them
first_indent = nil
value_lines = lines.take_while do |l|
l = l.rstrip
indent = l[indent_regexp]
if indent == l || (first_indent && indent.size >= first_indent)
true
elsif first_indent.nil? && indent.size > base_indent_size
first_indent = indent.size
true
end
end
first_indent ||= base_indent_size
value_lines.map! { |l| (l[first_indent..] || '').chomp }

if value_lines.size != lines.size && !value_lines.last.empty?
warn "#{filename}:#{line_no} Multiline directive :#{directive}: should end with a blank line."
end
value_lines.pop while value_lines.last&.empty?
end
value_lines
end
end
end
45 changes: 35 additions & 10 deletions lib/rdoc/markup/pre_process.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,15 @@ def initialize(input_file_name, include_path)
# RDoc::CodeObject#metadata for details.

def handle text, code_object = nil, &block
first_line = 1
if RDoc::Comment === text then
comment = text
text = text.text
first_line = comment.line || 1
end

# regexp helper (square brackets for optional)
# $1 $2 $3 $4 $5
# [prefix][\]:directive:[spaces][param]newline
text = text.lines.map.with_index(first_line) do |line, num|
next line unless line =~ /\A([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):([\w-]+):([ \t]*)(.+)?(\r?\n|$)/
text = text.gsub(/^([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):([\w-]+):([ \t]*)(.+)?(\r?\n|$)/) do
# skip something like ':toto::'
next $& if $4.empty? and $5 and $5[0, 1] == ':'

Expand All @@ -122,21 +119,49 @@ def handle text, code_object = nil, &block
comment.format = $5.downcase
next "#{$1.strip}\n"
end

handle_directive $1, $3, $5, code_object, text.encoding, num, &block
end.join
handle_directive $1, $3, $5, code_object, text.encoding, &block
end

if comment then
comment.text = text
else
comment = text
end

run_post_processes(comment, code_object)

text
end

# Apply directives to a code object

def run_pre_processes(comment_text, code_object, start_line_no, type)
comment_text, directives = parse_comment(comment_text, start_line_no, type)
directives.each do |directive, (param, line_no)|
handle_directive('', directive, param, code_object)
end
if code_object.is_a?(RDoc::AnyMethod) && (call_seq, = directives['call-seq']) && call_seq
code_object.call_seq = call_seq.lines.map(&:chomp).reject(&:empty?).join("\n") if call_seq
end
format, = directives['markup']
[comment_text, format]
end


# Perform post preocesses to a code object

def run_post_processes(comment, code_object)
self.class.post_processors.each do |handler|
handler.call comment, code_object
end
end

text
# Parse comment and return [normalized_comment_text, directives_hash]

def parse_comment(text, line_no, type)
RDoc::Comment.parse(text, @input_file_name, line_no, type) do |filename, prefix_indent|
include_file(filename, prefix_indent, text.encoding)
end
end

##
Expand All @@ -151,7 +176,7 @@ def handle text, code_object = nil, &block
# When 1.8.7 support is ditched prefix can be defaulted to ''

def handle_directive prefix, directive, param, code_object = nil,
encoding = nil, line = nil
encoding = nil
blankline = "#{prefix.strip}\n"
directive = directive.downcase

Expand Down Expand Up @@ -227,7 +252,7 @@ def handle_directive prefix, directive, param, code_object = nil,

blankline
else
result = yield directive, param, line if block_given?
result = yield directive, param if block_given?

case result
when nil then
Expand Down
28 changes: 6 additions & 22 deletions lib/rdoc/parser/c.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,8 +608,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
body = args[1]
offset, = args[2]

comment.remove_private if comment

# try to find the whole body
body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content

Expand All @@ -622,7 +620,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
override_comment = find_override_comment class_name, meth_obj
comment = override_comment if override_comment

comment.normalize
find_modifiers comment, meth_obj if comment

#meth_obj.params = params
Expand All @@ -640,7 +637,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false

find_body class_name, args[3], meth_obj, file_content, true

comment.normalize
find_modifiers comment, meth_obj

meth_obj.start_collecting_tokens
Expand All @@ -664,7 +660,6 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
comment = find_override_comment class_name, meth_obj

if comment then
comment.normalize
find_modifiers comment, meth_obj
meth_obj.comment = comment

Expand Down Expand Up @@ -743,7 +738,6 @@ def find_class_comment class_name, class_mod
end

comment = new_comment comment, @top_level, :c
comment.normalize

look_for_directives_in class_mod, comment

Expand Down Expand Up @@ -804,9 +798,6 @@ def find_const_comment(type, const_name, class_name = nil)
# Handles modifiers in +comment+ and updates +meth_obj+ as appropriate.

def find_modifiers comment, meth_obj
comment.normalize
comment.extract_call_seq meth_obj

look_for_directives_in meth_obj, comment
end

Expand All @@ -820,10 +811,10 @@ def find_override_comment class_name, meth_obj
comment = if @content =~ %r%Document-method:
\s+#{class_name}#{prefix}#{name}
\s*?\n((?>.*?\*/))%xm then
"/*#{$1}"
"/*\n#{$1}"
elsif @content =~ %r%Document-method:
\s#{name}\s*?\n((?>.*?\*/))%xm then
"/*#{$1}"
"/*\n#{$1}"
end

return unless comment
Expand Down Expand Up @@ -1099,17 +1090,10 @@ def load_variable_map map_name
# This method modifies the +comment+

def look_for_directives_in context, comment
@preprocess.handle comment, context do |directive, param|
case directive
when 'main' then
@options.main_page = param
''
when 'title' then
@options.default_title = param if @options.respond_to? :default_title=
''
end
end

comment.text, format = @preprocess.run_pre_processes(comment.text, context, comment.line || 1, :c)
comment.format = format if format
@preprocess.run_post_processes(comment, context)
comment.normalized = true
comment
end

Expand Down
Loading