Class: WebRobots::RobotsTxt::Record

Inherits:

Object

• Object
• WebRobots::RobotsTxt::Record

show all

Defined in:

lib/webrobots/robotstxt.rb

Instance Attribute Summary (collapse)

• - (Object) delay readonly

  Returns the value of attribute delay.

• - (Object) options readonly

  Returns the value of attribute options.

Instance Method Summary (collapse)

• - (Boolean) allow?(request_uri)
• - (Boolean) default?
• - (Record) initialize(agentlines, rulelines) constructor

  A new instance of Record.

• - (Boolean) match?(user_agent)

604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622

# File 'lib/webrobots/robotstxt.rb', line 604

def initialize(agentlines, rulelines)
  @patterns = agentlines.map { |agentline| agentline.pattern }
  @acls = []
  @delay = nil
  @options = {}
  rulelines.each { |ruleline|
    case ruleline
    when AccessControlLine
      @acls << ruleline
    when CrawlDelayLine
      @delay = ruleline.delay
    else
      @options[ruleline.token.downcase] = ruleline.value
    end
  }
  @acls.replace @acls.sort_by { |x|
    [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
  }
end

624
625
626

# File 'lib/webrobots/robotstxt.rb', line 624

def delay
  @delay
end

624
625
626

# File 'lib/webrobots/robotstxt.rb', line 624

def options
  @options
end

636
637
638
639
640
641
642
643

# File 'lib/webrobots/robotstxt.rb', line 636

def allow?(request_uri)
  @acls.each { |acl|
    if acl.match?(request_uri)
      return acl.allow?
    end
  }
  return true
end

632
633
634

# File 'lib/webrobots/robotstxt.rb', line 632

def default?
  @patterns.include?(//)
end

626
627
628
629
630

# File 'lib/webrobots/robotstxt.rb', line 626

def match?(user_agent)
  @patterns.any? { |pattern|
    pattern.match(user_agent)
  }
end