Class: WebRobots::RobotsTxt::Record

Inherits:
Object
  • Object
show all
Defined in:
lib/webrobots/robotstxt.rb

Instance Attribute Summary (collapse)

Instance Method Summary (collapse)

Constructor Details

- (Record) initialize(agentlines, rulelines)

Returns a new instance of Record



604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
# File 'lib/webrobots/robotstxt.rb', line 604

def initialize(agentlines, rulelines)
  @patterns = agentlines.map { |agentline| agentline.pattern }
  @acls = []
  @delay = nil
  @options = {}
  rulelines.each { |ruleline|
    case ruleline
    when AccessControlLine
      @acls << ruleline
    when CrawlDelayLine
      @delay = ruleline.delay
    else
      @options[ruleline.token.downcase] = ruleline.value
    end
  }
  @acls.replace @acls.sort_by { |x|
    [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
  }
end

Instance Attribute Details

- (Object) delay (readonly)

Returns the value of attribute delay



624
625
626
# File 'lib/webrobots/robotstxt.rb', line 624

def delay
  @delay
end

- (Object) options (readonly)

Returns the value of attribute options



624
625
626
# File 'lib/webrobots/robotstxt.rb', line 624

def options
  @options
end

Instance Method Details

- (Boolean) allow?(request_uri)

Returns:

  • (Boolean)


636
637
638
639
640
641
642
643
# File 'lib/webrobots/robotstxt.rb', line 636

def allow?(request_uri)
  @acls.each { |acl|
    if acl.match?(request_uri)
      return acl.allow?
    end
  }
  return true
end

- (Boolean) default?

Returns:

  • (Boolean)


632
633
634
# File 'lib/webrobots/robotstxt.rb', line 632

def default?
  @patterns.include?(//)
end

- (Boolean) match?(user_agent)

Returns:

  • (Boolean)


626
627
628
629
630
# File 'lib/webrobots/robotstxt.rb', line 626

def match?(user_agent)
  @patterns.any? { |pattern|
    pattern.match(user_agent)
  }
end