Top Level Namespace

Defined Under Namespace

Modules: FaultTolerantRouter

Instance Method Summary collapse

Instance Method Details

#command(c) ⇒ Object


1
2
3
4
# File 'lib/fault_tolerant_router/monitor.rb', line 1

def command(c)
  `#{c}` unless DEMO
  puts "Command: #{c}" if DEBUG
end

#generate_config(file_path) ⇒ Object


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
# File 'lib/fault_tolerant_router/generate_config.rb', line 1

def generate_config(file_path)
  if File.exists?(file_path)
    puts "Configuration file #{file_path} already exists, will not overwrite!"
    exit 1
  end
  begin
    open(file_path, 'w') do |file|
      file.puts <<END
#add as many uplinks as needed
uplinks:
- interface: eth1
  ip: 1.0.0.2
  gateway: 1.0.0.1
  description: Example Provider 1
  #optional parameter
  weight: 1
  #optional parameter, default is true
  default_route: true
- interface: eth2
  ip: 2.0.0.2
  gateway: 2.0.0.1
  description: Example Provider 2
  #optional parameter
  weight: 2
  #optional parameter, default is true
  default_route: true
- interface: eth3
  ip: 3.0.0.2
  gateway: 3.0.0.1
  description: Example Provider 3
  #optional parameter
  weight: 1
  #optional parameter, default is true
  default_route: true

downlinks:
  lan: eth0
  #leave blank if you have no DMZ
  dmz:

tests:
  #add as many ips as needed, make sure they are reliable ones, these are Google DNS, OpenDNS DNS, public DNS server
  #list order is not important, because the list is shuffled before every test
  ips:
  - 8.8.8.8
  - 8.8.4.4
  - 208.67.222.222
  - 208.67.220.220
  - 4.2.2.2
  - 4.2.2.3
  #number of successful pinged addresses to consider an uplink to be functional
  required_successful: 4
  #ping retries in case of ping error
  ping_retries: 1
  #seconds between a check of the uplinks and the next one
  interval: 60

log:
  #file: "/var/log/fault_tolerant_router.log"
  file: "/tmp/fault_tolerant_router.log"
  #max log file size (in bytes)
  max_size: 1024000
  #number of old log files to keep
  old_files: 10

email:
  send: false
  sender: [email protected]
  recipients:
  - [email protected]
  - [email protected]
  - [email protected]
  #see http://ruby-doc.org/stdlib-2.2.0/libdoc/net/smtp/rdoc/Net/SMTP.html
  smtp_parameters:
    address: smtp.gmail.com
    port: 587
    #domain: domain.com
    authentication: :login
    enable_starttls_auto: true
    user_name: [email protected]
    password: secret-password

#base ip route table
base_table: 1

#base ip rule priority, must be higher than 32767 (default priority, see "ip rule")
base_priority: 40000

#base fwmark
base_fwmark: 1
END
    end
    puts "Example configuration saved to #{file_path}"
  rescue
    puts "Error while saving configuration file #{file_path}!"
    exit 1
  end
end

#generate_iptablesObject


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
# File 'lib/fault_tolerant_router/generate_iptables.rb', line 1

def generate_iptables
  puts <<END
#Integrate with your existing "iptables-save" configuration, or adapt to work
#with any other iptables configuration system

*mangle
:PREROUTING ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:INPUT ACCEPT [0:0]

#New outbound connections: force a connection to use a specific uplink instead
#of participating in the multipath routing. This can be useful if you have an
#SMTP server that should always send emails originating from a specific IP
#address (because of PTR DNS records), or if you have some service that you want
#always to use a particular slow/fast uplink.
#
#Uncomment if needed.
#
#NB: these are just examples, you can add as many options as needed: -s, -d,
#    --sport, etc.

END
  UPLINKS.each_with_index do |uplink, i|
    puts "##{uplink[:description]}"
    puts "#[0:0] -A PREROUTING -i #{LAN_INTERFACE} -m state --state NEW -p tcp --dport XXX -j CONNMARK --set-mark #{BASE_FWMARK + i}"
    puts "#[0:0] -A PREROUTING -i #{DMZ_INTERFACE} -m state --state NEW -p tcp --dport XXX -j CONNMARK --set-mark #{BASE_FWMARK + i}" if DMZ_INTERFACE
  end
  puts <<END

#Mark packets with the outgoing interface:
#
#- Established outbound connections: mark non-first packets (first packet will
#  be marked as 0, as a standard unmerked packet, because the connection has not
#  yet been marked with CONNMARK --set-mark)
#
#- New outbound connections: mark first packet, only effective if marking has
#  been done in the section above
#
#- Inbound connections: mark returning packets (from LAN/DMZ to WAN)

[0:0] -A PREROUTING -i #{LAN_INTERFACE} -j CONNMARK --restore-mark
END
  puts "[0:0] -A PREROUTING -i #{DMZ_INTERFACE} -j CONNMARK --restore-mark" if DMZ_INTERFACE
  puts <<END

#New inbound connections: mark the connection with the incoming interface.

END
  UPLINKS.each_with_index do |uplink, i|
    puts "##{uplink[:description]}"
    puts "[0:0] -A PREROUTING -i #{uplink[:interface]} -m state --state NEW -j CONNMARK --set-mark #{BASE_FWMARK + i}"
  end
  puts <<END

#New outbound connections: mark the connection with the outgoing interface
#(chosen by the multipath routing).

END
  UPLINKS.each_with_index do |uplink, i|
    puts "##{uplink[:description]}"
    puts "[0:0] -A POSTROUTING -o #{uplink[:interface]} -m state --state NEW -j CONNMARK --set-mark #{BASE_FWMARK + i}"
  end
  puts <<END

COMMIT


*nat
:PREROUTING ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]

#DNAT: WAN --> LAN/DMZ. The original destination IP (-d) can be any of the IP
#addresses assigned to the uplink interface. XXX.XXX.XXX.XXX can be any of your
#LAN/DMZ IPs.
#
#Uncomment if needed.
#
#NB: these are just examples, you can add as many options as you wish: -s,
#    --sport, --dport, etc.

END
  UPLINKS.each do |uplink|
    puts "##{uplink[:description]}"
    puts "#[0:0] -A PREROUTING -i #{uplink[:interface]} -d #{uplink[:ip]} -j DNAT --to-destination XXX.XXX.XXX.XXX"
  end
  puts <<END

#SNAT: LAN/DMZ --> WAN. Force an outgoing connection to use a specific source
#address instead of the default one of the outgoing interface. Of course this
#only makes sense if more than one IP address is assigned to the uplink
#interface.
#
#Uncomment if needed.
#
#NB: these are just examples, you can add as many options as needed: -d,
#    --sport, --dport, etc.

END
  UPLINKS.each do |uplink|
    puts "##{uplink[:description]}"
    puts "#[0:0] -A POSTROUTING -s XXX.XXX.XXX.XXX -o #{uplink[:interface]} -j SNAT --to-source YYY.YYY.YYY.YYY"
  end
  puts <<END

#SNAT: LAN --> WAN

END
  UPLINKS.each do |uplink|
    puts "##{uplink[:description]}"
    puts "[0:0] -A POSTROUTING -o #{uplink[:interface]} -j SNAT --to-source #{uplink[:ip]}"
  end
  puts <<END

COMMIT


*filter

:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
:LAN_WAN - [0:0]
:WAN_LAN - [0:0]
END

  if DMZ_INTERFACE
    puts ':DMZ_WAN - [0:0]'
    puts ':WAN_DMZ - [0:0]'
  end

  puts <<END

#This is just a very basic example, add your own rules for the INPUT chain.

[0:0] -A INPUT -i lo -j ACCEPT
[0:0] -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT

[0:0] -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT

END
  UPLINKS.each do |uplink|
    puts "[0:0] -A FORWARD -i #{LAN_INTERFACE} -o #{uplink[:interface]} -j LAN_WAN"
  end
  UPLINKS.each do |uplink|
    puts "[0:0] -A FORWARD -i #{uplink[:interface]} -o #{LAN_INTERFACE} -j WAN_LAN"
  end
  if DMZ_INTERFACE
    UPLINKS.each do |uplink|
      puts "[0:0] -A FORWARD -i #{DMZ_INTERFACE} -o #{uplink[:interface]} -j DMZ_WAN"
    end
    UPLINKS.each do |uplink|
      puts "[0:0] -A FORWARD -i #{uplink[:interface]} -o #{DMZ_INTERFACE} -j WAN_DMZ"
    end
  end
  puts <<END

#This is just a very basic example, add your own rules for the FORWARD chain.

[0:0] -A LAN_WAN -j ACCEPT
[0:0] -A WAN_LAN -j REJECT
END
  if DMZ_INTERFACE
    puts '[0:0] -A DMZ_WAN -j ACCEPT'
    puts '[0:0] -A WAN_DMZ -j ACCEPT'
  end
  puts <<END

COMMIT
END
end

#monitorObject


46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
# File 'lib/fault_tolerant_router/monitor.rb', line 46

def monitor
  logger = Logger.new(LOG_FILE, LOG_OLD_FILES, LOG_MAX_SIZE)

  #enable all the uplinks
  UPLINKS.each do |uplink|
    uplink[:working] = true
    uplink[:default_route] ||= uplink[:default_route].nil?
    uplink[:enabled] = uplink[:default_route]
  end

  #clean all previous configurations, try to clean more than needed (double) to avoid problems in case of changes in the
  #number of uplinks between different executions
  ((UPLINKS.size * 2 + 1) * 2).times do |i|
    command "ip rule del priority #{BASE_PRIORITY + i} &> /dev/null"
  end
  ((UPLINKS.size + 1) * 2).times do |i|
    command "ip route del table #{BASE_TABLE + i} &> /dev/null"
  end

  #disable "reverse path filtering" on the uplink interfaces
  command 'echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter'
  UPLINKS.each do |uplink|
    command "echo 2 > /proc/sys/net/ipv4/conf/#{uplink[:interface]}/rp_filter"
  end

  #- locally generated packets having as source ip the ethX ip
  #- returning packets of inbound connections coming from ethX
  #- non-first packets of outbound connections for which the first packet has been sent to ethX via multipath routing
  UPLINKS.each_with_index do |uplink, i|
    command "ip route add table #{BASE_TABLE + i} default via #{uplink[:gateway]} src #{uplink[:ip]}"
    command "ip rule add priority #{BASE_PRIORITY + i} from #{uplink[:ip]} lookup #{BASE_TABLE + i}"
    command "ip rule add priority #{BASE_PRIORITY + UPLINKS.size + i} fwmark #{BASE_FWMARK + i} lookup #{BASE_TABLE + i}"
  end  #first packet of outbound connections

  command "ip rule add priority #{BASE_PRIORITY + UPLINKS.size * 2} from all lookup #{BASE_TABLE + UPLINKS.size}"
  set_default_route

  loop do
    #for each uplink...
    UPLINKS.each do |uplink|
      #set current "working" state as the previous one
      uplink[:previously_working] = uplink[:working]      #set current "enabled" state as the previous one

      uplink[:previously_enabled] = uplink[:enabled]
      uplink[:successful_tests] = 0
      uplink[:unsuccessful_tests] = 0      #for each test (in random order)...

      TEST_IPS.shuffle.each_with_index do |test, i|
        successful_test = false        #retry for several times...

        PING_RETRIES.times do
          if DEBUG
            print "Uplink #{uplink[:description]}: ping #{test}... "
            STDOUT.flush
          end
          if ping(test, uplink[:ip])
            successful_test = true
            puts 'ok' if DEBUG            #avoid more pings to the same ip after a successful one

            break
          else
            puts 'error' if DEBUG
          end
        end
        if successful_test
          uplink[:successful_tests] += 1
        else
          uplink[:unsuccessful_tests] += 1
        end        #if not currently doing the last test...

        if i + 1 < TEST_IPS.size
          if uplink[:successful_tests] >= REQUIRED_SUCCESSFUL_TESTS
            puts "Uplink #{uplink[:description]}: avoiding more tests because there are enough positive ones" if DEBUG
            break
          elsif TEST_IPS.size - uplink[:unsuccessful_tests] < REQUIRED_SUCCESSFUL_TESTS
            puts "Uplink #{uplink[:description]}: avoiding more tests because too many have been failed" if DEBUG
            break
          end
        end
      end
      uplink[:working] = uplink[:successful_tests] >= REQUIRED_SUCCESSFUL_TESTS
      uplink[:enabled] = uplink[:working] && uplink[:default_route]
    end

    #only consider uplinks flagged as default route
    if UPLINKS.find_all { |uplink| uplink[:default_route] }.all? { |uplink| !uplink[:working] }
      UPLINKS.find_all { |uplink| uplink[:default_route] }.each { |uplink| uplink[:enabled] = true }
      puts 'No uplink seems to be working, enabling all of them' if DEBUG
    end

    UPLINKS.each do |uplink|
      description = case
                      when uplink[:enabled] && !uplink[:previously_enabled] then
                        ', enabled'
                      when !uplink[:enabled] && uplink[:previously_enabled] then
                        ', disabled'
                      else
                        ''
                    end
      puts "Uplink #{uplink[:description]}: #{uplink[:successful_tests]} successful tests, #{uplink[:unsuccessful_tests]} unsuccessful tests#{description}"
    end if DEBUG

    #set a new default route if there are changes between the previous and the current uplinks situation
    set_default_route if UPLINKS.any? { |uplink| uplink[:enabled] != uplink[:previously_enabled] }

    if UPLINKS.any? { |uplink| uplink[:working] != uplink[:previously_working] }
      body = ''
      UPLINKS.each do |uplink|
        body += "Uplink #{uplink[:description]}: #{uplink[:previously_working] ? 'up' : 'down'}"
        if uplink[:previously_working] == uplink[:working]
          body += "\n"
        else
          body += " --> #{uplink[:working] ? 'up' : 'down'}\n"
        end
      end

      logger.warn(body.gsub("\n", ';'))

      if SEND_EMAIL
        begin
          send_email(body)
        rescue Exception => e
          puts "Problem sending email: #{e}" if DEBUG
          logger.error("Problem sending email: #{e}")
        end
      end
    end

    if DEMO
      puts "Waiting just 5 seconds because we are in demo mode, otherwise would wait #{TEST_INTERVAL} seconds..."
      sleep 5
    else
      puts "Waiting #{TEST_INTERVAL} seconds..." if DEBUG
      sleep TEST_INTERVAL
    end
  end
end

#ping(ip, source) ⇒ Object


6
7
8
9
10
11
12
13
14
# File 'lib/fault_tolerant_router/monitor.rb', line 6

def ping(ip, source)
  if DEMO
    sleep 0.1
    rand(3) > 0
  else
    `ping -n -c 1 -W 2 -I #{source} #{ip}`
    $?.to_i == 0
  end
end

#send_email(body) ⇒ Object


36
37
38
39
40
41
42
43
44
# File 'lib/fault_tolerant_router/monitor.rb', line 36

def send_email(body)
  mail = Mail.new
  mail.from = EMAIL_SENDER
  mail.to = EMAIL_RECIPIENTS
  mail.subject = 'Uplinks status change'
  mail.body = body
  mail.delivery_method :smtp, SMTP_PARAMETERS
  mail.deliver
end

#set_default_routeObject


16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# File 'lib/fault_tolerant_router/monitor.rb', line 16

def set_default_route  #find the enabled uplinks

  enabled_uplinks = UPLINKS.find_all { |uplink| uplink[:enabled] }  #do not use balancing if there is just one enabled uplink

  if enabled_uplinks.size == 1
    nexthops = "via #{enabled_uplinks.first[:gateway]}"
  else
    nexthops = enabled_uplinks.collect do |uplink|
      #the "weight" parameter is optional
      weight = uplink[:weight] ? " weight #{uplink[:weight]}" : ''
      "nexthop via #{uplink[:gateway]}#{weight}"
    end
    nexthops = nexthops.join(' ')
  end  #set the route for first packet of outbound connections

  command "ip route replace table #{BASE_TABLE + UPLINKS.size} default #{nexthops}"  #apply the routing changes

  command 'ip route flush cache'
end