class Fluent::ForwardOutput::Node
Attributes
available[R]
failure[R]
host[R]
name[R]
port[R]
sockaddr[R]
standby[R]
state[R]
usock[RW]
weight[R]
Public Class Methods
new(sender, server, failure:)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 340 def initialize(sender, server, failure)) @sender = sender @log = sender.log @compress = sender.compress @name = server.name @host = server.host @port = server.port @weight = server.weight @standby = server.standby @failure = failure @available = true @state = nil @usock = nil @username = server.username @password = server.password @shared_key = server.shared_key || (sender.security && sender.security.shared_key) || "" @shared_key_salt = generate_salt @shared_key_nonce = "" @unpacker = Fluent::Engine.msgpack_unpacker @resolved_host = nil @resolved_time = 0 resolved_host # check dns end
Public Instance Methods
available?()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 375 def available? @available end
check_helo(message)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 613 def check_helo(message) @log.debug "checking helo" # ['HELO', options(hash)] unless message.size == 2 && message[0] == 'HELO' return false end opts = message[1] || {} # make shared_key_check failed (instead of error) if protocol version mismatch exist @shared_key_nonce = opts['nonce'] || '' @authentication = opts['auth'] || '' true end
check_pong(message)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 645 def check_pong(message) @log.debug "checking pong" # ['PONG', bool(authentication result), 'reason if authentication failed', # self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)] unless message.size == 5 && message[0] == 'PONG' return false, 'invalid format for PONG message' end _pong, auth_result, reason, hostname, shared_key_hexdigest = message unless auth_result return false, 'authentication failed: ' + reason end if hostname == @sender.security.self_hostname return false, 'same hostname between input and output: invalid configuration' end clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key_nonce).update(@shared_key).hexdigest unless shared_key_hexdigest == clientside return false, 'shared key mismatch' end return true, nil end
connect()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 387 def connect TCPSocket.new(resolved_host, port) end
disable!()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 379 def disable! @available = false end
establish_connection(sock)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 401 def establish_connection(sock) while available? && @state != :established begin # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly. # We need rewrite around here using new socket/server plugin helper. buf = sock.read_nonblock(@sender.read_length) if buf.empty? sleep @sender.read_interval next end @unpacker.feed_each(buf) do |data| on_read(sock, data) end rescue IO::WaitReadable # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable. # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock. # http://docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock sleep @sender.read_interval unless @state == :established rescue SystemCallError => e @log.warn "disconnected by error", host: @host, port: @port, error: e disable! break rescue EOFError @log.warn "disconnected", host: @host, port: @port disable! break end end end
generate_ping()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 626 def generate_ping @log.debug "generating ping" # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key), # username || '', sha512\_hex(auth\_salt + username + password) || ''] shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt) .update(@sender.security.self_hostname) .update(@shared_key_nonce) .update(@shared_key) .hexdigest ping = ['PING', @sender.security.self_hostname, @shared_key_salt, shared_key_hexdigest] if !@authentication.empty? password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest ping.push(@username, password_hexdigest) else ping.push('','') end ping end
generate_salt()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 609 def generate_salt SecureRandom.hex(16) end
heartbeat(detect=true)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 592 def heartbeat(detect=true) now = Time.now.to_f @failure.add(now) if detect && !@available && @failure.sample_size > @sender.recover_sample_size @available = true @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port true else nil end end
on_read(sock, data)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 670 def on_read(sock, data) @log.trace __callee__ case @state when :helo unless check_helo(data) @log.warn "received invalid helo message from #{@name}" disable! # shutdown return end sock.write(generate_ping.to_msgpack) @state = :pingpong when :pingpong succeeded, reason = check_pong(data) unless succeeded @log.warn "connection refused to #{@name}: #{reason}" disable! # shutdown return end @state = :established @log.debug "connection established", host: @host, port: @port else raise "BUG: unknown session state: #{@state}" end end
resolved_host()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 533 def resolved_host case @sender.expire_dns_cache when 0 # cache is disabled resolve_dns! when nil # persistent cache @resolved_host ||= resolve_dns! else now = Engine.now rh = @resolved_host if !rh || now - @resolved_time >= @sender.expire_dns_cache rh = @resolved_host = resolve_dns! @resolved_time = now end rh end end
send_data(tag, chunk)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 431 def send_data(tag, chunk) sock = connect @state = @sender.security ? :helo : :established begin set_socket_options(sock) if @state != :established establish_connection(sock) end unless available? raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}" end option = { 'size' => chunk.size_of_events, 'compressed' => @compress } option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response # out_forward always uses Raw32 type for content. # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size. sock.write @sender.forward_header # beginArray(3) sock.write tag.to_msgpack # 1. writeRaw(tag) chunk.open(compressed: @compress) do |chunk_io| sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32 IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es) end sock.write option.to_msgpack # 3. writeOption(option) if @sender.require_ack_response # Waiting for a response here results in a decrease of throughput because a chunk queue is locked. # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses # and process them asynchronously. if IO.select([sock], nil, nil, @sender.ack_response_timeout) raw_data = begin sock.recv(1024) rescue Errno::ECONNRESET "" end # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF. # If this happens we assume the data wasn't delivered and retry it. if raw_data.empty? @log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port disable! raise ForwardOutputConnectionClosedError, "node #{@host}:#{@port} closed connection" else @unpacker.feed(raw_data) res = @unpacker.read if res['ack'] != option['chunk'] # Some errors may have occured when ack and chunk id is different, so send the chunk again. raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different" end end else # IO.select returns nil on timeout. # There are 2 types of cases when no response has been received: # (1) the node does not support sending responses # (2) the node does support sending response but responses have not arrived for some reasons. @log.warn "no response from node. regard it as unavailable.", host: @host, port: @port disable! raise ForwardOutputACKTimeoutError, "node #{host}:#{port} does not return ACK" end end heartbeat(false) res # for test ensure sock.close_write sock.close end end
send_heartbeat()
click to toggle source
FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
# File lib/fluent/plugin/out_forward.rb, line 505 def send_heartbeat case @sender.heartbeat_type when :tcp sock = connect begin opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; } sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt) # opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval # sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt) ## don't send any data to not cause a compatibility problem # sock.write FORWARD_TCP_HEARTBEAT_DATA # successful tcp connection establishment is considered as valid heartbeat heartbeat(true) ensure sock.close_write sock.close end when :udp @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host) when :none # :none doesn't use this class raise "BUG: heartbeat_type none must not use Node" else raise "BUG: unknown heartbeat_type '#{@sender.heartbeat_type}'" end end
set_socket_options(sock)
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 391 def set_socket_options(sock) opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; } sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt) opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt) sock end
standby?()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 383 def standby? @standby end
tick()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 562 def tick now = Time.now.to_f if !@available if @failure.hard_timeout?(now) @failure.clear end return nil end if @failure.hard_timeout?(now) @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, hard_timeout: true @available = false @resolved_host = nil # expire cached host @failure.clear return true end if @sender.phi_failure_detector phi = @failure.phi(now) if phi > @sender.phi_threshold @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, phi: phi, phi_threshold: @sender.phi_threshold @available = false @resolved_host = nil # expire cached host @failure.clear return true end end false end
to_msgpack(out = '')
click to toggle source
TODO: to_msgpack(string) is deprecated
# File lib/fluent/plugin/out_forward.rb, line 605 def to_msgpack(out = '') [@host, @port, @weight, @available].to_msgpack(out) end
Private Instance Methods
resolve_dns!()
click to toggle source
# File lib/fluent/plugin/out_forward.rb, line 554 def resolve_dns! addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM) addrinfo = @sender.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_heartbeat addrinfo[3] end