IOError(EOFError() during request(https://logs.us-east-1.amazonaws.com/))

[omus]

[2019-10-22 14:18:15 | warn | CloudWatchLogs]: IOError(EOFError() during request(https://logs.us-east-1.amazonaws.com/))
handshake(::MbedTLS.SSLContext) at /root/.julia/packages/MbedTLS/X4xar/src/ssl.jl:79
handshake! at /root/.julia/packages/MbedTLS/X4xar/src/MbedTLS.jl:141 [inlined]
#sslconnection#18(::Bool, ::MbedTLS.SSLConfig, ::Base.Iterators.Pairs{Symbol,Union{Nothing, Int64},Tuple{Symbol,Symbol},NamedTuple{(:iofunction, :verbose),Tuple{Nothing,Int64}}}, ::Function, ::Sockets.TCPSocket, ::SubString{String}) at /root/.julia/packages/HTTP/hJSsm/src/ConnectionPool.jl:637
#getconnection#11(::Int64, ::Int64, ::Int64, ::Int64, ::Bool, ::Base.Iterators.Pairs{Symbol,Union{Nothing, Int64},Tuple{Symbol,Symbol},NamedTuple{(:iofunction, :verbose),Tuple{Nothing,Int64}}}, ::Function, ::Type{HTTP.ConnectionPool.Transaction{MbedTLS.SSLContext}}, ::SubString{String}, ::SubString{String}) at ./none:0
(::getfield(HTTP.ConnectionPool, Symbol("#kw##getconnection")))(::NamedTuple{(:reuse_limit, :iofunction, :verbose, :require_ssl_verification),Tuple{Int64,Nothing,Int64,Bool}}, ::typeof(HTTP.ConnectionPool.getconnection), ::Type{HTTP.ConnectionPool.Transaction{MbedTLS.SSLContext}}, ::SubString{String}, ::SubString{String}) at ./none:0
#request#1(::Nothing, ::Type, ::Int64, ::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol,Symbol},NamedTuple{(:iofunction, :verbose, :require_ssl_verification),Tuple{Nothing,Int64,Bool}}}, ::Function, ::Type{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}, ::HTTP.URIs.URI, ::HTTP.Messages.Request, ::String) at /root/.julia/packages/HTTP/hJSsm/src/ConnectionRequest.jl:41
(::getfield(HTTP, Symbol("#kw##request")))(::NamedTuple{(:iofunction, :verbose, :require_ssl_verification),Tuple{Nothing,Int64,Bool}}, ::typeof(HTTP.request), ::Type{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}, ::HTTP.URIs.URI, ::HTTP.Messages.Request, ::String) at ./none:0
#request#1(::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol,Symbol},NamedTuple{(:iofunction, :verbose, :require_ssl_verification),Tuple{Nothing,Int64,Bool}}}, ::Function, ::Type{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}, ::HTTP.URIs.URI, ::Vararg{Any,N} where N) at /root/.julia/packages/HTTP/hJSsm/src/ExceptionRequest.jl:19
#request at ./none:0 [inlined]
#request#1(::VersionNumber, ::String, ::Nothing, ::Nothing, ::Base.Iterators.Pairs{Symbol,Integer,Tuple{Symbol,Symbol},NamedTuple{(:verbose, :require_ssl_verification),Tuple{Int64,Bool}}}, ::Function, ::Type{HTTP.MessageRequest.MessageLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}, ::String, ::HTTP.URIs.URI, ::Array{Pair{SubString{String},SubString{String}},1}, ::String) at /root/.julia/packages/HTTP/hJSsm/src/MessageRequest.jl:47
(::getfield(HTTP, Symbol("#kw##request")))(::NamedTuple{(:verbose, :require_ssl_verification),Tuple{Int64,Bool}}, ::typeof(HTTP.request), ::Type{HTTP.MessageRequest.MessageLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}, ::String, ::HTTP.URIs.URI, ::Array{Pair{SubString{String},SubString{String}},1}, ::String) at ./none:0
http_request(::Dict{Symbol,Any}) at /root/.julia/packages/AWSCore/xaNIu/src/http.jl:42
macro expansion at /root/.julia/packages/AWSCore/xaNIu/src/AWSCore.jl:410 [inlined]
macro expansion at /root/.julia/packages/Retry/0jMye/src/repeat_try.jl:192 [inlined]
do_request(::Dict{Symbol,Any}) at /root/.julia/packages/AWSCore/xaNIu/src/AWSCore.jl:393
#service_json#16(::Base.Iterators.Pairs{Symbol,Any,NTuple{6,Symbol},NamedTuple{(:service, :version, :json_version, :target, :operation, :args),Tuple{String,String,String,String,String,Base.Iterators.Pairs{Symbol,Any,NTuple{4,Symbol},NamedTuple{(:logEvents, :logGroupName, :logStreamName, :sequenceToken),Tuple{Array{CloudWatchLogs.LogEvent,1},String,String,String}}}}}}, ::Function, ::Dict{Symbol,Any}) at /root/.julia/packages/AWSCore/xaNIu/src/AWSCore.jl:262
#service_json at ./none:0 [inlined]
logs(::Dict{Symbol,Any}, ::String, ::Base.Iterators.Pairs{Symbol,Any,NTuple{4,Symbol},NamedTuple{(:logEvents, :logGroupName, :logStreamName, :sequenceToken),Tuple{Array{CloudWatchLogs.LogEvent,1},String,String,String}}}) at /root/.julia/packages/AWSCore/xaNIu/src/Services.jl:1425
#logs#86(::Base.Iterators.Pairs{Symbol,Any,NTuple{4,Symbol},NamedTuple{(:logEvents, :logGroupName, :logStreamName, :sequenceToken),Tuple{Array{CloudWatchLogs.LogEvent,1},String,String,String}}}, ::Function, ::Dict{Symbol,Any}, ::Vararg{Any,N} where N) at /root/.julia/packages/AWSCore/xaNIu/src/Services.jl:1438
(::getfield(AWSCore.Services, Symbol("#kw##logs")))(::NamedTuple{(:logEvents, :logGroupName, :logStreamName, :sequenceToken),Tuple{Array{CloudWatchLogs.LogEvent,1},String,String,String}}, ::typeof(AWSCore.Services.logs), ::Dict{Symbol,Any}, ::Vararg{Any,N} where N) at ./none:0
_put_log_events(::CloudWatchLogs.CloudWatchLogStream, ::Array{CloudWatchLogs.LogEvent,1}) at /root/.julia/packages/CloudWatchLogs/dwPFC/src/stream.jl:214
macro expansion at /root/.julia/packages/Mocking/hFQQ4/src/mock.jl:29 [inlined]
#20 at /root/.julia/packages/CloudWatchLogs/dwPFC/src/stream.jl:299 [inlined]
(::getfield(Base, Symbol("###44#45#46")){ExponentialBackOff,getfield(CloudWatchLogs, Symbol("#retry_cond#24")){CloudWatchLogs.CloudWatchLogStream},getfield(CloudWatchLogs, Symbol("##20#26")){CloudWatchLogs.CloudWatchLogStream,Array{CloudWatchLogs.LogEvent,1}}})(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function) at ./error.jl:203
#44 at ./error.jl:199 [inlined]
submit_logs(::CloudWatchLogs.CloudWatchLogStream, ::Array{CloudWatchLogs.LogEvent,1}) at /root/.julia/packages/CloudWatchLogs/dwPFC/src/stream.jl:305
macro expansion at /root/.julia/packages/Mocking/hFQQ4/src/mock.jl:29 [inlined]
process_available_logs!(::CloudWatchLogs.CloudWatchLogHandler{DefaultFormatter}) at /root/.julia/packages/CloudWatchLogs/dwPFC/src/handler.jl:72
process_logs!(::CloudWatchLogs.CloudWatchLogHandler{DefaultFormatter}) at /root/.julia/packages/CloudWatchLogs/dwPFC/src/handler.jl:93
(::getfield(CloudWatchLogs, Symbol("##30#31")){CloudWatchLogs.CloudWatchLogHandler{DefaultFormatter}})() at ./task.jl:259

[iamed2]

Perhaps the HTTP.jl connection pool is retaining bad connections?

This is already being retried three times in AWSCore. That makes me think there's some sort of systemic issue.

[nickrobinson251]

Saw this reliably today on EIS!404