Platform hangs, lots of HTTP call interceptor fail errors #7381
Comments
dominictory
added
bug
|
@dominictory do you have a specific repro case? |
nino-filigran
added
needs more info
Honestly not sure, past few versions I've experienced a very unstable platform where sometimes the page I want takes a while to load, and then just gives me 'an unknown error has occurred' in the UI. Having inspected the logs when this happens, I get the above errors. I've noticed similar issue mentioned on Slack fairly recently but I can't work out exactly what the issue is. |
|
Seeing the same thing in a clustered deployment, lots of container restarts. UI is almost unusable post 6.1.X, dashboard load time is >12 seconds {
"category": "APP",
"errors": [
{
"attributes": {
"genre": "TECHNICAL",
"http_status": 500,
"promise": {},
"reason": {
"code": "ERR_HTTP_HEADERS_SENT"
}
},
"message": "Engine unhandled rejection",
"name": "UNKNOWN_ERROR",
"stack": "UNKNOWN_ERROR: Engine unhandled rejection\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at process.<anonymous> (/opt/opencti/build/src/boot.js:60:16)\n at process.emit (node:events:519:28)\n at process.emit (/opt/opencti/build/node_modules/source-map-support/source-map-support.js:516:21)\n at emit (node:internal/process/promises:150:20)\n at processPromiseRejections (node:internal/process/promises:284:27)\n at processTicksAndRejections (node:internal/process/task_queues:96:32)"
}
],
"level": "error",
"message": "Engine unhandled rejection",
"source": "backend",
"timestamp": "2024-06-17T20:21:27.867Z",
"version": "6.1.6"
}
{
"category": "APP",
"errors": [
{
"attributes": {
"genre": "TECHNICAL",
"http_status": 500,
"referer": "https://intel/dashboard/data/import"
},
"message": "Http call interceptor fail",
"name": "UNKNOWN_ERROR",
"stack": "UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:455:18)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"
},
{
"message": "stream is not readable",
"name": "InternalServerError",
"stack": "InternalServerError: stream is not readable\n at readStream (/opt/opencti/build/node_modules/raw-body/index.js:185:17)\n at getBody (/opt/opencti/build/node_modules/raw-body/index.js:116:12)\n at read (/opt/opencti/build/node_modules/body-parser/lib/read.js:79:3)\n at fn (/opt/opencti/build/node_modules/body-parser/lib/types/json.js:138:5)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at cors (/opt/opencti/build/node_modules/cors/lib/index.js:188:7)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:224:17)\n at originCallback (/opt/opencti/build/node_modules/cors/lib/index.js:214:15)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:219:13)\n at optionsCallback (/opt/opencti/build/node_modules/cors/lib/index.js:199:9)\n at fn (/opt/opencti/build/node_modules/cors/lib/index.js:204:7)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at Function.handle (/opt/opencti/build/node_modules/express/lib/router/index.js:175:3)\n at router (/opt/opencti/build/node_modules/express/lib/router/index.js:47:12)"
}
],
"level": "error",
"message": "Http call interceptor fail",
"source": "backend",
"timestamp": "2024-06-17T20:21:55.993Z",
"version": "6.1.6"
}
{
"category": "APP",
"errors": [
{
"attributes": {
"genre": "TECHNICAL",
"http_status": 500,
"referer": "https://intel/dashboard/data/import"
},
"message": "Http call interceptor fail",
"name": "UNKNOWN_ERROR",
"stack": "UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:455:18)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"
},
{
"message": "stream is not readable",
"name": "InternalServerError",
"stack": "InternalServerError: stream is not readable\n at readStream (/opt/opencti/build/node_modules/raw-body/index.js:185:17)\n at getBody (/opt/opencti/build/node_modules/raw-body/index.js:116:12)\n at read (/opt/opencti/build/node_modules/body-parser/lib/read.js:79:3)\n at fn (/opt/opencti/build/node_modules/body-parser/lib/types/json.js:138:5)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at cors (/opt/opencti/build/node_modules/cors/lib/index.js:188:7)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:224:17)\n at originCallback (/opt/opencti/build/node_modules/cors/lib/index.js:214:15)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:219:13)\n at optionsCallback (/opt/opencti/build/node_modules/cors/lib/index.js:199:9)\n at fn (/opt/opencti/build/node_modules/cors/lib/index.js:204:7)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at Function.handle (/opt/opencti/build/node_modules/express/lib/router/index.js:175:3)\n at router (/opt/opencti/build/node_modules/express/lib/router/index.js:47:12)"
}
],
"level": "error",
"message": "Http call interceptor fail",
"source": "backend",
"timestamp": "2024-06-17T20:22:02.012Z",
"version": "6.1.6"
} |
|
Do you still see this happening? |
I do, it seems to be every few hours roughly, although the platform is not unusable:
|
|
Can you check the CPU usage of the platform? Do you have 100% cpu usage? Thanks |
Total CPU usage is currently around ~50%. Platform showing >98%, sometimes over 100% usage. Where should I look for total open sessions? There's quite a few for the users going back a few days. The platform is stable but every now and then it does produce these errors, I suspect when many bundles are being processed, or maybe there's some other issue. |
|
Hello {"category":"APP","codeStack":"vQe@https://[...]/static/js/front-43R7MMGU.js:91:66684
render@https://[...]/static/js/front-43R7MMGU.js:91:67547\nN8i</a.render@https://[...]/static/js/front-43R7MMGU.js:49:38490...I was trying to make better firewall rules, but forgot the rabbitmq service... Found out this error (seconds after the first one) "message":"connect ETIMEDOUT 172.31.16.223:15671","operation":"Unspecified","query_attributes":[[{"arguments":[],"name":"rabbitMQMetrics"}]],"size":2,"source":"backend","time":129561,"timestamp":"2024-09-03T15:59:37.133Z","type":"READ_ERROR"Changed the RabbitMQ management port firewall rule, and now my opencti is up ! @dominictory, look at your connections with rabbitmq (both RABBITMQ__PORT, RABBITMQ__PORT_MANAGEMENT) |
|
@dominictory do you still have the issue? |
Yes, however not very often now. My platform is relatively stable. Last time I saw this was 2 days ago: |
|
@nino-filigran Hi, after upgrading from 6.2.18 to 6.3.1, I am seeing a lot of more of these errors. New logs below: As well as: |
|
Also the below almost immediately after requesting a page |
|
You have some "message":"Request timed out" in your logs, so maybe related to a problem of connectivity or an overloaded elastic? |
Hi, you mentioned open sessions before, should I check this per user, since I have a user per connector? |
Even if i turn off every connectors to stop overloading elastic, it still times out (message":"Request timed out). But most of the time error logs occur like this; opencti-worker; opencti-server; |
|
Also seeing this, the "node build/back.js" process goes to 100% CPU usage and the platform stops responding for users. |
|
Our platform is very unstable at the moment with these issues, on latest version 6.3.6. |
|
Hi, i found the same issue on my platform, this the error logs, (running on docker [opencti/platform:6.3.5]) |
|
@nauliajati that looks like an error caused by a missing file and not the same as this. |
|
I have determined that what triggers our issues is connectors ingesting large amounts of data. |
|
@dominictory, @nauliajati @simonbjorzen-ts have you all identified the root cause of the problem, leading to having HTTP call interceptor fail error? The rationale for this request is that if an HTTP call interceptor fails, something goes wrong. If we manage to identify & fix what's causing the problem, you won't get the HTTP call interceptor. |
I've collected and sorted all the different variations of this error. It appears the bulk of them are |
|
Ok lets try to answer this ticket :) Now lets explain all the different error in this ticket that finised in a Interceptor fail error
Its a request cancelled by the client. Its not really an error on the server side but a consequence of request cancelling. Maybe related to client cancelling the request because the platform is too long to answer so more a consequence on the client based on a platform that have some performance issue.
If the file is not accessible for some reason we throw an error that is then catch and log by the generic interceptor. For me we should not propagate the error to make the situation clearer.
This one is more related to a problem in code that hide the real problem has we try to modify the response headers after his generation. If you have a repro case on this one, please open a dedicated ticket.
Looks like related to user session but the error is really unclear. Like the previous one, If you have a repro case on this one, please open a dedicated ticket. I hope my explanation make the situation clearer for you. I will close this ticket as it started to cumulate too much different potential problems. Please open dedicated ticket for each problem if you manage to have a reprocase you can share. At the same time I will work on #8870 to cleanup the situation. |
richard-julien
added
question
Thanks! Will revisit my issues and see if can get a repro case. My main issue at the moment is #8846 |
Description
Whilst resource appears fine for my stack, CTI regularly hangs, and the platform often presents me with 'an unknown error has occurred'. I am seeing a lot of HTTP call interceptor fail errors in the logs, as below:
ory":"APP","errors":[{"attributes":{"genre":"TECHNICAL","http_status":500},"message":"Http call interceptor fail","name":"UNKNOWN_ERROR","stack":"UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:455:18)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"},{"message":"request aborted","name":"BadRequestError","stack":"BadRequestError: request aborted\n at IncomingMessage.onAborted (/opt/opencti/build/node_modules/raw-body/index.js:245:10)\n at IncomingMessage.emit (node:events:519:28)\n at IncomingMessage._destroy (node:_http_incoming:224:10)\n at _destroy (node:internal/streams/destroy:121:10)\n at IncomingMessage.destroy (node:internal/streams/destroy:83:5)\n at abortIncoming (node:_http_server:797:9)\n at socketOnClose (node:_http_server:791:3)\n at TLSSocket.emit (node:events:531:35)\n at node:net:338:12\n at Socket.done (node:_tls_wrap:657:7)\n at Object.onceWrapper (node:events:634:26)\n at Socket.emit (node:events:531:35)\n at TCP.<anonymous> (node:net:338:12)"}],"level":"error","message":"Http call interceptor fail","source":"backend","timestamp":"2024-06-13T09:30:10.272Z","version":"6.1.10"}Another error I'm seeing that I've not seen before is as below:
Environment
OpenCTI 6.1.10
Ubuntu 22.04 LTS
Docker
Additional information
Also seeing a lot of these now:
(node:7) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 abort listeners added to [EventEmitter]. Use emitter.setMaxListeners() to increase limitOther notable errors in RabbitMQ:
The text was updated successfully, but these errors were encountered: