After enabling of tls in our TYK server it runs out of filehandles and stop working

We have enabled tls in our on premise TYK dashboard and gateway after that it runs out of filehandles and stop working

Redirecting to /bin/systemctl status tyk-dashboard.service
● tyk-dashboard.service - Tyk API Dashboard
Loaded: loaded (/usr/lib/systemd/system/tyk-dashboard.service; enabled; vendor preset: disabled)
Active: active (running) since Thu 2018-10-18 11:24:32 CEST; 3h 57min ago
Main PID: 13062 (tyk-analytics)
CGroup: /system.slice/tyk-dashboard.service
└─13062 /opt/tyk-dashboard/tyk-analytics --conf /opt/tyk-dashboard/tyk_analytics.conf

Oct 18 15:22:00 sktudv01tyk01.ccta.dk tyk-analytics[13062]: 2018/10/18 15:22:00 http: Accept error: accept tcp [::]:3000: accept4: too many open files; retrying in 1s
Oct 18 15:22:01 sktudv01tyk01.ccta.dk tyk-analytics[13062]: 2018/10/18 15:22:01 http: Accept error: accept tcp [::]:3000: accept4: too many open files; retrying in 1s

our conf files looks like this

tyk.conf
{
“listen_port”: 8443,
“node_secret”: “secret”,
“secret”: “secret”,
“template_path”: “/opt/tyk-gateway/templates”,
“use_db_app_configs”: true,
“db_app_conf_options”: {
“connection_string”: “”,
“node_is_segmented”: false,
“tags”: []
},
“disable_dashboard_zeroconf”: false,
“app_path”: “/opt/tyk-gateway/apps”,
“middleware_path”: “/opt/tyk-gateway/middleware”,
“storage”: {
“type”: “redis”,
“host”: “localhost”,
“port”: 6379,
“username”: “”,
“password”: “”,
“database”: 0,
“optimisation_max_idle”: 2000,
“optimisation_max_active”: 4000
},
“enable_analytics”: true,
“analytics_config”: {
“type”: “”,
“ignored_ips”: [],
“enable_detailed_recording”: true,
“enable_geo_ip”: false,
“geo_ip_db_path”: “”,
“normalise_urls”: {
“enabled”: true,
“normalise_uuids”: true,
“normalise_numbers”: true,
“custom_patterns”: []
}
},
“health_check”: {
“enable_health_checks”: false,
“health_check_value_timeouts”: 60
},
“optimisations_use_async_session_write”: true,
“allow_master_keys”: false,
“policies”: {
“policy_source”: “service”,
“policy_connection_string”: “”,
“policy_record_name”: “tyk_policies”,
“allow_explicit_policy_id”: true
},
“hash_keys”: true,
“suppress_redis_signal_reload”: false,
“use_redis_log”: true,
“close_connections”: true,
“enable_non_transactional_rate_limiter”: true,
“enable_sentinel_rate_limiter”: false,
“experimental_process_org_off_thread”: false,
“local_session_cache”: {
“disable_cached_session_state”: false
},
“http_server_options”: {
“enable_websockets”: true,
“use_ssl”: true,
“server_name”: “.ccta.dk",
“certificates”: [
{
“domain_name”: "
.ccta.dk”,
“cert_file”: “/etc/pki/tls/certs/dev.api.data.ccta.dk.crt”,
“key_file”: “/etc/pki/tls/private/dev.api.data.ccta.dk.key”

}
],
“ssl_insecure_skip_verify”: true

},
“uptime_tests”: {
“disable”: false,
“config”: {
“enable_uptime_analytics”: true,
“failure_trigger_sample_size”: 2,
“time_wait”: 10,
“checker_pool_size”: 50
}
},
“hostname”: “”,
“enable_custom_domains”: true,
“enable_jsvm”: true,
“oauth_redirect_uri_separator”: “;”,
“coprocess_options”: {
“enable_coprocess”: false,
“coprocess_grpc_server”: “”
},
“pid_file_location”: “./tyk-gateway.pid”,
“allow_insecure_configs”: true,
“public_key_path”: “”,
“close_idle_connections”: false,
“allow_remote_config”: false,
“enable_bundle_downloader”: true,
“bundle_base_url”: “”,
“global_session_lifetime”: 100,
“force_global_session_lifetime”: false,
“max_idle_connections_per_host”: 500
}

tyk_analytics.conf
{
“listen_port”: 3000,
“tyk_api_config”: {
“Host”: “http://localhost”,
“Port”: “8080”,
“Secret”: “secret”
},
“mongo_url”: “mongodb://127.0.0.1/tyk_analytics”,
“mongo_use_ssl”: false,
“mongo_ssl_insecure_skip_verify”: false,
“page_size”: 10,
“admin_secret”: “12345”,
“shared_node_secret”: “secret”,
“redis_port”: 6379,
“redis_host”: “localhost”,
“redis_password”: “”,
“enable_cluster”: false,
“redis_use_ssl”: false,
“redis_ssl_insecure_skip_verify”: false,
“force_api_defaults”: false,
“notify_on_change”: true,
“license_key”: “secret”,
“redis_database”: 0,
“redis_hosts”: null,
“hash_keys”: true,
“email_backend”: {
“enable_email_notifications”: false,
“code”: “”,
“settings”: null,
“default_from_email”: “”,
“default_from_name”: “”,
“dashboard_hostname”: “”
},
“hide_listen_path”: false,
“sentry_code”: “”,
“sentry_js_code”: “”,
“use_sentry”: false,
“enable_master_keys”: false,
“enable_duplicate_slugs”: true,
“show_org_id”: true,
“host_config”: {
“enable_host_names”: true,
“disable_org_slug_prefix”: true,
“hostname”: “localhost”,
“override_hostname”: “localhost”,
“portal_domains”: {},
“portal_root_path”: “/portal”,
“generate_secure_paths”: false,
“secure_cookies”: false,
“use_strict_hostmatch”: false
},
“http_server_options”: {
“use_ssl”: true,
“servername”: “localhost”,
“certificates”: [
{
“domain_name”: “dev.api.data.ccta.dk”,
“cert_file”: “/etc/pki/tls/certs/dev.api.data.ccta.dk.crt”,
“key_file”: “/etc/pki/tls/private/dev.api.data.ccta.dk.key”
}
],
“min_version”: 0
},
“security”: {
“allow_admin_reset_password”: false,
“login_failure_username_limit”: 0,
“login_failure_ip_limit”: 0,
“login_failure_expiration”: 0,
“audit_log_path”: “/var/log/tyk/tyk-audit.log”
},
“ui”: {
“languages”: {
“Chinese”: “cn”,
“English”: “en”,
“French”: “fr”,
“Korean”: “ko”
},
“hide_help”: false,
“default_lang”: “en”,
“login_page”: {},
“nav”: {},
“uptime”: {},
“portal_section”: null,
“designer”: {},
“dont_show_admin_sockets”: false,
“dont_allow_license_management”: false,
“dont_allow_license_management_view”: false,
“cloud”: false
},
“home_dir”: “/opt/tyk-dashboard”,
“identity_broker”: {
“enabled”: false,
“host”: {
“connection_string”: “http://localhost:3010”,
“secret”: “secret”
}
},
“tagging_options”: {
“tag_all_apis_by_org”: false
},
“use_sharded_analytics”: false,
“enable_aggregate_lookups”: true,
“enable_analytics_cache”: false,
“aggregate_lookup_cutoff”: “01/07/2016”,
“maintenance_mode”: false,
“allow_explicit_policy_id”: false,
“private_key_path”: “”,
“node_schema_path”: “”,
“oauth_redirect_uri_separator”: “;”,
“statsd_connection_string”: “”,
“statsd_prefix”: “”,
“disable_parallel_sessions”: false,
“dashboard_session_lifetime”: 0,
“alternative_dashboard_url”: “”,
“sso_permission_defaults”: null,
“sso_default_group_id”: “”,
“sso_custom_login_url”: “”,
“sso_custom_portal_login_url”: “”,
“notifications_listen_port”: 5000,
“portal_session_lifetime”: 0,
“enable_delete_key_by_hash”: false

  our file handle conf 
  
  # /etc/security/limits.conf              

  *       hard    maxlogins       10       
  *          soft     nproc          80000 
  *          hard     nproc          80000 
  *          soft     nofile         80000 
  *          hard     nofile         80000 
  root       soft     nproc          80000 
  root       hard     nproc          80000 
  root       soft     nofile         80000 
  root       hard     nofile         80000 

and in sysctl.conf i added

fs.file-max=80000

I have changed so the acual service now have a limit for 80000 files, and thats works, but my tyk services keep open new conn files

lsof | grep tyk | wc -l
59085

lsof | grep 13607

t yk 13607 13627 root 928u IPv4 621783 0t0 TCP sktudv01tyk01.ccta.dk:35826->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 929u IPv4 621784 0t0 TCP sktudv01tyk01.ccta.dk:35828->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 930u IPv4 621078 0t0 TCP sktudv01tyk01.ccta.dk:35830->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 931u IPv4 616730 0t0 TCP sktudv01tyk01.ccta.dk:35832->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 932u IPv4 613019 0t0 TCP sktudv01tyk01.ccta.dk:35834->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED):heart:

[[email protected] tyk-pump]# cat /proc/13607/limits
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 31191 31191 processes
Max open files 80000 80000 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 31191 31191 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
[[email protected] tyk-pump]#

When i start the gateway and pump up it dont creates new handles, but when i starts the dashboard up i starts create new handles every second.

I am doing a Proof of concept on TYK for the danish government and need this to work if we should decide to go on with tyk gateway.

I am seeing this problem now as well after updating to the latest Tyk dashboard packages last night. Did you ever find a resolution?

Hi franfabrizio
Unfortunately i didn’t find any solutions,
But my systems is stabilized with about 650000 open files ( lsof | grep tyk | wc -l )

I need to set my tyk-gateway and tyk-dashboard services to allow 80000 max open files.
In the my tyk services conf located here

/etc/systemd/system/multi-user.target.wants/

I suspect this to either be a bug or work by design which i hope it is not.

I have contacted TYK about this issue and await them to answer

Hi
We found an issue where connections could leak between gateways and dashboard when they are using TLS. We have found a fix and it should be fixed in our next patch - currently scheduled for middle of next week.

Thanks