We have enabled tls in our on premise TYK dashboard and gateway after that it runs out of filehandles and stop working
Redirecting to /bin/systemctl status tyk-dashboard.service
● tyk-dashboard.service - Tyk API Dashboard
Loaded: loaded (/usr/lib/systemd/system/tyk-dashboard.service; enabled; vendor preset: disabled)
Active: active (running) since Thu 2018-10-18 11:24:32 CEST; 3h 57min ago
Main PID: 13062 (tyk-analytics)
CGroup: /system.slice/tyk-dashboard.service
└─13062 /opt/tyk-dashboard/tyk-analytics --conf /opt/tyk-dashboard/tyk_analytics.conf
Oct 18 15:22:00 sktudv01tyk01.ccta.dk tyk-analytics[13062]: 2018/10/18 15:22:00 http: Accept error: accept tcp [::]:3000: accept4: too many open files; retrying in 1s
Oct 18 15:22:01 sktudv01tyk01.ccta.dk tyk-analytics[13062]: 2018/10/18 15:22:01 http: Accept error: accept tcp [::]:3000: accept4: too many open files; retrying in 1s
our conf files looks like this
tyk.conf
{
“listen_port”: 8443,
“node_secret”: “secret”,
“secret”: “secret”,
“template_path”: “/opt/tyk-gateway/templates”,
“use_db_app_configs”: true,
“db_app_conf_options”: {
“connection_string”: “”,
“node_is_segmented”: false,
“tags”: []
},
“disable_dashboard_zeroconf”: false,
“app_path”: “/opt/tyk-gateway/apps”,
“middleware_path”: “/opt/tyk-gateway/middleware”,
“storage”: {
“type”: “redis”,
“host”: “localhost”,
“port”: 6379,
“username”: “”,
“password”: “”,
“database”: 0,
“optimisation_max_idle”: 2000,
“optimisation_max_active”: 4000
},
“enable_analytics”: true,
“analytics_config”: {
“type”: “”,
“ignored_ips”: [],
“enable_detailed_recording”: true,
“enable_geo_ip”: false,
“geo_ip_db_path”: “”,
“normalise_urls”: {
“enabled”: true,
“normalise_uuids”: true,
“normalise_numbers”: true,
“custom_patterns”: []
}
},
“health_check”: {
“enable_health_checks”: false,
“health_check_value_timeouts”: 60
},
“optimisations_use_async_session_write”: true,
“allow_master_keys”: false,
“policies”: {
“policy_source”: “service”,
“policy_connection_string”: “”,
“policy_record_name”: “tyk_policies”,
“allow_explicit_policy_id”: true
},
“hash_keys”: true,
“suppress_redis_signal_reload”: false,
“use_redis_log”: true,
“close_connections”: true,
“enable_non_transactional_rate_limiter”: true,
“enable_sentinel_rate_limiter”: false,
“experimental_process_org_off_thread”: false,
“local_session_cache”: {
“disable_cached_session_state”: false
},
“http_server_options”: {
“enable_websockets”: true,
“use_ssl”: true,
“server_name”: “.ccta.dk",
“certificates”: [
{
“domain_name”: ".ccta.dk”,
“cert_file”: “/etc/pki/tls/certs/dev.api.data.ccta.dk.crt”,
“key_file”: “/etc/pki/tls/private/dev.api.data.ccta.dk.key”
}
],
“ssl_insecure_skip_verify”: true
},
“uptime_tests”: {
“disable”: false,
“config”: {
“enable_uptime_analytics”: true,
“failure_trigger_sample_size”: 2,
“time_wait”: 10,
“checker_pool_size”: 50
}
},
“hostname”: “”,
“enable_custom_domains”: true,
“enable_jsvm”: true,
“oauth_redirect_uri_separator”: “;”,
“coprocess_options”: {
“enable_coprocess”: false,
“coprocess_grpc_server”: “”
},
“pid_file_location”: “./tyk-gateway.pid”,
“allow_insecure_configs”: true,
“public_key_path”: “”,
“close_idle_connections”: false,
“allow_remote_config”: false,
“enable_bundle_downloader”: true,
“bundle_base_url”: “”,
“global_session_lifetime”: 100,
“force_global_session_lifetime”: false,
“max_idle_connections_per_host”: 500
}
tyk_analytics.conf
{
“listen_port”: 3000,
“tyk_api_config”: {
“Host”: “http://localhost”,
“Port”: “8080”,
“Secret”: “secret”
},
“mongo_url”: “mongodb://127.0.0.1/tyk_analytics”,
“mongo_use_ssl”: false,
“mongo_ssl_insecure_skip_verify”: false,
“page_size”: 10,
“admin_secret”: “12345”,
“shared_node_secret”: “secret”,
“redis_port”: 6379,
“redis_host”: “localhost”,
“redis_password”: “”,
“enable_cluster”: false,
“redis_use_ssl”: false,
“redis_ssl_insecure_skip_verify”: false,
“force_api_defaults”: false,
“notify_on_change”: true,
“license_key”: “secret”,
“redis_database”: 0,
“redis_hosts”: null,
“hash_keys”: true,
“email_backend”: {
“enable_email_notifications”: false,
“code”: “”,
“settings”: null,
“default_from_email”: “”,
“default_from_name”: “”,
“dashboard_hostname”: “”
},
“hide_listen_path”: false,
“sentry_code”: “”,
“sentry_js_code”: “”,
“use_sentry”: false,
“enable_master_keys”: false,
“enable_duplicate_slugs”: true,
“show_org_id”: true,
“host_config”: {
“enable_host_names”: true,
“disable_org_slug_prefix”: true,
“hostname”: “localhost”,
“override_hostname”: “localhost”,
“portal_domains”: {},
“portal_root_path”: “/portal”,
“generate_secure_paths”: false,
“secure_cookies”: false,
“use_strict_hostmatch”: false
},
“http_server_options”: {
“use_ssl”: true,
“servername”: “localhost”,
“certificates”: [
{
“domain_name”: “dev.api.data.ccta.dk”,
“cert_file”: “/etc/pki/tls/certs/dev.api.data.ccta.dk.crt”,
“key_file”: “/etc/pki/tls/private/dev.api.data.ccta.dk.key”
}
],
“min_version”: 0
},
“security”: {
“allow_admin_reset_password”: false,
“login_failure_username_limit”: 0,
“login_failure_ip_limit”: 0,
“login_failure_expiration”: 0,
“audit_log_path”: “/var/log/tyk/tyk-audit.log”
},
“ui”: {
“languages”: {
“Chinese”: “cn”,
“English”: “en”,
“French”: “fr”,
“Korean”: “ko”
},
“hide_help”: false,
“default_lang”: “en”,
“login_page”: {},
“nav”: {},
“uptime”: {},
“portal_section”: null,
“designer”: {},
“dont_show_admin_sockets”: false,
“dont_allow_license_management”: false,
“dont_allow_license_management_view”: false,
“cloud”: false
},
“home_dir”: “/opt/tyk-dashboard”,
“identity_broker”: {
“enabled”: false,
“host”: {
“connection_string”: “http://localhost:3010”,
“secret”: “secret”
}
},
“tagging_options”: {
“tag_all_apis_by_org”: false
},
“use_sharded_analytics”: false,
“enable_aggregate_lookups”: true,
“enable_analytics_cache”: false,
“aggregate_lookup_cutoff”: “01/07/2016”,
“maintenance_mode”: false,
“allow_explicit_policy_id”: false,
“private_key_path”: “”,
“node_schema_path”: “”,
“oauth_redirect_uri_separator”: “;”,
“statsd_connection_string”: “”,
“statsd_prefix”: “”,
“disable_parallel_sessions”: false,
“dashboard_session_lifetime”: 0,
“alternative_dashboard_url”: “”,
“sso_permission_defaults”: null,
“sso_default_group_id”: “”,
“sso_custom_login_url”: “”,
“sso_custom_portal_login_url”: “”,
“notifications_listen_port”: 5000,
“portal_session_lifetime”: 0,
“enable_delete_key_by_hash”: false
our file handle conf
# /etc/security/limits.conf
* hard maxlogins 10
* soft nproc 80000
* hard nproc 80000
* soft nofile 80000
* hard nofile 80000
root soft nproc 80000
root hard nproc 80000
root soft nofile 80000
root hard nofile 80000
and in sysctl.conf i added
fs.file-max=80000
I have changed so the acual service now have a limit for 80000 files, and thats works, but my tyk services keep open new conn files
lsof | grep tyk | wc -l
59085
lsof | grep 13607
t yk 13607 13627 root 928u IPv4 621783 0t0 TCP sktudv01tyk01.ccta.dk:35826->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 929u IPv4 621784 0t0 TCP sktudv01tyk01.ccta.dk:35828->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 930u IPv4 621078 0t0 TCP sktudv01tyk01.ccta.dk:35830->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 931u IPv4 616730 0t0 TCP sktudv01tyk01.ccta.dk:35832->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
tyk 13607 13627 root 932u IPv4 613019 0t0 TCP sktudv01tyk01.ccta.dk:35834->sktudv01tyk01.ccta.dk:hbci (ESTABLISHED)
[root@sktudv01tyk01 tyk-pump]# cat /proc/13607/limits
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 31191 31191 processes
Max open files 80000 80000 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 31191 31191 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
[root@sktudv01tyk01 tyk-pump]#
When i start the gateway and pump up it dont creates new handles, but when i starts the dashboard up i starts create new handles every second.
I am doing a Proof of concept on TYK for the danish government and need this to work if we should decide to go on with tyk gateway.