Hi,
Good day to you…
We have a REDIS Cluster, having 2 nodes.
The respective AWS Redis Node Type is: cache.m3.medium
TYK Pump is pulling data from the above redis cluster and pushing into AWS ElasticSearch.
The mentioned application is working properly. In the tykpump.log, we are able to see how many no of records are getting pushed from Redis to AWS ElasticSearch.
But still, the BytesUsedForCache for the above REDIS cluster is getting increased everyday.
We have a limit of No of Bytes, being used by the TYK Redis database, as 400 MB only.
Beyond that limit, we can not allow Redis no of consumed bytes to be increased.
Below is our pump.conf configuration file:
{
“analytics_storage_type”: “redis”,
“analytics_storage_config”: {
“type”: “redis”,
“host”: “ext-tyk-redis.cexboz.ng.0001.usw2.cache.amazonaws.com”,
“port”: 6379,
“hosts”: null,
“username”: “xxxxxx”,
“password”: “xxxxxxxx”,
“database”: 4,
“optimisation_max_idle”: 100,
“optimisation_max_active”: 0,
“enable_cluster”: false,
“retry_delay”: 10,
“retry_frequency”: 5
},
“purge_delay”: 60,
“pumps”: {
“csv”: {
“name”: “csv”,
“meta”: {
“csv_dir”: “/opt/tyk-pump-csv/”
}
},
“awses”: {
“name”: “apigateway-analytics”,
“meta”: {
“domain”: “apigateway-analytics”,
“index_name”: “prod_ext_apigateway_analytics”,
“elasticsearch_url”: “https://xxxx-analytics.us-west-2.es.amazonaws.com”,
“region”: “us-west-2”,
“document_type”: “tyk_analytics”,
“rolling_index”: true,
“chunk_size”: 500
}
}
},
“dont_purge_uptime_data”: true,
“logpath”: “/var/log/tyk-pump.log”
}
Below is our tyk_analytics.conf configuration file:
{
“listen_port”: 3000,
“notifications_listen_port”: 5000,
“tyk_api_config”: {
“Host”: “https://apigateway.juniper.net”,
“Port”: “443”,
“Secret”: “xxxxx”
},
“mongo_url”: “mongodb://xxxxxxx/apigateway-prod”,
“page_size”: 10,
“admin_secret”: “xxxxx”,
“shared_node_secret”: “xxxxxx”,
“redis_port”: 6379,
“redis_host”: “ext-tyk-redis.cexboz.ng.0001.usw2.cache.amazonaws.com”,
“redis_password”: “xxxxxx”,
“enable_cluster”: false,
“force_api_defaults”: false,
“notify_on_change”: true,
“license_key”: “yyyyyyyyyyyyy”,
“redis_database”: 4,
“redis_hosts”: null,
“hash_keys”: true,
“email_backend”: {
“enable_email_notifications”: true,
“code”: “amazonses”,
“settings”: {
“Endpoint”: “https://email.us-west-2.amazonaws.com”,
“AccessKeyId”: “xxxxxx”,
“SecretAccessKey”: “xxxxxxxx”
},
“default_from_email”: “[email protected]”,
“default_from_name”: “Juniper Tyk Admins”
},
“hide_listen_path”: false,
“sentry_code”: “”,
“sentry_js_code”: “”,
“use_sentry”: false,
“enable_master_keys”: false,
“enable_duplicate_slugs”: true,
“show_org_id”: true,
“host_config”: {
“override_hostname”: “”,
“disable_org_slug_prefix”: true,
“enable_host_names”: false,
“hostname”: “”,
“portal_domains”: {},
“portal_root_path”: “/portal”
},
“http_server_options”: {
“use_ssl”: false,
“certificates”: [
{
“domain_name”: “”,
“cert_file”: “”,
“key_file”: “”
}
],
“min_version”: 0
},
“ui”: {
“login_page”: {},
“nav”: {},
“uptime”: {},
“portal_section”: null,
“designer”: {},
“dont_show_admin_sockets”: false,
“dont_allow_license_management”: false,
“dont_allow_license_management_view”: false
},
“home_dir”: “/opt/tyk-dashboard”,
“identity_broker”: {
“enabled”: false,
“host”: {
“connection_string”: “”,
“secret”: “”
}
},
“tagging_options”: {
“tag_all_apis_by_org”: false
}
}
Below is our tyk.conf configuration file:
{
“listen_port”: 8080,
“secret”: “xxxxx”,
“node_secret”: “xxxx”,
“template_path”: “/opt/tyk-gateway/templates”,
“middleware_path”: “/opt/tyk-gateway/middleware”,
“use_db_app_configs”: true,
“db_app_conf_options”: {
“connection_string”: “https://tykdashboard.junipercloud.net”,
“node_is_segmented”: false,
“tags”: [“External Tyk Prod”]
},
“app_path”: “/opt/tyk-gateway/apps/”,
“storage”: {
“type”: “redis”,
“host”: “ext-tyk-redis.cexboz.ng.0001.usw2.cache.amazonaws.com”,
“port”: 6379,
“username”: “”,
“password”: “”,
“database”: 4,
“optimisation_max_idle”: 2000,
“optimisation_max_active”: 4000
},
“oauth_token_expired_retain_period”: 300,
“enable_analytics”: true,
“analytics_config”: {
“enable_detailed_recording”: false
},
“health_check”: {
“enable_health_checks”: false,
“health_check_value_timeouts”: 60
},
“uptime_tests”: {
“disable”: true
},
“optimisations_use_async_session_write”: true,
“enable_non_transactional_rate_limiter”: true,
“enable_sentinel_rate_limiter”: false,
“allow_master_keys”: false,
“policies”: {
“policy_source”: “service”,
“policy_connection_string”: “https://tykdashboard.junipercloud.net”
},
“hash_keys”: true,
“close_connections”: false,
“allow_insecure_configs”: true,
“coprocess_options”: {
“enable_coprocess”: false,
“coprocess_grpc_server”: “”
},
“enable_bundle_downloader”: true,
“bundle_base_url”: “”,
“global_session_lifetime”: 100,
“force_global_session_lifetime”: false,
“max_idle_connections_per_host”: 500,
“enable_custom_domains”: true,
“http_server_options”: {
“enable_websockets”: true,
“use_ssl”: true,
“ssl_ciphers”: [
“TLS_RSA_WITH_AES_128_GCM_SHA256”,
“TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA”
],
“certificates”: [{
“domain_name”: “apigateway-mtls.juniper.net”,
“cert_file”: “/opt/tyk-gateway/mtls-cert.pem”,
“key_file”: “/opt/tyk-gateway/mtls-key.pem”
}, {
“domain_name”: “apigateway.juniper.net”,
“cert_file”: “/opt/tyk-gateway/cert.pem”,
“key_file”: “/opt/tyk-gateway/key.pem”
}
],
“min_version”: 771,
“flush_interval”: 1,
“skip_target_path_escaping”: true
},
“enable_key_logging”: true,
“log_level”: “info”
}
Can you please help to how we can reduce the No. of Bytes, getting consumed by REDIS database ?
Also, how can we confirm that the analytics data are being properly being pulled from the Redis and for the analytics data itself, the no of bytes consumed by the Redis are not getting increased ?
Thanks,
Meghna Bhattacharya