Tyk round robin passes request to a "down" upstream server

I have four nginx containers running on the same docker network as tyk.
Each nginx instance hosts a page /hello.html which simply identifies the server it was served from.
like “Hello from Server 1”, "“Hello from Server 2” etc.

When the nginx instance are working, and I repeatedly access the /testapi/hello.html, the responses come as expected from different nginx instances in a round robin fashion.

Now if I stop one of the four nginx instances, I get the following message approximately every fourth time I send the request. :
"error": "There was a problem proxying the request"

tyk never seems to learn that the server is down. Ideally, it should mark the sever as down and round robin between the remaining three servers.

I have also added uptime tests. I have basically added the URL to index.html on each nginx instance. I suppose, the HTTP Response code of 200 when accessing those URLs would be enough to confirm their health.

Also, if a server is down, the health check should fail and that server should not be sent any requests till health checks pass.

But tyk still keeps attempting to serve user requests from the down server.

The API Definition:

{
“api_model”: {},
“api_definition”: {
“api_id”: “21d06a908ff342676e66f593964b4723”,
“jwt_issued_at_validation_skew”: 0,
“upstream_certificates”: {},
“use_keyless”: false,
“enable_coprocess_auth”: false,
“base_identity_provided_by”: “”,
“custom_middleware”: {
“pre”: [],
“post”: [],
“post_key_auth”: [],
“auth_check”: {
“name”: “”,
“path”: “”,
“require_session”: false
},
“response”: [],
“driver”: “”,
“id_extractor”: {
“extract_from”: “”,
“extract_with”: “”,
“extractor_config”: {}
}
},
“disable_quota”: true,
“custom_middleware_bundle”: “”,
“cache_options”: {
“cache_timeout”: 60,
“enable_cache”: true,
“cache_all_safe_requests”: false,
“cache_response_codes”: [],
“enable_upstream_cache_control”: false,
“cache_control_ttl_header”: “”
},
“enable_ip_blacklisting”: false,
“tag_headers”: [],
“pinned_public_keys”: {},
“expire_analytics_after”: 0,
“domain”: “”,
“openid_options”: {
“providers”: [],
“segregate_by_client”: false
},
“jwt_policy_field_name”: “”,
“jwt_default_policies”: [],
“active”: true,
“jwt_expires_at_validation_skew”: 0,
“config_data”: {},
“notifications”: {
“shared_secret”: “”,
“oauth_on_keychange_url”: “”
},
“jwt_client_base_field”: “”,
“auth”: {
“use_param”: false,
“param_name”: “”,
“use_cookie”: false,
“cookie_name”: “”,
“auth_header_name”: “Authorization”,
“use_certificate”: false,
“validate_signature”: false,
“signature”: {
“algorithm”: “”,
“header”: “”,
“secret”: “”,
“allowed_clock_skew”: 0,
“error_code”: 0,
“error_message”: “”
}
},
“check_host_against_uptime_tests”: true,
“auth_provider”: {
“name”: “”,
“storage_engine”: “”,
“meta”: {}
},
“blacklisted_ips”: [],
“hmac_allowed_clock_skew”: -1,
“dont_set_quota_on_create”: false,
“uptime_tests”: {
“check_list”: [
{
“body”: “”,
“headers”: {},
“method”: “GET”,
“url”: “http://n1/index.html
},
{
“body”: “”,
“headers”: {},
“method”: “GET”,
“url”: “http://n2/index.html
},
{
“body”: “”,
“headers”: {},
“method”: “GET”,
“url”: “http://n3/index.html
},
{
“body”: “”,
“headers”: {},
“method”: “GET”,
“url”: “http://n4/index.html
}
],
“config”: {
“expire_utime_after”: 0,
“service_discovery”: {
“use_discovery_service”: false,
“query_endpoint”: “”,
“use_nested_query”: false,
“parent_data_path”: “”,
“data_path”: “”,
“cache_timeout”: 60
},
“recheck_wait”: 0
}
},
“enable_jwt”: false,
“do_not_track”: false,
“name”: “testapi”,
“slug”: “billdesk”,
“oauth_meta”: {
“allowed_access_types”: [],
“allowed_authorize_types”: [],
“auth_login_redirect”: “”
},
“CORS”: {
“enable”: false,
“max_age”: 24,
“allow_credentials”: false,
“exposed_headers”: [],
“allowed_headers”: [],
“options_passthrough”: false,
“debug”: false,
“allowed_origins”: [],
“allowed_methods”: []
},
“event_handlers”: {
“events”: {}
},
“proxy”: {
“target_url”: “”,
“service_discovery”: {
“endpoint_returns_list”: false,
“cache_timeout”: 0,
“parent_data_path”: “”,
“query_endpoint”: “”,
“use_discovery_service”: false,
“_sd_show_port_path”: false,
“target_path”: “”,
“use_target_list”: false,
“use_nested_query”: false,
“data_path”: “”,
“port_data_path”: “”
},
“check_host_against_uptime_tests”: true,
“transport”: {
“ssl_insecure_skip_verify”: false,
“ssl_ciphers”: [],
“ssl_min_version”: 0,
“proxy_url”: “”
},
“target_list”: [
http://n1/”,
http://n2/”,
http://n3/”,
http://n4/
],
“preserve_host_header”: false,
“strip_listen_path”: true,
“enable_load_balancing”: true,
“listen_path”: “/testapi/”,
“disable_strip_slash”: false
},
“client_certificates”: [],
“use_basic_auth”: false,
“version_data”: {
“not_versioned”: true,
“default_version”: “”,
“versions”: {
“Default”: {
“name”: “Default”,
“expires”: “”,
“paths”: {
“ignored”: [],
“white_list”: [],
“black_list”: []
},
“use_extended_paths”: true,
“extended_paths”: {
“ignored”: [],
“white_list”: [],
“black_list”: [],
“transform”: [],
“transform_response”: [],
“transform_jq”: [],
“transform_jq_response”: [],
“transform_headers”: [],
“transform_response_headers”: [],
“hard_timeouts”: [],
“circuit_breakers”: [],
“url_rewrites”: [],
“virtual”: [],
“size_limits”: [],
“method_transforms”: [],
“track_endpoints”: [],
“do_not_track_endpoints”: [],
“validate_json”: [],
“internal”: []
},
“global_headers”: {},
“global_headers_remove”: [],
“global_size_limit”: 0,
“override_target”: “”
}
}
},
“use_standard_auth”: true,
“session_lifetime”: 0,
“hmac_allowed_algorithms”: [],
“disable_rate_limit”: true,
“definition”: {
“location”: “header”,
“key”: “x-api-version”,
“strip_path”: false
},
“use_oauth2”: false,
“jwt_source”: “”,
“jwt_signing_method”: “”,
“jwt_not_before_validation_skew”: 0,
“jwt_identity_base_field”: “”,
“allowed_ips”: [],
“org_id”: “5d884ef733924a00012d47ce”,
“enable_ip_whitelisting”: false,
“global_rate_limit”: {
“rate”: 0,
“per”: 0
},
“enable_context_vars”: false,
“tags”: [],
“basic_auth”: {
“disable_caching”: false,
“cache_ttl”: 0,
“extract_from_body”: false,
“body_user_regexp”: “”,
“body_password_regexp”: “”
},
“session_provider”: {
“name”: “”,
“storage_engine”: “”,
“meta”: {}
},
“strip_auth_data”: false,
“id”: “5d8853a233924a00012d47d2”,
“certificates”: [],
“enable_signature_checking”: false,
“use_openid”: false,
“internal”: false,
“jwt_skip_kid”: false,
“enable_batch_request_support”: false,
“response_processors”: [],
“use_mutual_tls_auth”: false
},
“hook_references”: [],
“is_site”: false,
“sort_by”: 0,
“user_group_owners”: [],
“user_owners”: []
}