Health check on load balancer target


#1

Hi,
I have a problem with APIs with Load Balancer targets.
I checked “Check host against downtime list” on “Uptime Tests” tab, but when a target in LB list is down, Tyk doesn’t skip it, so I miss some traffic.
I have to set something else?

thanks for your support.


#2

Hi, what Tyk version are you running?


#3

Hi,
Tyk-gateway 2.1.0.2 and tyk-dashboard 1.1 running on Ubuntu 14.04.


#4

This is a known bug in the Tyk 2.1 Load Balancer implementation, I would suggest you to try the nightly builds and let me know if you still face this issue.

You may find the nightly builds here:
https://tyk.io/docs/tyk-api-gateway-v-2-0/installation-options-setup/nightly-builds/

I’m pretty sure Tyk 2.2 solves this and will be out soon!


#5

Unfortunately , the problem seem to persist on nightly-07-01-2016 build too.


#6

Hi, are you able to share your API definition? This could be useful for us to diagnose the problem.
You can also share it to me via PM or e-mail.
I’m a member of the Tyk team.
Regards


#7

Yes, no problem, this is my test config:

{
    "id": "5776722183e9477a95000002",
    "name": "test",
    "slug": "test",
    "api_id": "9aa69c2f8628446a7a5994b36d61d940",
    "org_id": "577670a983e9477a95000001",
    "use_keyless": false,
    "use_oauth2": false,
    "use_openid": false,
    "openid_options": {
        "providers": [],
        "segregate_by_client": false
    },
    "oauth_meta": {
        "allowed_access_types": [],
        "allowed_authorize_types": [],
        "auth_login_redirect": ""
    },
    "auth": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization"
    },
    "use_basic_auth": false,
    "enable_jwt": false,
    "jwt_signing_method": "",
    "jwt_source": "",
    "jwt_identity_base_field": "",
    "jwt_client_base_field": "",
    "jwt_policy_field_name": "",
    "notifications": {
        "shared_secret": "",
        "oauth_on_keychange_url": ""
    },
    "enable_signature_checking": false,
    "hmac_allowed_clock_skew": -1,
    "definition": {
        "location": "header",
        "key": "x-api-version"
    },
    "version_data": {
        "not_versioned": true,
        "versions": {
            "Default": {
                "name": "Default",
                "expires": "",
                "paths": {
                    "ignored": [],
                    "white_list": [],
                    "black_list": []
                },
                "use_extended_paths": true,
                "extended_paths": {
                    "ignored": [],
                    "white_list": [],
                    "black_list": [],
                    "cache": [],
                    "transform": [],
                    "transform_response": [],
                    "transform_headers": [],
                    "transform_response_headers": [],
                    "hard_timeouts": [],
                    "circuit_breakers": [],
                    "url_rewrites": [],
                    "virtual": [],
                    "size_limits": []
                },
                "global_headers": {},
                "global_headers_remove": [],
                "global_size_limit": 0,
                "override_target": ""
            }
        }
    },
    "uptime_tests": {
        "check_list": [],
        "config": {
            "expire_utime_after": 0,
            "service_discovery": {
                "use_discovery_service": false,
                "query_endpoint": "",
                "use_nested_query": false,
                "parent_data_path": "",
                "data_path": "",
                "port_data_path": "",
                "target_path": "",
                "use_target_list": false,
                "cache_timeout": 60,
                "endpoint_returns_list": false
            },
            "recheck_wait": 0
        }
    },
    "proxy": {
        "preserve_host_header": false,
        "listen_path": "/test/",
        "target_url": "",
        "strip_listen_path": true,
        "enable_load_balancing": true,
        "target_list": [
            "http://host1:9001",
            "http://host2:9001"
        ],
        "check_host_against_uptime_tests": true,
        "service_discovery": {
            "use_discovery_service": false,
            "query_endpoint": "",
            "use_nested_query": false,
            "parent_data_path": "",
            "data_path": "hostname",
            "port_data_path": "port",
            "target_path": "/api-slug",
            "use_target_list": false,
            "cache_timeout": 60,
            "endpoint_returns_list": false
        }
    },
    "custom_middleware": {
        "pre": [],
        "post": [],
        "response": []
    },
    "cache_options": {
        "cache_timeout": 60,
        "enable_cache": false,
        "cache_all_safe_requests": false,
        "cache_response_codes": [],
        "enable_upstream_cache_control": false
    },
    "session_lifetime": 0,
    "active": true,
    "auth_provider": {
        "name": "",
        "storage_engine": "",
        "meta": {}
    },
    "session_provider": {
        "name": "",
        "storage_engine": "",
        "meta": null
    },
    "event_handlers": {
        "events": {}
    },
    "enable_batch_request_support": false,
    "enable_ip_whitelisting": false,
    "allowed_ips": [],
    "dont_set_quota_on_create": false,
    "expire_analytics_after": 0,
    "response_processors": [],
    "CORS": {
        "enable": false,
        "allowed_origins": [],
        "allowed_methods": [],
        "allowed_headers": [],
        "exposed_headers": [],
        "allow_credentials": false,
        "max_age": 24,
        "options_passthrough": false,
        "debug": false
    },
    "domain": "",
    "do_not_track": false,
    "tags": []
}

#8

Thank you, we’ll check it


#9

Hi Carlo, it seems that your uptime tests check list is empty:

    "uptime_tests": {
        "check_list": [],
        "config": {
            "expire_utime_after": 0,
            "service_discovery": {
                "use_discovery_service": false,
                "query_endpoint": "",
                "use_nested_query": false,
                "parent_data_path": "",
                "data_path": "",
                "port_data_path": "",
                "target_path": "",
                "use_target_list": false,
                "cache_timeout": 60,
                "endpoint_returns_list": false
            },
            "recheck_wait": 0
        }
    }

#10

Yes,
I also had tried same configuration but with these tests unsuccessufully.

"uptime_tests": {
    "check_list": [
        {
            "url": "http://host1:9001",
            "method": "GET",
            "headers": {},
            "body": ""
        },
        {
            "url": "http://host2:9001",
            "method": "GET",
            "headers": {},
            "body": ""
        }
    ],
    "config": {
        "expire_utime_after": 0,
        "service_discovery": {
            "use_discovery_service": false,
            "query_endpoint": "",
            "use_nested_query": false,
            "parent_data_path": "",
            "data_path": "",
            "port_data_path": "",
            "target_path": "",
            "use_target_list": false,
            "cache_timeout": 60,
            "endpoint_returns_list": false
        },
        "recheck_wait": 0
    }
},

#11

Can you share your log? The uptime/downtime events should show up there.
Also check check_host_against_uptime_tests.


#12

Sure,
here a recent tyk-gateway.log

e[90m[Jul  5 15:23:10]e[0m e[33m WARNe[0m e[36mhost-check-mgr:e[0m Host checker data is no longer purged by Tyk Gateway, please use Tyk-Pump.
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m Connection dropped, connecting..
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mhost-check-mgr:e[0m Starting Poller
e[90m[Jul  5 15:23:10]e[0m e[33m WARNe[0m e[36mmain:e[0m Cache purging is no longer part of Tyk Gateway, please use Tyk-Pump.
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Setting up Server
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m --> Standard listener (http)
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Registering node.
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mdashboard:e[0m Node registered e[32mide[0m=2d4a5c95-5972-48f2-450f-b77bd068fb91
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Starting heartbeat.
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Detected 1 APIs
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Loading API configurations.
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m --> Loading API: test
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m ----> Tracking: (no host)
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m ----> Checking security policy: Token
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m ----> Setting Listen Path: /test/
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mhost-check-mgr:e[0m Loading uptime tests...
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mhost-check-mgr:e[0m ---> Adding uptime test: http://host1:9001
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mhost-check-mgr:e[0m ---> Adding uptime test: http://host2:9001
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Initialised API Definitions
e[90m[Jul  5 15:23:10]e[0m e[31mERRORe[0m Failed to decode body: json: cannot unmarshal string into Go value of type []main.DBPolicy
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m Gateway started (-nightly-07-01-2016)
e[90m[Jul  5 15:23:10]e[0m e[32m INFOe[0m e[36mmain:e[0m --> Listening on port: 8080
e[90m[Jul  5 15:23:44]e[0m e[32m INFOe[0m e[36mauth-mgr:e[0m Reset quota for key. e[32minbound-keye[0m=577670a983e9477a95000001f18fb37e070e4aaa47a2c175af496301 e[32mkeye[0m=quota-49cf61f8
e[90m[Jul  5 15:23:44]e[0m e[32m INFOe[0m e[36mapi:e[0m Generated new key. e[32mkeye[0m=577670a983e9477a95000001f18fb37e070e4aaa47a2c175af496301 e[32mstatuse[0m=ok
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mproxy:e[0m http: proxy error: dial tcp 127.0.0.1:9001: getsockopt: connection refused e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mserver_namee[0m=host1:9001 e[31muser_ide[0m=****6301 e[31muser_ipe[0m=127.0.0.1 e[31muser_namee[0m=
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mgateway:e[0m request error: There was a problem proxying the request e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mpathe[0m=/ e[31mserver_namee[0m= e[31muser_ide[0m= e[31muser_ipe[0m=127.0.0.1, 127.0.0.1
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mproxy:e[0m http: proxy error: dial tcp 127.0.0.1:9001: getsockopt: connection refused e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mserver_namee[0m=host1:9001 e[31muser_ide[0m=****6301 e[31muser_ipe[0m=127.0.0.1 e[31muser_namee[0m=
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mgateway:e[0m request error: There was a problem proxying the request e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mpathe[0m=/ e[31mserver_namee[0m= e[31muser_ide[0m= e[31muser_ipe[0m=127.0.0.1, 127.0.0.1
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mproxy:e[0m http: proxy error: dial tcp 127.0.0.1:9001: getsockopt: connection refused e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mserver_namee[0m=host1:9001 e[31muser_ide[0m=****6301 e[31muser_ipe[0m=127.0.0.1 e[31muser_namee[0m=
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mgateway:e[0m request error: There was a problem proxying the request e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mpathe[0m=/ e[31mserver_namee[0m= e[31muser_ide[0m= e[31muser_ipe[0m=127.0.0.1, 127.0.0.1
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mproxy:e[0m http: proxy error: dial tcp 127.0.0.1:9001: getsockopt: connection refused e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mserver_namee[0m=host1:9001 e[31muser_ide[0m=****6301 e[31muser_ipe[0m=127.0.0.1 e[31muser_namee[0m=
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mgateway:e[0m request error: There was a problem proxying the request e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mpathe[0m=/ e[31mserver_namee[0m= e[31muser_ide[0m= e[31muser_ipe[0m=127.0.0.1, 127.0.0.1
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mproxy:e[0m http: proxy error: dial tcp 127.0.0.1:9001: getsockopt: connection refused e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mserver_namee[0m=host1:9001 e[31muser_ide[0m=****6301 e[31muser_ipe[0m=127.0.0.1 e[31muser_namee[0m=
e[90m[Jul  5 15:24:17]e[0m e[31mERRORe[0m e[36mgateway:e[0m request error: There was a problem proxying the request e[31mapi_ide[0m=9aa69c2f8628446a7a5994b36d61d940 e[31morg_ide[0m=577670a983e9477a95000001 e[31mpathe[0m=/ e[31mserver_namee[0m= e[31muser_ide[0m= e[31muser_ipe[0m=127.0.0.1, 127.0.0.1

#13

Thank you, I’m checking


#14

Hi Carlo, can you check your tyk.conf?

There is a global setting for uptime tests:
https://tyk.io/docs/tyk-api-gateway-v-2-0/uptime-tests/

Also, Tyk 2.2 is out :slight_smile: