Not able to get data from ingress endpoint

Hi, kindly consider the following
I have been trying to call endpoints via the ingress but it always results in a 404, not being found, and when the management endpoint is called I get 200 OK responses.
earlier we noticed that we had cache enabled and maybe it was caching the corrupt data so we disabled it and saw that the API is accessible through ingress but that only happened once with only one API and rest still gives 404

Following is the api_definition of the API which responds with success while called through ingress

{
  "created_at": "2022-05-10T08:59:37Z",
  "api_model": {},
  "api_definition": {
    "api_id": "829fc54a9b0d4ca445a04d5743edc3e9",
    "jwt_issued_at_validation_skew": 0,
    "upstream_certificates": {},
    "use_keyless": false,
    "enable_coprocess_auth": false,
    "base_identity_provided_by": "",
    "custom_middleware": {
      "pre": [],
      "post": [],
      "post_key_auth": [],
      "auth_check": {
        "name": "",
        "path": "",
        "require_session": false,
        "raw_body_only": false
      },
      "response": [],
      "driver": "",
      "id_extractor": {
        "extract_from": "",
        "extract_with": "",
        "extractor_config": {}
      }
    },
    "disable_quota": false,
    "custom_middleware_bundle": "",
    "cache_options": {
      "cache_timeout": 60,
      "enable_cache": false,
      "cache_all_safe_requests": false,
      "cache_response_codes": [],
      "enable_upstream_cache_control": false,
      "cache_control_ttl_header": "",
      "cache_by_headers": []
    },
    "enable_ip_blacklisting": false,
    "tag_headers": [],
    "jwt_scope_to_policy_mapping": {},
    "pinned_public_keys": {},
    "expire_analytics_after": 0,
    "domain": "",
    "openid_options": {
      "providers": [],
      "segregate_by_client": false
    },
    "jwt_policy_field_name": "",
    "enable_proxy_protocol": false,
    "jwt_default_policies": [],
    "active": true,
    "jwt_expires_at_validation_skew": 0,
    "config_data": {},
    "notifications": {
      "shared_secret": "",
      "oauth_on_keychange_url": ""
    },
    "jwt_client_base_field": "",
    "auth": {
      "use_param": false,
      "param_name": "",
      "use_cookie": false,
      "cookie_name": "",
      "auth_header_name": "Authorization",
      "use_certificate": false,
      "validate_signature": false,
      "signature": {
        "algorithm": "",
        "header": "",
        "secret": "",
        "allowed_clock_skew": 0,
        "error_code": 0,
        "error_message": ""
      }
    },
    "check_host_against_uptime_tests": false,
    "auth_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "blacklisted_ips": [],
    "graphql": {
      "schema": "",
      "enabled": false,
      "engine": {
        "field_configs": [],
        "data_sources": []
      },
      "type_field_configurations": [],
      "execution_mode": "",
      "proxy": {
        "auth_headers": {}
      },
      "subgraph": {
        "sdl": ""
      },
      "supergraph": {
        "subgraphs": [],
        "merged_sdl": "",
        "global_headers": {}
      },
      "version": "",
      "playground": {
        "enabled": false,
        "path": ""
      }
    },
    "hmac_allowed_clock_skew": -1,
    "dont_set_quota_on_create": false,
    "uptime_tests": {
      "check_list": [],
      "config": {
        "expire_utime_after": 0,
        "service_discovery": {
          "use_discovery_service": false,
          "query_endpoint": "",
          "use_nested_query": false,
          "parent_data_path": "",
          "data_path": "",
          "cache_timeout": 60
        },
        "recheck_wait": 0
      }
    },
    "enable_jwt": false,
    "do_not_track": false,
    "name": "shell-2",
    "slug": "shell-2",
    "oauth_meta": {
      "allowed_access_types": [],
      "allowed_authorize_types": [],
      "auth_login_redirect": ""
    },
    "CORS": {
      "enable": false,
      "max_age": 0,
      "allow_credentials": false,
      "exposed_headers": [],
      "allowed_headers": [],
      "options_passthrough": false,
      "debug": false,
      "allowed_origins": [],
      "allowed_methods": []
    },
    "event_handlers": {
      "events": {}
    },
    "proxy": {
      "target_url": "https://6260ea34e7361dff91fc380e.mockapi.io/data",
      "service_discovery": {
        "endpoint_returns_list": false,
        "cache_timeout": 0,
        "parent_data_path": "",
        "query_endpoint": "",
        "use_discovery_service": false,
        "_sd_show_port_path": false,
        "target_path": "",
        "use_target_list": false,
        "use_nested_query": false,
        "data_path": "",
        "port_data_path": ""
      },
      "check_host_against_uptime_tests": false,
      "transport": {
        "ssl_insecure_skip_verify": false,
        "ssl_min_version": 0,
        "proxy_url": "",
        "ssl_ciphers": []
      },
      "target_list": [],
      "preserve_host_header": false,
      "strip_listen_path": true,
      "enable_load_balancing": false,
      "listen_path": "/shell-2/",
      "disable_strip_slash": true
    },
    "client_certificates": [],
    "use_basic_auth": false,
    "version_data": {
      "not_versioned": true,
      "default_version": "Default",
      "versions": {
        "Default": {
          "name": "Default",
          "expires": "",
          "paths": {
            "ignored": [],
            "white_list": [],
            "black_list": []
          },
          "use_extended_paths": true,
          "extended_paths": {
            "ignored": [],
            "white_list": [],
            "black_list": [],
            "transform": [],
            "transform_response": [],
            "transform_jq": [],
            "transform_jq_response": [],
            "transform_headers": [],
            "transform_response_headers": [],
            "hard_timeouts": [],
            "circuit_breakers": [],
            "url_rewrites": [],
            "virtual": [],
            "size_limits": [],
            "method_transforms": [],
            "track_endpoints": [],
            "do_not_track_endpoints": [],
            "validate_json": [],
            "internal": []
          },
          "global_headers": {},
          "global_headers_remove": [],
          "global_response_headers": {},
          "global_response_headers_remove": [],
          "ignore_endpoint_case": false,
          "global_size_limit": 0,
          "override_target": ""
        }
      }
    },
    "jwt_scope_claim_name": "",
    "use_standard_auth": true,
    "session_lifetime": 0,
    "hmac_allowed_algorithms": [],
    "disable_rate_limit": false,
    "definition": {
      "location": "",
      "key": "",
      "strip_path": false
    },
    "use_oauth2": false,
    "jwt_source": "",
    "jwt_signing_method": "",
    "jwt_not_before_validation_skew": 0,
    "use_go_plugin_auth": false,
    "jwt_identity_base_field": "",
    "allowed_ips": [],
    "request_signing": {
      "is_enabled": false,
      "secret": "",
      "key_id": "",
      "algorithm": "",
      "header_list": [],
      "certificate_id": "",
      "signature_header": ""
    },
    "org_id": "6274a88d73241d0001e38567",
    "enable_ip_whitelisting": false,
    "global_rate_limit": {
      "rate": 0,
      "per": 0
    },
    "protocol": "",
    "enable_context_vars": false,
    "tags": [
      "edge"
    ],
    "basic_auth": {
      "disable_caching": false,
      "cache_ttl": 0,
      "extract_from_body": false,
      "body_user_regexp": "",
      "body_password_regexp": ""
    },
    "listen_port": 0,
    "session_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "auth_configs": {
      "authToken": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "jwt": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      }
    },
    "strip_auth_data": false,
    "id": "627a2979e5cfca00014df029",
    "certificates": [],
    "enable_signature_checking": false,
    "use_openid": false,
    "internal": false,
    "jwt_skip_kid": false,
    "enable_batch_request_support": false,
    "enable_detailed_recording": false,
    "response_processors": [],
    "use_mutual_tls_auth": false
  },
  "hook_references": [],
  "is_site": false,
  "sort_by": 0,
  "user_group_owners": [],
  "user_owners": []
}

api_definition of one that responds with 404

{
  "created_at": "2022-05-12T13:40:23Z",
  "api_model": {},
  "api_definition": {
    "api_id": "849d7fca6a34413b4b6dac93047b62a6",
    "jwt_issued_at_validation_skew": 0,
    "upstream_certificates": {},
    "use_keyless": false,
    "enable_coprocess_auth": false,
    "base_identity_provided_by": "",
    "custom_middleware": {
      "pre": [],
      "post": [],
      "post_key_auth": [],
      "auth_check": {
        "name": "",
        "path": "",
        "require_session": false,
        "raw_body_only": false
      },
      "response": [],
      "driver": "",
      "id_extractor": {
        "extract_from": "",
        "extract_with": "",
        "extractor_config": {}
      }
    },
    "disable_quota": false,
    "custom_middleware_bundle": "",
    "cache_options": {
      "cache_timeout": 60,
      "enable_cache": false,
      "cache_all_safe_requests": false,
      "cache_response_codes": [],
      "enable_upstream_cache_control": false,
      "cache_control_ttl_header": "",
      "cache_by_headers": []
    },
    "enable_ip_blacklisting": false,
    "tag_headers": [],
    "jwt_scope_to_policy_mapping": {},
    "pinned_public_keys": {},
    "expire_analytics_after": 0,
    "domain": "",
    "openid_options": {
      "providers": [],
      "segregate_by_client": false
    },
    "jwt_policy_field_name": "",
    "enable_proxy_protocol": false,
    "jwt_default_policies": [],
    "active": true,
    "jwt_expires_at_validation_skew": 0,
    "config_data": {},
    "notifications": {
      "shared_secret": "",
      "oauth_on_keychange_url": ""
    },
    "jwt_client_base_field": "",
    "auth": {
      "use_param": false,
      "param_name": "",
      "use_cookie": false,
      "cookie_name": "",
      "auth_header_name": "Authorization",
      "use_certificate": false,
      "validate_signature": false,
      "signature": {
        "algorithm": "",
        "header": "",
        "secret": "",
        "allowed_clock_skew": 0,
        "error_code": 0,
        "error_message": ""
      }
    },
    "check_host_against_uptime_tests": false,
    "auth_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "blacklisted_ips": [],
    "graphql": {
      "schema": "",
      "enabled": false,
      "engine": {
        "field_configs": [],
        "data_sources": []
      },
      "type_field_configurations": [],
      "execution_mode": "proxyOnly",
      "proxy": {
        "auth_headers": {}
      },
      "subgraph": {
        "sdl": ""
      },
      "supergraph": {
        "subgraphs": [],
        "merged_sdl": "",
        "global_headers": {}
      },
      "version": "2",
      "playground": {
        "enabled": false,
        "path": ""
      }
    },
    "hmac_allowed_clock_skew": -1,
    "dont_set_quota_on_create": false,
    "uptime_tests": {
      "check_list": [],
      "config": {
        "expire_utime_after": 0,
        "service_discovery": {
          "use_discovery_service": false,
          "query_endpoint": "",
          "use_nested_query": false,
          "parent_data_path": "",
          "data_path": "",
          "cache_timeout": 60
        },
        "recheck_wait": 0
      }
    },
    "enable_jwt": false,
    "do_not_track": false,
    "name": "test-shell1",
    "slug": "test-shell1",
    "oauth_meta": {
      "allowed_access_types": [],
      "allowed_authorize_types": [],
      "auth_login_redirect": ""
    },
    "CORS": {
      "enable": false,
      "max_age": 24,
      "allow_credentials": false,
      "exposed_headers": [],
      "allowed_headers": [
        "Origin",
        "Accept",
        "Content-Type",
        "X-Requested-With",
        "Authorization"
      ],
      "options_passthrough": false,
      "debug": false,
      "allowed_origins": [
        "*"
      ],
      "allowed_methods": [
        "GET",
        "POST",
        "HEAD"
      ]
    },
    "event_handlers": {
      "events": {}
    },
    "proxy": {
      "target_url": "https://62306be3f113bfceed50bfd3.mockapi.io/somestuff",
      "service_discovery": {
        "endpoint_returns_list": false,
        "cache_timeout": 0,
        "parent_data_path": "",
        "query_endpoint": "",
        "use_discovery_service": false,
        "_sd_show_port_path": false,
        "target_path": "",
        "use_target_list": false,
        "use_nested_query": false,
        "data_path": "",
        "port_data_path": ""
      },
      "check_host_against_uptime_tests": false,
      "transport": {
        "ssl_insecure_skip_verify": false,
        "ssl_min_version": 0,
        "proxy_url": "",
        "ssl_ciphers": []
      },
      "target_list": [],
      "preserve_host_header": false,
      "strip_listen_path": true,
      "enable_load_balancing": false,
      "listen_path": "/test-shell1/",
      "disable_strip_slash": true
    },
    "client_certificates": [],
    "use_basic_auth": false,
    "version_data": {
      "not_versioned": true,
      "default_version": "",
      "versions": {
        "Default": {
          "name": "Default",
          "expires": "",
          "paths": {
            "ignored": [],
            "white_list": [],
            "black_list": []
          },
          "use_extended_paths": true,
          "extended_paths": {
            "ignored": [],
            "white_list": [],
            "black_list": [],
            "transform": [],
            "transform_response": [],
            "transform_jq": [],
            "transform_jq_response": [],
            "transform_headers": [],
            "transform_response_headers": [],
            "hard_timeouts": [],
            "circuit_breakers": [],
            "url_rewrites": [],
            "virtual": [],
            "size_limits": [],
            "method_transforms": [],
            "track_endpoints": [],
            "do_not_track_endpoints": [],
            "validate_json": [],
            "internal": []
          },
          "global_headers": {},
          "global_headers_remove": [],
          "global_response_headers": {},
          "global_response_headers_remove": [],
          "ignore_endpoint_case": false,
          "global_size_limit": 0,
          "override_target": ""
        }
      }
    },
    "jwt_scope_claim_name": "",
    "use_standard_auth": true,
    "session_lifetime": 0,
    "hmac_allowed_algorithms": [],
    "disable_rate_limit": false,
    "definition": {
      "location": "header",
      "key": "x-api-version",
      "strip_path": false
    },
    "use_oauth2": false,
    "jwt_source": "",
    "jwt_signing_method": "",
    "jwt_not_before_validation_skew": 0,
    "use_go_plugin_auth": false,
    "jwt_identity_base_field": "",
    "allowed_ips": [],
    "request_signing": {
      "is_enabled": false,
      "secret": "",
      "key_id": "",
      "algorithm": "",
      "header_list": [],
      "certificate_id": "",
      "signature_header": ""
    },
    "org_id": "6274a88d73241d0001e38567",
    "enable_ip_whitelisting": false,
    "global_rate_limit": {
      "rate": 0,
      "per": 0
    },
    "protocol": "",
    "enable_context_vars": false,
    "tags": [
      "edge",
      "aws-us-east-1"
    ],
    "basic_auth": {
      "disable_caching": false,
      "cache_ttl": 0,
      "extract_from_body": false,
      "body_user_regexp": "",
      "body_password_regexp": ""
    },
    "listen_port": 0,
    "session_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "auth_configs": {
      "authToken": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "basic": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "coprocess": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "hmac": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "jwt": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "oauth": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "oidc": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      }
    },
    "strip_auth_data": false,
    "id": "627d0e47e5cfca00014df03f",
    "certificates": [],
    "enable_signature_checking": false,
    "use_openid": false,
    "internal": false,
    "jwt_skip_kid": false,
    "enable_batch_request_support": false,
    "enable_detailed_recording": false,
    "response_processors": [],
    "use_mutual_tls_auth": false
  },
  "hook_references": [],
  "is_site": false,
  "sort_by": 0,
  "user_group_owners": [],
  "user_owners": []
}

I am not sure. It’s works sometimes and sometimes it doesn’t. I was thinking the segment tag could be the issue but I eventually get a 200 response as long as I hit it multiple times.

The only distinct difference I can tell is that Tyk does not return a "Not Found" in quotes or string format. So I have to assume this is coming from upstream. Can you check where this is being thrown upstream. Maybe you could identify where the issue is.

The only other question that comes to mind is do you have anything between Tyk and your backend? Maybe a Load Balancer or something else. That could be the culprit actually throwing the Not Found message

Hi @Olu, after debugging, we have found that while calling the edge gateway for the endpoint it won’t strip the listen_path and will result in “Not Found” as the non-stripped URL does not exist in the upstream URL.
We have selected the “Strip the listen path” option on the Core Settings page of the API.
But while calling the API through the management gateway, the listen path is stripped as expected.

3.93.108.162 - - [16/May/2022:05:41:36 +0000] "GET /part-of-upstream HTTP/1.1" 200 31 "-" "curl/7.58.0"
75.101.249.106 - - [16/May/2022:05:41:49 +0000] "GET /part-of-upstream/test-api2/ HTTP/1.1" 200 42 "-" "curl/7.58.0"

This is our upstream nginx logs. First line was called via the mgw domain and the second line was called via the edge gw domain. Both had same settings.

I am going try to reproduce the behaviour on my end. Can you confirm you are using Tyk Cloud and the Gateway version is 4.0?

I have similar issues with Tyk Cloud v4.0.0 as well. Trying to figure out what configuration I am missing. Is there any place where we can see the services logs in Tyk Cloud?

Yes, the configuration is Tyk Cloud with gateway version 4.0

hi, were you able to figure out what is causing the issue

Sorry for the delay. I was a bit busy this week and the last.

So I tried reproducing the issue via the cloud 4.0.0 setup but could not reproduce it. I tested against httpbin and nginx as the upstream using the API definition you shared that returns 404.however, I only get 200 OK responses.

To troubleshoot further, can you send me as a private message the full URL as shown in the image below

In the meantime, you can isolate and test the API definition with our Open Source gateway offering?

The same gateway is used across our Self Managed and Cloud offering, so you would be validating the issue with the assistance of the gateway logs in debug mode.

@Joseph_Burns Welcome to the community and apologies for the delayed response.

At the moment, the gateway logs aren’t exposed to user’s of the cloud. We are actively working on this and would make an announcement when we implement this.

Regarding the issue you are facing, can you describe it? Or is it the same listen path issue as this thread?

As suggested to @Parsenjit_kumar, you can use our Open Source offering to quickly isolate, test and validate your API definition. The gateway is the same across board so the behavior should be the same.

HI @olu, thanks for the reply. yes you are right we were also getting 200 Ok responses but after a few post operations using dashboard API caused this and since no logs can be seen we are not able to figure out what caused it… sure I will try with local docker setup and will let you know and for your further consideration here is the URL
https://dashboard.cloud-ara.tyk.io/deployments/dashboard/bf5cec7f-c37c-4664-a6fb-d72039513ad3/bd1a350d-d348-483d-b45a-94f86ca59df1/7b9dacd2-bfd4-44a7-8ce4-9d75fc92f6af/f17ac6df-0e6e-4e0b-9e18-e30e654d94b0

hi @Olu, I tried accessing the endpoint via the local setup and it is returning 200 Ok for any API which is registered in gateway

@Parsenjit_kumar Thanks for sharing the dashboard URL and confirming it works on the Open Source version.

I’ll revert if find something internally.