Not able to get data from ingress endpoint

Hi, kindly consider the following
I have been trying to call endpoints via the ingress but it always results in a 404, not being found, and when the management endpoint is called I get 200 OK responses.
earlier we noticed that we had cache enabled and maybe it was caching the corrupt data so we disabled it and saw that the API is accessible through ingress but that only happened once with only one API and rest still gives 404

Following is the api_definition of the API which responds with success while called through ingress

{
  "created_at": "2022-05-10T08:59:37Z",
  "api_model": {},
  "api_definition": {
    "api_id": "829fc54a9b0d4ca445a04d5743edc3e9",
    "jwt_issued_at_validation_skew": 0,
    "upstream_certificates": {},
    "use_keyless": false,
    "enable_coprocess_auth": false,
    "base_identity_provided_by": "",
    "custom_middleware": {
      "pre": [],
      "post": [],
      "post_key_auth": [],
      "auth_check": {
        "name": "",
        "path": "",
        "require_session": false,
        "raw_body_only": false
      },
      "response": [],
      "driver": "",
      "id_extractor": {
        "extract_from": "",
        "extract_with": "",
        "extractor_config": {}
      }
    },
    "disable_quota": false,
    "custom_middleware_bundle": "",
    "cache_options": {
      "cache_timeout": 60,
      "enable_cache": false,
      "cache_all_safe_requests": false,
      "cache_response_codes": [],
      "enable_upstream_cache_control": false,
      "cache_control_ttl_header": "",
      "cache_by_headers": []
    },
    "enable_ip_blacklisting": false,
    "tag_headers": [],
    "jwt_scope_to_policy_mapping": {},
    "pinned_public_keys": {},
    "expire_analytics_after": 0,
    "domain": "",
    "openid_options": {
      "providers": [],
      "segregate_by_client": false
    },
    "jwt_policy_field_name": "",
    "enable_proxy_protocol": false,
    "jwt_default_policies": [],
    "active": true,
    "jwt_expires_at_validation_skew": 0,
    "config_data": {},
    "notifications": {
      "shared_secret": "",
      "oauth_on_keychange_url": ""
    },
    "jwt_client_base_field": "",
    "auth": {
      "use_param": false,
      "param_name": "",
      "use_cookie": false,
      "cookie_name": "",
      "auth_header_name": "Authorization",
      "use_certificate": false,
      "validate_signature": false,
      "signature": {
        "algorithm": "",
        "header": "",
        "secret": "",
        "allowed_clock_skew": 0,
        "error_code": 0,
        "error_message": ""
      }
    },
    "check_host_against_uptime_tests": false,
    "auth_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "blacklisted_ips": [],
    "graphql": {
      "schema": "",
      "enabled": false,
      "engine": {
        "field_configs": [],
        "data_sources": []
      },
      "type_field_configurations": [],
      "execution_mode": "",
      "proxy": {
        "auth_headers": {}
      },
      "subgraph": {
        "sdl": ""
      },
      "supergraph": {
        "subgraphs": [],
        "merged_sdl": "",
        "global_headers": {}
      },
      "version": "",
      "playground": {
        "enabled": false,
        "path": ""
      }
    },
    "hmac_allowed_clock_skew": -1,
    "dont_set_quota_on_create": false,
    "uptime_tests": {
      "check_list": [],
      "config": {
        "expire_utime_after": 0,
        "service_discovery": {
          "use_discovery_service": false,
          "query_endpoint": "",
          "use_nested_query": false,
          "parent_data_path": "",
          "data_path": "",
          "cache_timeout": 60
        },
        "recheck_wait": 0
      }
    },
    "enable_jwt": false,
    "do_not_track": false,
    "name": "shell-2",
    "slug": "shell-2",
    "oauth_meta": {
      "allowed_access_types": [],
      "allowed_authorize_types": [],
      "auth_login_redirect": ""
    },
    "CORS": {
      "enable": false,
      "max_age": 0,
      "allow_credentials": false,
      "exposed_headers": [],
      "allowed_headers": [],
      "options_passthrough": false,
      "debug": false,
      "allowed_origins": [],
      "allowed_methods": []
    },
    "event_handlers": {
      "events": {}
    },
    "proxy": {
      "target_url": "https://6260ea34e7361dff91fc380e.mockapi.io/data",
      "service_discovery": {
        "endpoint_returns_list": false,
        "cache_timeout": 0,
        "parent_data_path": "",
        "query_endpoint": "",
        "use_discovery_service": false,
        "_sd_show_port_path": false,
        "target_path": "",
        "use_target_list": false,
        "use_nested_query": false,
        "data_path": "",
        "port_data_path": ""
      },
      "check_host_against_uptime_tests": false,
      "transport": {
        "ssl_insecure_skip_verify": false,
        "ssl_min_version": 0,
        "proxy_url": "",
        "ssl_ciphers": []
      },
      "target_list": [],
      "preserve_host_header": false,
      "strip_listen_path": true,
      "enable_load_balancing": false,
      "listen_path": "/shell-2/",
      "disable_strip_slash": true
    },
    "client_certificates": [],
    "use_basic_auth": false,
    "version_data": {
      "not_versioned": true,
      "default_version": "Default",
      "versions": {
        "Default": {
          "name": "Default",
          "expires": "",
          "paths": {
            "ignored": [],
            "white_list": [],
            "black_list": []
          },
          "use_extended_paths": true,
          "extended_paths": {
            "ignored": [],
            "white_list": [],
            "black_list": [],
            "transform": [],
            "transform_response": [],
            "transform_jq": [],
            "transform_jq_response": [],
            "transform_headers": [],
            "transform_response_headers": [],
            "hard_timeouts": [],
            "circuit_breakers": [],
            "url_rewrites": [],
            "virtual": [],
            "size_limits": [],
            "method_transforms": [],
            "track_endpoints": [],
            "do_not_track_endpoints": [],
            "validate_json": [],
            "internal": []
          },
          "global_headers": {},
          "global_headers_remove": [],
          "global_response_headers": {},
          "global_response_headers_remove": [],
          "ignore_endpoint_case": false,
          "global_size_limit": 0,
          "override_target": ""
        }
      }
    },
    "jwt_scope_claim_name": "",
    "use_standard_auth": true,
    "session_lifetime": 0,
    "hmac_allowed_algorithms": [],
    "disable_rate_limit": false,
    "definition": {
      "location": "",
      "key": "",
      "strip_path": false
    },
    "use_oauth2": false,
    "jwt_source": "",
    "jwt_signing_method": "",
    "jwt_not_before_validation_skew": 0,
    "use_go_plugin_auth": false,
    "jwt_identity_base_field": "",
    "allowed_ips": [],
    "request_signing": {
      "is_enabled": false,
      "secret": "",
      "key_id": "",
      "algorithm": "",
      "header_list": [],
      "certificate_id": "",
      "signature_header": ""
    },
    "org_id": "6274a88d73241d0001e38567",
    "enable_ip_whitelisting": false,
    "global_rate_limit": {
      "rate": 0,
      "per": 0
    },
    "protocol": "",
    "enable_context_vars": false,
    "tags": [
      "edge"
    ],
    "basic_auth": {
      "disable_caching": false,
      "cache_ttl": 0,
      "extract_from_body": false,
      "body_user_regexp": "",
      "body_password_regexp": ""
    },
    "listen_port": 0,
    "session_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "auth_configs": {
      "authToken": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "jwt": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      }
    },
    "strip_auth_data": false,
    "id": "627a2979e5cfca00014df029",
    "certificates": [],
    "enable_signature_checking": false,
    "use_openid": false,
    "internal": false,
    "jwt_skip_kid": false,
    "enable_batch_request_support": false,
    "enable_detailed_recording": false,
    "response_processors": [],
    "use_mutual_tls_auth": false
  },
  "hook_references": [],
  "is_site": false,
  "sort_by": 0,
  "user_group_owners": [],
  "user_owners": []
}

api_definition of one that responds with 404

{
  "created_at": "2022-05-12T13:40:23Z",
  "api_model": {},
  "api_definition": {
    "api_id": "849d7fca6a34413b4b6dac93047b62a6",
    "jwt_issued_at_validation_skew": 0,
    "upstream_certificates": {},
    "use_keyless": false,
    "enable_coprocess_auth": false,
    "base_identity_provided_by": "",
    "custom_middleware": {
      "pre": [],
      "post": [],
      "post_key_auth": [],
      "auth_check": {
        "name": "",
        "path": "",
        "require_session": false,
        "raw_body_only": false
      },
      "response": [],
      "driver": "",
      "id_extractor": {
        "extract_from": "",
        "extract_with": "",
        "extractor_config": {}
      }
    },
    "disable_quota": false,
    "custom_middleware_bundle": "",
    "cache_options": {
      "cache_timeout": 60,
      "enable_cache": false,
      "cache_all_safe_requests": false,
      "cache_response_codes": [],
      "enable_upstream_cache_control": false,
      "cache_control_ttl_header": "",
      "cache_by_headers": []
    },
    "enable_ip_blacklisting": false,
    "tag_headers": [],
    "jwt_scope_to_policy_mapping": {},
    "pinned_public_keys": {},
    "expire_analytics_after": 0,
    "domain": "",
    "openid_options": {
      "providers": [],
      "segregate_by_client": false
    },
    "jwt_policy_field_name": "",
    "enable_proxy_protocol": false,
    "jwt_default_policies": [],
    "active": true,
    "jwt_expires_at_validation_skew": 0,
    "config_data": {},
    "notifications": {
      "shared_secret": "",
      "oauth_on_keychange_url": ""
    },
    "jwt_client_base_field": "",
    "auth": {
      "use_param": false,
      "param_name": "",
      "use_cookie": false,
      "cookie_name": "",
      "auth_header_name": "Authorization",
      "use_certificate": false,
      "validate_signature": false,
      "signature": {
        "algorithm": "",
        "header": "",
        "secret": "",
        "allowed_clock_skew": 0,
        "error_code": 0,
        "error_message": ""
      }
    },
    "check_host_against_uptime_tests": false,
    "auth_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "blacklisted_ips": [],
    "graphql": {
      "schema": "",
      "enabled": false,
      "engine": {
        "field_configs": [],
        "data_sources": []
      },
      "type_field_configurations": [],
      "execution_mode": "proxyOnly",
      "proxy": {
        "auth_headers": {}
      },
      "subgraph": {
        "sdl": ""
      },
      "supergraph": {
        "subgraphs": [],
        "merged_sdl": "",
        "global_headers": {}
      },
      "version": "2",
      "playground": {
        "enabled": false,
        "path": ""
      }
    },
    "hmac_allowed_clock_skew": -1,
    "dont_set_quota_on_create": false,
    "uptime_tests": {
      "check_list": [],
      "config": {
        "expire_utime_after": 0,
        "service_discovery": {
          "use_discovery_service": false,
          "query_endpoint": "",
          "use_nested_query": false,
          "parent_data_path": "",
          "data_path": "",
          "cache_timeout": 60
        },
        "recheck_wait": 0
      }
    },
    "enable_jwt": false,
    "do_not_track": false,
    "name": "test-shell1",
    "slug": "test-shell1",
    "oauth_meta": {
      "allowed_access_types": [],
      "allowed_authorize_types": [],
      "auth_login_redirect": ""
    },
    "CORS": {
      "enable": false,
      "max_age": 24,
      "allow_credentials": false,
      "exposed_headers": [],
      "allowed_headers": [
        "Origin",
        "Accept",
        "Content-Type",
        "X-Requested-With",
        "Authorization"
      ],
      "options_passthrough": false,
      "debug": false,
      "allowed_origins": [
        "*"
      ],
      "allowed_methods": [
        "GET",
        "POST",
        "HEAD"
      ]
    },
    "event_handlers": {
      "events": {}
    },
    "proxy": {
      "target_url": "https://62306be3f113bfceed50bfd3.mockapi.io/somestuff",
      "service_discovery": {
        "endpoint_returns_list": false,
        "cache_timeout": 0,
        "parent_data_path": "",
        "query_endpoint": "",
        "use_discovery_service": false,
        "_sd_show_port_path": false,
        "target_path": "",
        "use_target_list": false,
        "use_nested_query": false,
        "data_path": "",
        "port_data_path": ""
      },
      "check_host_against_uptime_tests": false,
      "transport": {
        "ssl_insecure_skip_verify": false,
        "ssl_min_version": 0,
        "proxy_url": "",
        "ssl_ciphers": []
      },
      "target_list": [],
      "preserve_host_header": false,
      "strip_listen_path": true,
      "enable_load_balancing": false,
      "listen_path": "/test-shell1/",
      "disable_strip_slash": true
    },
    "client_certificates": [],
    "use_basic_auth": false,
    "version_data": {
      "not_versioned": true,
      "default_version": "",
      "versions": {
        "Default": {
          "name": "Default",
          "expires": "",
          "paths": {
            "ignored": [],
            "white_list": [],
            "black_list": []
          },
          "use_extended_paths": true,
          "extended_paths": {
            "ignored": [],
            "white_list": [],
            "black_list": [],
            "transform": [],
            "transform_response": [],
            "transform_jq": [],
            "transform_jq_response": [],
            "transform_headers": [],
            "transform_response_headers": [],
            "hard_timeouts": [],
            "circuit_breakers": [],
            "url_rewrites": [],
            "virtual": [],
            "size_limits": [],
            "method_transforms": [],
            "track_endpoints": [],
            "do_not_track_endpoints": [],
            "validate_json": [],
            "internal": []
          },
          "global_headers": {},
          "global_headers_remove": [],
          "global_response_headers": {},
          "global_response_headers_remove": [],
          "ignore_endpoint_case": false,
          "global_size_limit": 0,
          "override_target": ""
        }
      }
    },
    "jwt_scope_claim_name": "",
    "use_standard_auth": true,
    "session_lifetime": 0,
    "hmac_allowed_algorithms": [],
    "disable_rate_limit": false,
    "definition": {
      "location": "header",
      "key": "x-api-version",
      "strip_path": false
    },
    "use_oauth2": false,
    "jwt_source": "",
    "jwt_signing_method": "",
    "jwt_not_before_validation_skew": 0,
    "use_go_plugin_auth": false,
    "jwt_identity_base_field": "",
    "allowed_ips": [],
    "request_signing": {
      "is_enabled": false,
      "secret": "",
      "key_id": "",
      "algorithm": "",
      "header_list": [],
      "certificate_id": "",
      "signature_header": ""
    },
    "org_id": "6274a88d73241d0001e38567",
    "enable_ip_whitelisting": false,
    "global_rate_limit": {
      "rate": 0,
      "per": 0
    },
    "protocol": "",
    "enable_context_vars": false,
    "tags": [
      "edge",
      "aws-us-east-1"
    ],
    "basic_auth": {
      "disable_caching": false,
      "cache_ttl": 0,
      "extract_from_body": false,
      "body_user_regexp": "",
      "body_password_regexp": ""
    },
    "listen_port": 0,
    "session_provider": {
      "name": "",
      "storage_engine": "",
      "meta": {}
    },
    "auth_configs": {
      "authToken": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "basic": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "coprocess": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "hmac": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "jwt": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "oauth": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      },
      "oidc": {
        "use_param": false,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "Authorization",
        "use_certificate": false,
        "validate_signature": false,
        "signature": {
          "algorithm": "",
          "header": "",
          "secret": "",
          "allowed_clock_skew": 0,
          "error_code": 0,
          "error_message": ""
        }
      }
    },
    "strip_auth_data": false,
    "id": "627d0e47e5cfca00014df03f",
    "certificates": [],
    "enable_signature_checking": false,
    "use_openid": false,
    "internal": false,
    "jwt_skip_kid": false,
    "enable_batch_request_support": false,
    "enable_detailed_recording": false,
    "response_processors": [],
    "use_mutual_tls_auth": false
  },
  "hook_references": [],
  "is_site": false,
  "sort_by": 0,
  "user_group_owners": [],
  "user_owners": []
}

I am not sure. It’s works sometimes and sometimes it doesn’t. I was thinking the segment tag could be the issue but I eventually get a 200 response as long as I hit it multiple times.

The only distinct difference I can tell is that Tyk does not return a "Not Found" in quotes or string format. So I have to assume this is coming from upstream. Can you check where this is being thrown upstream. Maybe you could identify where the issue is.

The only other question that comes to mind is do you have anything between Tyk and your backend? Maybe a Load Balancer or something else. That could be the culprit actually throwing the Not Found message

Hi @Olu, after debugging, we have found that while calling the edge gateway for the endpoint it won’t strip the listen_path and will result in “Not Found” as the non-stripped URL does not exist in the upstream URL.
We have selected the “Strip the listen path” option on the Core Settings page of the API.
But while calling the API through the management gateway, the listen path is stripped as expected.

3.93.108.162 - - [16/May/2022:05:41:36 +0000] "GET /part-of-upstream HTTP/1.1" 200 31 "-" "curl/7.58.0"
75.101.249.106 - - [16/May/2022:05:41:49 +0000] "GET /part-of-upstream/test-api2/ HTTP/1.1" 200 42 "-" "curl/7.58.0"

This is our upstream nginx logs. First line was called via the mgw domain and the second line was called via the edge gw domain. Both had same settings.

I am going try to reproduce the behaviour on my end. Can you confirm you are using Tyk Cloud and the Gateway version is 4.0?

I have similar issues with Tyk Cloud v4.0.0 as well. Trying to figure out what configuration I am missing. Is there any place where we can see the services logs in Tyk Cloud?

Yes, the configuration is Tyk Cloud with gateway version 4.0