URL rewriting not working

Support Installation and Deployment
, ,
1

I have an API definition that listens to:
/v1/centres
I need to redirect all accesses to:
/v1/centres/{id}/movies
to a different API definition as these are on a different target. I want them to go to:
/movies/v1/centres/{id}/movies
and have that API Definition pass:
/v1/centres/{id}/movies
to the target. I have an API Definition that listens to:
/movies
and strips the listen path to have the desired pass through. What I’ve set up seems correct but is not working. Here are the API Definitions. What is not correct, the access goes to the centre service target?

Centres API

{
    "id": "58a16f7e6e39310001cf8fef",
    "name": "Centre Service Rewrite",
    "slug": "centre-service-rewrite",
    "api_id": "24c8281d54364c806bfe96915739c096",
    "org_id": "589a65006e3931000128cd5c",
    "use_keyless": false,
    "use_oauth2": false,
    "use_openid": false,
    "openid_options": {
        "providers": [],
        "segregate_by_client": false
    },
    "oauth_meta": {
        "allowed_access_types": [],
        "allowed_authorize_types": [],
        "auth_login_redirect": ""
    },
    "auth": {
        "use_param": true,
        "param_name": "",
        "use_cookie": false,
        "cookie_name": "",
        "auth_header_name": "api_key"
    },
    "use_basic_auth": false,
    "enable_jwt": false,
    "use_standard_auth": true,
    "enable_coprocess_auth": false,
    "jwt_signing_method": "",
    "jwt_source": "",
    "jwt_identity_base_field": "",
    "jwt_client_base_field": "",
    "jwt_policy_field_name": "",
    "notifications": {
        "shared_secret": "",
        "oauth_on_keychange_url": ""
    },
    "enable_signature_checking": false,
    "hmac_allowed_clock_skew": -1,
    "base_identity_provided_by": "",
    "definition": {
        "location": "url",
        "key": "version"
    },
    "version_data": {
        "not_versioned": true,
        "versions": {
            "v1": {
                "name": "v1",
                "expires": "",
                "paths": {
                    "ignored": [],
                    "white_list": [],
                    "black_list": []
                },
                "use_extended_paths": true,
                "extended_paths": {
                    "url_rewrites": [
                        {
                            "path": "/v1/centres/{id}/movies",
                            "method": "GET",
                            "match_pattern": "/v1/centres/(w+)/movies",
                            "rewrite_to": "/movies/v1/centres/$1/movies"
                        }
                    ]
                },
                "global_headers": {
                },
                "global_headers_remove": [],
                "global_size_limit": 0,
                "override_target": ""
            }
        }
    },
    "uptime_tests": {
        "check_list": [],
        "config": {
            "expire_utime_after": 0,
            "service_discovery": {
                "use_discovery_service": false,
                "query_endpoint": "",
                "use_nested_query": false,
                "parent_data_path": "",
                "data_path": "",
                "port_data_path": "",
                "target_path": "",
                "use_target_list": false,
                "cache_timeout": 0,
                "endpoint_returns_list": false
            },
            "recheck_wait": 0
        }
    },
    "proxy": {
        "preserve_host_header": false,
        "listen_path": "/v1/centres",
        "target_url": "http://www.mocky.io/v2/58a175411000000f295586b0",
        "strip_listen_path": false,
        "enable_load_balancing": false,
        "target_list": [],
        "check_host_against_uptime_tests": false,
        "service_discovery": {
            "use_discovery_service": false,
            "query_endpoint": "",
            "use_nested_query": false,
            "parent_data_path": "",
            "data_path": "",
            "port_data_path": "",
            "target_path": "",
            "use_target_list": false,
            "cache_timeout": 0,
            "endpoint_returns_list": false
        }
    },
    "disable_rate_limit": false,
    "disable_quota": false,
    "custom_middleware": {
        "pre": [
            {
                "name": "addRequestIdMiddleware",
                "path": "/opt/tyk-gateway/middleware/add_request_id.js",
                "require_session": false
            }
        ],
        "post": [],
        "post_key_auth": [],
        "auth_check": {
            "name": "",
            "path": "",
            "require_session": false
        },
        "response": [],
        "driver": "",
        "id_extractor": {
            "extract_from": "",
            "extract_with": "",
            "extractor_config": {}
        }
    },
    "custom_middleware_bundle": "",
    "cache_options": {
        "cache_timeout": 600,
        "enable_cache": true,
        "cache_all_safe_requests": false,
        "cache_response_codes": [],
        "enable_upstream_cache_control": true
    },
    "session_lifetime": 0,
    "active": true,
    "auth_provider": {
        "name": "",
        "storage_engine": "",
        "meta": {}
    },
    "session_provider": {
        "name": "",
        "storage_engine": "",
        "meta": null
    },
    "event_handlers": {
        "events": {}
    },
    "enable_batch_request_support": false,
    "enable_ip_whitelisting": false,
    "allowed_ips": [],
    "dont_set_quota_on_create": false,
    "expire_analytics_after": 0,
    "response_processors": [],
    "CORS": {
        "enable": false,
        "allowed_origins": [],
        "allowed_methods": [],
        "allowed_headers": [],
        "exposed_headers": [],
        "allow_credentials": false,
        "max_age": 0,
        "options_passthrough": false,
        "debug": false
    },
    "domain": "api.dev.westfield.io",
    "do_not_track": false,
    "tags": [],
    "enable_context_vars": false
}

Movie API

    {
        "id": "58a174246e39310001cf8ff2",
        "name": "Movie Service",
        "slug": "movie-service",
        "api_id": "56540ae6433e42bd6b76f223fcaf9a34",
        "org_id": "589a65006e3931000128cd5c",
        "use_keyless": true,
        "use_oauth2": false,
        "use_openid": false,
        "openid_options": {
            "providers": [],
            "segregate_by_client": false
        },
        "oauth_meta": {
            "allowed_access_types": [],
            "allowed_authorize_types": [],
            "auth_login_redirect": ""
        },
        "auth": {
            "use_param": false,
            "param_name": "",
            "use_cookie": false,
            "cookie_name": "",
            "auth_header_name": ""
        },
        "use_basic_auth": false,
        "enable_jwt": false,
        "use_standard_auth": false,
        "enable_coprocess_auth": false,
        "jwt_signing_method": "",
        "jwt_source": "",
        "jwt_identity_base_field": "",
        "jwt_client_base_field": "",
        "jwt_policy_field_name": "",
        "notifications": {
            "shared_secret": "",
            "oauth_on_keychange_url": ""
        },
        "enable_signature_checking": false,
        "hmac_allowed_clock_skew": -1,
        "base_identity_provided_by": "",
        "definition": {
            "location": "header",
            "key": "x-api-version"
        },
        "version_data": {
            "not_versioned": true,
            "versions": {
                "Default": {
                    "name": "Default",
                    "expires": "",
                    "paths": {
                        "ignored": [],
                        "white_list": [],
                        "black_list": []
                    },
                    "use_extended_paths": true,
                    "extended_paths": {
                        "cache": [
                            ""
                        ]
                    },
                    "global_headers": {},
                    "global_headers_remove": [],
                    "global_size_limit": 0,
                    "override_target": ""
                }
            }
        },
        "uptime_tests": {
            "check_list": [],
            "config": {
                "expire_utime_after": 0,
                "service_discovery": {
                    "use_discovery_service": false,
                    "query_endpoint": "",
                    "use_nested_query": false,
                    "parent_data_path": "",
                    "data_path": "",
                    "port_data_path": "",
                    "target_path": "",
                    "use_target_list": false,
                    "cache_timeout": 60,
                    "endpoint_returns_list": false
                },
                "recheck_wait": 0
            }
        },
        "proxy": {
            "preserve_host_header": false,
            "listen_path": "/movies",
            "target_url": "http://www.mocky.io/v2/58a174e110000011295586af",
            "strip_listen_path": true,
            "enable_load_balancing": false,
            "target_list": [],
            "check_host_against_uptime_tests": false,
            "service_discovery": {
                "use_discovery_service": false,
                "query_endpoint": "",
                "use_nested_query": false,
                "parent_data_path": "",
                "data_path": "hostname",
                "port_data_path": "port",
                "target_path": "/api-slug",
                "use_target_list": false,
                "cache_timeout": 60,
                "endpoint_returns_list": false
            }
        },
        "disable_rate_limit": false,
        "disable_quota": false,
        "custom_middleware": {
            "pre": [],
            "post": [],
            "post_key_auth": [],
            "auth_check": {
                "name": "",
                "path": "",
                "require_session": false
            },
            "response": [],
            "driver": "",
            "id_extractor": {
                "extract_from": "",
                "extract_with": "",
                "extractor_config": {}
            }
        },
        "custom_middleware_bundle": "",
        "cache_options": {
            "cache_timeout": 60,
            "enable_cache": false,
            "cache_all_safe_requests": false,
            "cache_response_codes": [],
            "enable_upstream_cache_control": false
        },
        "session_lifetime": 0,
        "active": true,
        "auth_provider": {
            "name": "",
            "storage_engine": "",
            "meta": {}
        },
        "session_provider": {
            "name": "",
            "storage_engine": "",
            "meta": null
        },
        "event_handlers": {
            "events": {}
        },
        "enable_batch_request_support": false,
        "enable_ip_whitelisting": false,
        "allowed_ips": [],
        "dont_set_quota_on_create": false,
        "expire_analytics_after": 0,
        "response_processors": [],
        "CORS": {
            "enable": false,
            "allowed_origins": [],
            "allowed_methods": [],
            "allowed_headers": [],
            "exposed_headers": [],
            "allow_credentials": false,
            "max_age": 24,
            "options_passthrough": false,
            "debug": false
        },
        "domain": "api.dev.westfield.io",
        "do_not_track": false,
        "tags": [],
        "enable_context_vars": false
    }
2

Hi @gwshaw,

I suspect that you may have run into trouble because of the syntax you’re using for your Regular Expression. It looks like you’re using the Perl syntax but Tyk is written in Go and so uses the Go syntax to handle pattern matching which is slightly different (for instance, it isn’t possible to use w+ expression in Go and you typically have to use something like [A-Za-z0-9_-]+ instead).

Anyway, I hope that helps. Let me know if you have any further questions.

Kind regards,
Jess @ Tyk

1 Like
3

I took (w+) from the Tyk documentation: https://tyk.io/tyk-documentation/transform-traffic/url-rewrite/

Using /v1/centres/([A-Za-z0-9_-]+)/movies produces the same result. I’m using Postman to access http://api.dev.westfield.io/v1/centres/foo/movies?api_key=my_api_key

4

I think the issue here is that the first rewrite will not redirect to another API definition, it will send your rewritten URL to your target.

Also, the regex is a little wrong, we need to update it in the docs, this works:

v1\/centres\/(\w+)\/movies

You can see it in action here.

I rewrote it against httpbin as a path querystring argument (get?path=/movies/v1/centres/$1/movies), so I could see the output of the rewrite, as you can see the rewrite worked as expected:

{
  "args": {
    "path": "/movies/v1/centres/test/movies"
  },
  "headers": {
    "Accept": "*/*",
    "Accept-Encoding": "gzip",
    "Accept-Language": "en-GB,en;q=0.8,en-US;q=0.6,de;q=0.4,es;q=0.2",
    "Cache-Control": "no-cache",
    "Cf-Visitor": "{\"scheme\":\"https\"}",
    "Host": "httpbin.org",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
    "X-Scheme": "http"
  },
  "url": "http://httpbin.org/get?path=%2Fmovies%2Fv1%2Fcentres%2Ftest%2Fmovies"
}

If I wanted to target a different upstream host (or even a new API definition) I need to add a hostname component to the URL so it’s fully qualified like so:

https://httbin2.org/get?path=/movies/v1/centres/$1/movies

Tyk will detect that the hostname is different and use the new target hostname instead of the one configured in the API definition.

5

Martin:
Thanks. I put in the regex and that part works. You should put a link to that site in the docs as there is confusion on what regex syntax works here.

I also put in:
http://api.dev.westfield.io/movies/v1/centres/$1/movies
for the rewrite to address. I get this response:
{
“error”: “There was a problem proxying the request”
}
and this shows up in the logs:

bfe96915739c096 org_id=589a65006e3931000128cd5c server_name=api.dev.westfield.io user_id=“****5ef4” user_ip=172.19.0.1 user_name="[email protected]"
tyk-gateway_1 | time=“Feb 15 21:57:52” level=error msg=“request error: There was a problem proxying the request” api_id=24c8281d54364c806bfe96915739c096 org_id=589a65006e3931000128cd5c path=“/movies/v1/centres/foo/movies/movies/v1/centres/foo/movies” server_name=“http://www.mocky.io/v2/58a175411000000f295586b0” user_id= user_ip=172.19.0.1

The path above is incorrect.
Suggestions?

6

There will be another error message right before these ones that will give you the exact reason why the request failed (i.e. it is not shown to the client, but to the admin as it may be sensitive).

7

tyk-gateway_1 | time=“Feb 15 21:57:52” level=error msg=“http: proxy error: dial tcp 127.0.0.1:80: getsockopt: connection refused” api_id=24c8281d54364c806bfe96915739c096 org_id=589a65006e3931000128cd5c server_name=api.dev.westfield.io user_id=“****5ef4” user_ip=172.19.0.1 user_name="[email protected]"

8

Docker Compose is:

version: '2'
services:
    tyk-redis:
        image: redis
        ports:
            - "6379:6379"
    tyk-mongo:
        image: mongo
        ports:
            - "27017:27017"
    tyk-dashboard:
        image: tykio/tyk-dashboard:latest
        ports:
            - "3000:3000"
            - "5000:5000"
        volumes:
            - ./volumes/config/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf
        depends_on:
            - tyk-redis
            - tyk-mongo
    tyk-pump:
        image: tykio/tyk-pump-docker-pub:latest
        volumes:
            - ./volumes/config/pump.conf:/opt/tyk-pump/pump.conf
        depends_on:
            - tyk-redis
            - tyk-mongo
            - tyk-gateway
    tyk-gateway:
        image: tykio/tyk-gateway:latest
        ports:
            - "80:8181"
        volumes:
            - ./volumes/config/tyk.conf:/opt/tyk-gateway/tyk.conf
            - ./volumes/middleware:/opt/tyk-gateway/middleware
            - ./volumes/templates:/opt/tyk-gateway/templates
        depends_on:
            - tyk-redis
9

This is the problem, the target it is trying to access is 127.0.0.1 port 80, the gateway on loopback 127.0.0.1 within the container will be listening on 8181, not 80.

1 Like
10

I was able to get it to work by including the port number in the rewrite address.

Thanks!

1 Like
11

I did some benchmarking to a local server and find about a 25% slowdown due to the url rewrite. Possibly allowing local accesses to api_definitions would be faster? Or could I use localhost and would that work and be faster?

12

Looping an API definition into another one over TCP is inneficient, why not just rewrite to the correct server without running through the gateway again?

13

Feel free to open a feature request on our GH though, we’ll chuck it in our backlog