Can't make the virtual endpoints to work

jaroslav.psenicka · August 19, 2024, 12:52pm

Hi community,

I am new to Tyk, trying to implement a virtual endpoint.

the enable_jsvm is set to true in tyk.config
I have copied one of the examples, I see the “Virtual Test initialised” being printed in the gateway log during startup (see below)
I have configured the extended path as virtual

when requesting http://localhost:8080/employee/1, I am receiving content from the remote URL, not the virtual endpoint. Renaming the virtual endpoint to “/virtual” (and restart) gives me 404. Nothing in log (set to debug).

Could you help?
Many thanks in advance.

JP

the config, virtual extended path:

"use_extended_paths": true,
"extended_paths": {
  "virtual": [{
    "path": "/employee",
    "method": "GET",
    "use_session": false,
    "proxy_on_error": false,
    "response_function_name": "myUniqueFunctionName",
    "function_source_type": "file",
    "function_source_uri": "./middleware/myUniqueFunctionName.js"
  }]
}

the config, proxy:

"proxy": {
  "preserve_host_header": false,
  "listen_path": "/employee",
  "target_url": "https://dummy.restapiexample.com/api/v1/employee",
  "disable_strip_slash": false,
   ...
}

the function:

function myUniqueFunctionName(request, session, config) {  
  log("Request Body: " + request.Body)
  log("Session: " + JSON.stringify(session.allowance))
  log("Config: " + JSON.stringify(config.APIID))
  log("param-1: " + request.Params["param1"]) // case sensitive
  log("auth Header: " + request.Headers["Authorization"]) // case sensitive
  
  var responseObject = {
    Body: "VIRTUAL ENDPOINT EXAMPLE #1",
    Headers: {
      "x-test": "virtual-header",
      "x-test-2": "virtual-header-2"
    },
    Code: 200
  }
  
  return TykJsResponse(responseObject, session.meta_data)   
}

log("Virtual Test initialised")

log:

2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Tyk API Gateway 5.5.0" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=warning msg="Insecure configuration allowed" config.allow_insecure_configs=true prefix=checkup
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="storage: Redis is either down or was not configured"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="Could not set version in versionStore" error="storage: Redis is either down or was not configured" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Notifier will not work in hybrid mode" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="storage: Redis is either down or was not configured"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="No Primary instance found, assuming control" prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="storage: Redis is either down or was not configured"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="cannot set key in pollerCacheKey" error="storage: Redis is either down or was not configured"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Starting Poller" prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="---> Initialising checker" prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Config:TriggerLimit: 3"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Config:Timeout: ~10"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Config:WorkerPool: 12"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Init complete"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="---> Starting checker" prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Starting..."
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Check loop started..."
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Host reporter started..."
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="---> Checker started." prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="PIDFile location set to: /var/run/tyk/tyk-gateway.pid" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=warning msg="The control_api_port should be changed for production" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Initialising default org store" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Initialising Tyk REST API Endpoints" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="storage: Redis is either down or was not configured"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="Connection to Redis failed, reconnect in 10s" error="storage: Redis is either down or was not configured" prefix=pub-sub
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="job run successful" name=purge-oauth-tokens prefix=scheduler
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Creating new default Storage connection"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Creating new cache Storage connection"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Creating new analytics Storage connection"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Loaded API Endpoints" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="--> Standard listener (http)" port=":8080" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=warning msg="Starting HTTP server on:[::]:8080" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="[RATELIMIT] DRL with Redis Rate Limiter enabled (using pipeline)"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Initialising distributed rate limiter" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="DRL: Setting node ID: solo-045c8a78-defd-41e8-88ec-1e810b96974b|458b2b071661"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Tyk Gateway started (5.5.0)" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="--> Listening on address: (open interface)" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="--> Listening on port: 8080" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="--> PID: 1" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Starting gateway rate limiter notifications..."
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=error msg="storage: Redis is either down or was not configured"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Default JSVM timeout used: 5s" prefix=jsvm
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Loading policies" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Loading policy from dir /opt/tyk-gateway/policies/policies.json"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Policies found (1 total):" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg=" - " prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Loading API Specification from /opt/tyk-gateway/apps/employees.json"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Initializing JSVM"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Default JSVM timeout used: 5s" prefix=jsvm
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Checking for transform paths..."
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Checking for transform paths..."
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Loading JS Endpoint File: ./middleware/myUniqueFunctionName.js" prefix=jsvm
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Virtual Test initialised" prefix=jsvm type=log-msg
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Detected 1 APIs" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Loading API configurations." prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Tracking hostname" api_name=employee domain="(no host)" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Initialising Tyk REST API Endpoints" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Loaded API Endpoints" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="API bind on custom port:0" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Initializing API" api_id=employee api_name=employee org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Loading JS Paths" api_id=employee api_name=employee org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Checking security policy: Open" api_id=employee api_name=employee org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg=Init api_id=employee api_name=employee mw=VersionCheck org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg=Init api_id=employee api_name=employee mw=RateCheckMW org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg=Init api_id=employee api_name=employee mw=VirtualEndpoint org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Setting Listen Path: /employee" api_id=employee api_name=employee org_id=1
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="API Loaded" api_id=employee api_name=employee org_id=1 prefix=gateway server_name=-- user_id=-- user_ip=--
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Checker host list" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Loading uptime tests..." prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="--- Setting tracking list up" prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Reset initiated" prefix=host-check-mgr
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="[HOST CHECKER] Checker reset queued!"
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=debug msg="Checker host Done" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="Initialised API Definitions" prefix=main
2024-08-19 15:04:13 time="Aug 19 13:04:13" level=info msg="API reload complete" prefix=main
2024-08-19 15:04:23 time="Aug 19 13:04:23" level=debug msg="[HOST CHECKER] Host list reset"
2024-08-19 15:04:23 time="Aug 19 13:04:23" level=debug msg="Error trying to get value:key not found"
2024-08-19 15:04:23 time="Aug 19 13:04:23" level=debug msg="No Primary instance found, assuming control" prefix=host-check-mgr
2024-08-19 15:04:33 time="Aug 19 13:04:33" level=debug msg="Primary instance set, I am master" prefix=host-check-mgr

jaroslav.psenicka · August 19, 2024, 1:42pm

Oh, it looks like the following URL is working

http://localhost:8080/employee/virtual

so the virtual endpoint is somehow tied to the listen_path value even when defined with a leading slash. Is this expected behaviour? Could some of you experienced guys comment on it?

Thanks,
JP

virtual endpoint definition:

        "use_extended_paths": true,
        "extended_paths": {
          "virtual": [
            {
              "path": "/virtual",
              "method": "GET",
              "use_session": false,
              "proxy_on_error": false,
              "response_function_name": "myUniqueFunctionName",
              "function_source_type": "file",
              "function_source_uri": "./middleware/myUniqueFunctionName.js"
            }
          ]

Pete · August 21, 2024, 10:32am

Hi,

You’re right, the endpoints are under the listen path of the API as they are a property of that API.
The leading slash is in fact optional as the endpoints are always appended to the listen path.

A screenshot from the pro version makes this clearer.

Cheers,
Pete

jaroslav.psenicka · August 21, 2024, 12:28pm

Thanks Pete,

that changes the plan a little bit. My intention was

first, to publish a simple endpoint returning data from a backend service,
later on, we may need to merge data returned from the original backend service with another data, say the backend service is an user profile and we need to add a client classification field.

To implement this using the virtual endpoint, I probably need to implement both queries (to the backend as well as client classification service) manually in the function. I wonder whether there is another technique which allows “enriching” the data. Maybe the tranformation? Can I make a REST call from the (Go) function?

Thank you,
JP

Ubong · August 26, 2024, 10:04am

Hi @jaroslav.psenicka,

Yes, the docs show samples on making and even batching HTTP requests.