NPM and JSVM, custom basic-auth plugin more details?

Hi @bitsofinfo, the ID extractor isn’t currently available for JS middleware as we’re still migrating it to our rich plugin architecture.

I suggest following up with the simplest solution which it’s to use a Pre middleware and cover some of the ID extractor logic right there:

    var authMiddleware = new TykJS.TykMiddleware.NewMiddleware({});

    function authenticate(token) {
        log("Calling the backend")
        // Do the TykMakeHttpRequest logic here:
        return token == "fa1b865d9280d4a488afa30fd60216e7"
    }

    authMiddleware.NewProcessRequest(function(request,session,conf) {

        token = request.Headers["Authorization"]

        var session = JSON.parse(TykGetKeyData(token))
        if (session.status == "error") {
            log("Session not found, hitting the backend")
            var valid = authenticate(token)
            if(!valid) {
                request.ReturnOverrides.ResponseCode = 401;
                request.ReturnOverrides.ResponseError = "Failed auth"
                return authMiddleware.ReturnData(request, {})
            }
            session = {
                            "allowance": 100,
                            "rate": 100,
                            "per": 1,
                            "quota_max": -1,
                            "quota_renews": 1406121006,
                            "access_rights": {}
                    };
            TykSetKeyData(token, JSON.stringify(session), "0")
            return authMiddleware.ReturnData(request, {});
        }

        return authMiddleware.ReturnData(request, {});
    });

The calls to TykGetKeyData and TykSetKeyData covers the ID extractor functionality in this case, you should replace the authenticate function with your own logic.

The API definition custom middleware block will look like this:

    "custom_middleware": {
      "pre": [{
        "name": "authMiddleware",
        "path": "middleware/authMiddleware.js"
      }]
    },

Best regards.

Thanks @matiasb

So here is my updated plugin, I changed it from “auth_check” to “pre” as you noted.

Here is the log from the updated code below when invoked. The key error being “Master keys disallowed in configuration, key not added.” after my auth endpoint is successfully invoked and it tryes to set the key

2017-07-24T14:04:28.399589698Z time="Jul 24 14:04:28" level=warning msg="Failed to retrieve key detail." err="not found" key="****Iw==" status=fail
2017-07-24T14:04:28.399829778Z time="Jul 24 14:04:28" level=info msg="No session exists, attempting auth and creating" type=log-msg
2017-07-24T14:04:29.018437419Z time="Jul 24 14:04:29" level=info msg=200 type=log-msg
2017-07-24T14:04:29.018559040Z time="Jul 24 14:04:29" level=error msg="Master keys disallowed in configuration, key not added."
2017-07-24T14:04:29.025426429Z time="Jul 24 14:04:29" level=error msg="Failed to decode middleware request data on return from VM: invalid character 'u' looking for beginning of value"

log("====> myAuth Auth initialising");

var myAuth = new TykJS.TykMiddleware.NewMiddleware({});

myAuth.NewProcessRequest(function(request, session, config) {
    log("----> Running myAuth JSVM Auth Middleware");
    
    log("config= " + JSON.stringify(config));

    log("session= " + JSON.stringify(session));

    log("request= " + JSON.stringify(request));

        var rawAuthorization = request.Headers["Authorization"];

        if (rawAuthorization && rawAuthorization.length > 0) {
            rawAuthorization = rawAuthorization[0];
        } else {
            request.ReturnOverrides.ResponseCode = 401;
        	request.ReturnOverrides.ResponseError = 'Authentication required';
        	return myAuth.ReturnData(request, {});
        }


        log(rawAuthorization);
        
        var session = JSON.parse(TykGetKeyData(rawAuthorization))
        
        if (session.status == "error") {

			log("No session exists, attempting auth and creating");
			
			var newRequest = {
					"Method": "GET",
					"Body": "",
					"Headers": {
							"Authorization":rawAuthorization
					 },
					"Domain": "http://192.168.0.148:9080",
					"Resource": "/auth/util/xxxx"
			};

			var resp = TykMakeHttpRequest(JSON.stringify(newRequest));

			var respObj = JSON.parse(resp);

			log(respObj.Code);

			if (respObj.Code != 200) {
				request.ReturnOverrides.ResponseCode = 401;
				request.ReturnOverrides.ResponseError = 'MY Auth Failed: ' + respObj.code;
				return myAuth.ReturnData(request, {});
			}
			
			session = {
                            "allowance": 100,
                            "rate": 100,
                            "per": 1,
                            "quota_max": -1,
                            "quota_renews": 1406121006,
                            "access_rights": {}
                    };
                    
            TykSetKeyData(rawAuthorization, JSON.stringify(session), "0");
            return myAuth.ReturnData(request, {});
			
		} else {
			log("Pre-existing session exists... using, no MY auth check");
		}

    return myAuth.ReturnData(request, {});
});

// Ensure init with a post-declaration log message
log("====> myAuth initialised");

So I added “allow_master_keys” true, in the gateway tyk.conf file, now I get this. Looks like the key is being updated in Tyk, but now I get

"Session state is missing or unset! Please make sure that auth headers are properly applied."

2017-07-24T14:14:42.490281990Z time="Jul 24 14:14:42" level=warning msg="Failed to retrieve key detail." err="not found" key="****Iw==" status=fail
2017-07-24T14:14:42.490329969Z time="Jul 24 14:14:42" level=info msg="No session exists, attempting auth and creating" type=log-msg
2017-07-24T14:14:43.705279246Z time="Jul 24 14:14:43" level=info msg=200 type=log-msg
2017-07-24T14:14:43.705368576Z time="Jul 24 14:14:43" level=warning msg="No API Access Rights set, adding key to ALL."
2017-07-24T14:14:43.705386384Z time="Jul 24 14:14:43" level=info msg="Reset quota for key." inbound-key="****Iw==" key=quota-24edb24c
2017-07-24T14:14:43.705395856Z time="Jul 24 14:14:43" level=info msg="Reset quota for key." inbound-key="****Iw==" key=quota-24edb24c
2017-07-24T14:14:43.705791609Z time="Jul 24 14:14:43" level=info msg="Reset quota for key." inbound-key="****Iw==" key=quota-24edb24c
2017-07-24T14:14:43.705851971Z time="Jul 24 14:14:43" level=info msg="Reset quota for key." inbound-key="****Iw==" key=quota-24edb24c
2017-07-24T14:14:43.705869363Z time="Jul 24 14:14:43" level=info msg="Key added or updated." api_id=-- expires=0 key="****Iw==" org_id= path=-- server_name=system user_id=system user_ip=--
2017-07-24T14:14:43.712565120Z time="Jul 24 14:14:43" level=error msg="Failed to decode middleware request data on return from VM: invalid character 'u' looking for beginning of value"
2017-07-24T14:14:43.712639795Z time="Jul 24 14:14:43" level=error msg="request error: Session state is missing or unset! Please make sure that auth headers are properly applied." api_id=8c685ec987cf4fc16cef3d7cf2683a22 org_id=5975fbce52d58f000138b93f path="/" server_name="http://192.168.0.148:9080/test/auth" user_id= user_ip=172.18.0.1

Hi @bitsofinfo, looks like there’s a JS error somewhere, note the Failed to decode middleware request data on return from VM error.

Yeah I’ve looked for that but I’m not seeing what it is (pasted code above)

I mean this line executes as I see its log output from tyk “key added or updated”

TykSetKeyData(rawAuthorization, JSON.stringify(session), "0");

So whats wrong w/ this line?

return myAuth.ReturnData(request, {});

I’ve even added a try/catch around the return block and no error is thrown, I log the output of the call to “ReturnData” before returning and I get this below. I then just return it, then that log line appears, so I have no clue what is generating that undefined error other than whatever is invoking my function?

{\"Request\":{\"AddParams\":{},\"Body\":\"\",\"DeleteHeaders\":[],\"DeleteParams\":[],\"ExtendedParams\":{},\"Headers\":{\"Accept\":[\"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\"],\"Accept-Encoding\":[\"gzip, deflate, br\"],\"Accept-Language\":[\"en-US,en;q=0.8,pt;q=0.6\"],\"Authorization\":[\"Basic xxxxx==\"],\"Cache-Control\":[\"max-age=0\"],\"Connection\":[\"keep-alive\"],\"Cookie\":[\"csrf_token=mwtnbNEb56sH7RjlK0FaoBlFppL2fgjaaAZzWbl+hHc=; authorisation=xxxxx-d084-432d-6324-abccd88c8fcd\"],\"Upgrade-Insecure-Requests\":[\"1\"],\"User-Agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36\"]},\"IgnoreBody\":false,\"Params\":{\"wsdl\":[\"\"]},\"ReturnOverrides\":{\"ResponseCode\":0,\"ResponseError\":\"\"},\"SetHeaders\":{},\"URL\":\"/ocd/auth/10\"},\"SessionMeta\":{}}" type=log-msg

@matiasb

I am at a loss as to what is causing that undefined (note the log statements following this dumbed down JS middleware snippet)

log("====> myAuth Auth initialising");

var myAuth = new TykJS.TykMiddleware.NewMiddleware({});

myAuth.NewProcessRequest(function(request, session, config) {

    log("----> Running myAuth JSVM Auth Middleware");
    
    log("config= " + JSON.stringify(config));

    log("session= " + JSON.stringify(session));

    log("request= " + JSON.stringify(request));

        var rawAuthorization = request.Headers["Authorization"];

        if (rawAuthorization && rawAuthorization.length > 0) {
            rawAuthorization = rawAuthorization[0];
        } else {
            request.ReturnOverrides.ResponseCode = 401;
        	request.ReturnOverrides.ResponseError = 'Authentication required';
        	return myAuth.ReturnData(request, {});
        }


        log(rawAuthorization);
        
        var session = JSON.parse(TykGetKeyData(rawAuthorization))
        
        if (session.status == "error") {

			log("No session exists, attempting auth (fake) and creating");

			session = {
                            "allowance": 100,
                            "rate": 100,
                            "per": 1,
                            "quota_max": -1,
                            "quota_renews": 1406121006,
                            "access_rights": {}
                    };
			
			log("6");
            TykSetKeyData(rawAuthorization, JSON.stringify(session), "0");
            
    		log("7");
    	    return myAuth.ReturnData(request,{});
			
		} else {
			log("Pre-existing session exists... using, no MY auth check");
			log("8");
			
			try {
		    	var x = myAuth.ReturnData(request,{});
		    	log("9");
		    	log(JSON.stringify(x));
		    	log("10");
		    	return x;
		    	
		    } catch(e) {
		    	log(JSON.stringify(e));
		    }
		}

});

// Ensure init with a post-declaration log message
log("====> myAuth initialised");



2017-07-24T19:29:09.280516100Z time="Jul 24 19:29:09" level=info msg="----> Running myAuth JSVM Auth Middleware" type=log-msg
2017-07-24T19:29:09.281039200Z time="Jul 24 19:29:09" level=info msg="config= {\"config_data\":{\"foo\":\"bar\"}}" type=log-msg
2017-07-24T19:29:09.282068700Z time="Jul 24 19:29:09" level=info msg="session= {\"access_rights\":null,\"alias\":\"\",\"allowance\":0,\"apply_policy_id\":\"\",\"basic_auth_data\":{\"hash_type\":\"\",\"password\":\"\"},\"data_expires\":0,\"enable_detail_recording\":false,\"expires\":0,\"hmac_enabled\":false,\"hmac_string\":\"\",\"id_extractor_deadline\":0,\"is_inactive\":false,\"jwt_data\":{\"secret\":\"\"},\"last_check\":0,\"last_updated\":\"\",\"meta_data\":null,\"monitor\":{\"trigger_limits\":null},\"oauth_client_id\":\"\",\"oauth_keys\":null,\"org_id\":\"\",\"per\":0,\"quota_max\":0,\"quota_remaining\":0,\"quota_renewal_rate\":0,\"quota_renews\":0,\"rate\":0,\"session_lifetime\":0,\"tags\":null}" type=log-msg
2017-07-24T19:29:09.282483400Z time="Jul 24 19:29:09" level=info msg="request= {\"AddParams\":{},\"Body\":\"\",\"DeleteHeaders\":[],\"DeleteParams\":[],\"ExtendedParams\":{},\"Headers\":{\"Accept\":[\"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\"],\"Accept-Encoding\":[\"gzip, deflate, br\"],\"Accept-Language\":[\"en-US,en;q=0.8,pt;q=0.6\"],\"Authorization\":[\"Basic xxx==\"],\"Cache-Control\":[\"max-age=0\"],\"Connection\":[\"keep-alive\"],\"Cookie\":[\"csrf_token=mwtnbNEb56sH7RjlK0FaoBlFppL2fgjaaAZzWbl+hHc=; authorisation=8302fb04-f44f-434d-6d0c-c21dd2656c54\"],\"Upgrade-Insecure-Requests\":[\"1\"],\"User-Agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36\"]},\"IgnoreBody\":false,\"Params\":{\"wsdl\":[\"\"]},\"ReturnOverrides\":{\"ResponseCode\":0,\"ResponseError\":\"\"},\"SetHeaders\":{},\"URL\":\"/my/auth/10\"}" type=log-msg
2017-07-24T19:29:09.283081200Z time="Jul 24 19:29:09" level=info msg="Basic xxx==" type=log-msg
2017-07-24T19:29:09.284802700Z time="Jul 24 19:29:09" level=info msg="Retrieved key detail." key="****Iw==" status=ok
2017-07-24T19:29:09.284848800Z time="Jul 24 19:29:09" level=info msg="Pre-existing session exists... using, no MY auth check" type=log-msg
2017-07-24T19:29:09.284864800Z time="Jul 24 19:29:09" level=info msg=8 type=log-msg
2017-07-24T19:29:09.285206900Z time="Jul 24 19:29:09" level=info msg=9 type=log-msg
2017-07-24T19:29:09.285592100Z time="Jul 24 19:29:09" level=info msg="{\"Request\":{\"AddParams\":{},\"Body\":\"\",\"DeleteHeaders\":[],\"DeleteParams\":[],\"ExtendedParams\":{},\"Headers\":{\"Accept\":[\"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\"],\"Accept-Encoding\":[\"gzip, deflate, br\"],\"Accept-Language\":[\"en-US,en;q=0.8,pt;q=0.6\"],\"Authorization\":[\"Basic xxx==\"],\"Cache-Control\":[\"max-age=0\"],\"Connection\":[\"keep-alive\"],\"Cookie\":[\"csrf_token=mwtnbNEb56sH7RjlK0FaoBlFppL2fgjaaAZzWbl+hHc=; authorisation=8302fb04-f44f-434d-6d0c-c21dd2656c54\"],\"Upgrade-Insecure-Requests\":[\"1\"],\"User-Agent\":[\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36\"]},\"IgnoreBody\":false,\"Params\":{\"wsdl\":[\"\"]},\"ReturnOverrides\":{\"ResponseCode\":0,\"ResponseError\":\"\"},\"SetHeaders\":{},\"URL\":\"/my/auth/10\"},\"SessionMeta\":{}}" type=log-msg
2017-07-24T19:29:09.285655500Z time="Jul 24 19:29:09" level=info msg=10 type=log-msg
2017-07-24T19:29:09.289640900Z time="Jul 24 19:29:09" level=error msg="Failed to decode middleware request data on return from VM: invalid character 'u' looking for beginning of value"
2017-07-24T19:29:09.289697800Z time="Jul 24 19:29:09" level=error msg="request error: Session state is missing or unset! Please make sure that auth headers are properly applied." api_id=8c685ec987cf4fc16cef3d7cf2683a22 org_id=5975fbce52d58f000138b93f path="/" server_name="http://192.168.0.148:9080/service/xxx" user_id= user_ip=172.18.0.1

I think this is solved, the issue was the auth type on the API.

Was set to auth plugin, when I changed it to “keyless” now it seems to work when using the “pre” middleware instead of “auth_check”

I would suggest:

  1. Changing your auth mechanism to “Auth Token” and set the location of the token to use the Header x-internal-authorization
  2. In your plugin, hash the rawAuthorization string
  3. Use the hash as the key for the session (or, if you want to go a step further, use orgID+hash(rawAuthorization))
  4. Add the hash as an x-internal-authorization header

Reasoning:

  1. Open API’s do not process rate limits or quotas
  2. Open APIs, since there is no token, do not collect analytics per user
  3. It means you can modify the session from the dashboard (if you use the orgID as a prefix)

I would also strongly suggest turning off master keys, it basically means that the inbound session has access to ALL APIs if the access list is empty. This might be ok for a PoC, but it’s not a safe mode.

M.

Thanks

Changing your auth mechanism to “Auth Token” and set the location of the token to use the Header x-internal-authorization

Does this mean that the actual external CLIENT of my API must set this? Or that my PRE middleware JS can set it, prior to any of the auth token management in the gateway runs?

In your plugin, hash the rawAuthorization string

What hash function would you recommend, or are available w/ the JSVM env? Also this key is stored in redis correct, so without a secure hash, potentially their credentials would be exposed in redis to anyone with access to it, correct?

Use the hash as the key for the session (or, if you want to go a step further, use orgID+hash(rawAuthorization))

How can I get the apiId and orgId from within one of these JSVM middleware functions?

Add the hash as an x-internal-authorization header

I assume you mean like request.SetHeaders = { "x-internal-authorization" : authHash };

I would also strongly suggest turning off master keys, it basically means that the inbound session has access to ALL APIs if the access list is empty. This might be ok for a PoC, but it’s not a safe mode.

Makes sense, do you have an example of the proper structure I can use to set this dynamically on that session object that I return? The one’s I’ve seen seem to require an id/uuid, which… doesn’t pre-exist etc?

Your PRE middleware should set it, since it is a pre-processor, the header will carry through to the Tyk auth token middleware

There’s no hash available in the env, so you may need to use a pre-baked one like this one liner

If you have enable hash_keys in your tyk.conf, Tyk will secure the credentials:

hash_keys

Set this value to true to enable key hashing, this will start hashing all keys that are generated by Tyk in Redis. Enabling this means that keys that are created are only exposed once, during creation. If the key is lost or misplaced after this it will not be retrievable (however its hash can still be deleted, if known).

If this is set to true, the same value should be enabled in the Dashboard configuration so that the UI can react appropriately.

You don’t need the API ID, you can pass these values in the config_data portion of the API definition.

Correct

The ID you see is the API ID, so it would be this in the root of the session object:

        "access_rights": {
            "API_ID": {
                "api_name": "ANY NAME",
                "api_id": "API_ID",
                "versions": ["Default"],
                "allowed_urls": null
            }
        },
        "org_id": "53ac07777cbb8c2d53000002",

The api_name is for humans, so you can set it to anything. And thed API ID is the same as that in the API Definition (so you could pass it in config).

Just to complicate things, and maybe make things more controllable, you could put a dummy API ID in there (create an API with a single Mock on a whitelist for example), and then set the apply_policy_id field to a POlicy ID.

You can then manage the ACL for all tokens created by this middleware from the dashboard by editing the referenced Policy.

Great, I am doing it this way.

you would need to do that yourself or as per the suggestion in the other post, you let Tyk do it for you when you set an internal auth header.

If you have enable hash_keys in your tyk.conf, Tyk will secure the credentials:

So if this is enabled, then I DON’T need to pre-hash the key before setting it? Tyk will just do it with Murmur3 automatically when TykGetKeyData or TykSetKeyData is called w /a “cleartext” key value?

Yes, that is correct :slight_smile:

@Martin

In reply to this comment: NPM and JSVM, custom basic-auth plugin more details? - #29 by Martin

What I’m noticing is that since "hash_keys": true, is on in tyk.conf, this solution does not allow me to accurately see “Activity by Key” in the dashboard at http://localhost:3000/#/activity-key

Reason appears to be, because I have no way to hash the value in the same way that TykSetKeyData is doing… so my x-internal-authorization header value for the Auth Token based security never matches up.

What is the solution for this? Disable “hash_keys” completely?

It seems that what I should be able to do is:

a) set hash_keys:true in tyk.conf

b) setup my api w/ Auth Token security based on the x-internal-authorization header

c) In my middleware do the following

request.SetHeaders = { "x-internal-authorization" : murmur3(myAuthorizationHeaderValue) }; (but I can’t do this… no golang murmur3 function access exposed to JSVM?)

d) then lower in my middleware do

TykSetKeyData(myAuthorizationHeaderValue, JSON.stringify(newSession));

Without being able to set that header = to what TykSetKeyData hashes the key as, this seems to be of no use. That or turn OFF hash_keys but then I am foregoing some security for the keys in redis that I am not managing… no?

Setting hash_keys:false observations

That said, if i set hash_keys:false in both the dashboard and gateway’s tyk.conf and restart everything and the middlware sets x-internal-authorization to a non-hashed authorization header value (same as passed to TykSetKeyData) I then start seeing the key under “Activity by Key” but the numbers are not accurate at all and I don’t see it incrementing or at least if it does, there is significant latency, like > 10 minutes…

Also searching in the dashboard under keys → “View Keys”, no keys are listed and if I search for the literal non-hashed key value, its not found. (Note the dashboard view now looks different then when in hash_keys:true mode). The key I am searching for IS LISTED under “Activity by Key”

In your session object that you return, set the alias field to a value that allows you to identify the user, this will transfer through into the analytics along with the hash so that the key data becomes meaningful

You can also set tags on the key, these also transfer through and let you filter your aggregates.

You do not need to do this, just use the alias method. If you want to see the raw token in your analytics, then you should disable hashed keys altogether.

What value do you have you tyk pump set to purge at? We recommend a low value like 2 or 5 seconds

Keys → View Keys only applies to non-hashed mode, and if you want to see keys that are generated by your JS MW, you must prefix them with your Org ID (as was the initial suggestion) due to the tenancy filters:

What value do you have you tyk pump set to purge at? We recommend a low value like 2 or 5 seconds

pump.conf is set to "purge_delay": 2,, i’ve thrown 30 requests at the api, but only see a count of “1”. And its been > 5 minutes. Seems like it eventually flows in, but definitely, longer than 2… (minutes?) Seems to be 10m, like the 2 is having no effect. (I’m just using the docker quick start docker-compose based install)

You do not need to do this, just use the alias method.

So i re-enabled hash_keys in the dashboard and gateway. I also set an alias in the session I return

Now I see this under “Activity by Key” for the key I am looking for and can click through for the details.

murmur3Hash (myAlias)

I can search and find it by the murmur3Hash, but i get zero results when searching by the alias… Should I expect to be able to?

you must prefix them with your Org ID (as was the initial suggestion) due to the tenancy filters

Ok will try, that however I don’t understand this. Is there a doc or can you explain this? Are you saying there is something in the Auth Token header processing or TykSetKeyData that can detect the prefix of an Org ID and do something different? I don’t get it

Also, where can I find the org id

Search by alias hasn’t been enabled yet.

The dashboard can have many tenants, as in organisations/teams, tokens are separated by ownership. Because it is possible to have inhashed listings of keys in the dashboard, the key itself is namespaced and the org ID is part of the token.

The namespace is used to ensure the token being edited is owned by the org that is doing the editing.

As for your analytics - not sure what’s going on there, yu might want to try making the end date in your filter set to tomorrow, it could be a time zone thing.

Where can I find the value for the orgId?

Hi bitsofinfo,

have you had a look at the docs ?
You can use the Admin API to retrieve the org id.
(Just noticed that there’s a mistake on that section, it should be a GET pointing to /admin/organisations/.)

Otherwise you can find it in the “Edit user” section on the bottom as “RPC Credentials”.

Thanks,
Kos @ Tyk Support Team

thanks, found it under the rpc creds