gRPC plugin reload makes it unaccessible?

Support
1

Hello everyone,

I’m using a NodeJS app as a gRPC middleware. But I started to get an issue that I didn’t notice before.
When I update my NodeJS code and restart the server the communication with Tyk seems to be broken until I restart the gateway. I get the following error in the logs :

tyk[25891]: 2017/11/02 17:49:39 transport: http2Client.notifyError got notified that the client transport was broken EOF.

And then subsequent requests are just hanging… No error, no response, no timeout as far as I can see. But it only impacts the API using the middleware, not the other ones.

I’m almost sure last week I could restart node without having the issue.

Gateway version is 2.3.12.

Thanks !
Robert

2

Hi @Rouzz, do you get any error in your NodeJS program?

Are you getting “http2client.notifyError” on every request or just once?

Best.

3

Hi @matiasb,

There is no error displayed in the NodeJS console.

Sorry I wasn’t clear enough, the “http2client.notifyError” that I get appears as soon as I’m stopping the Node server. And then when I restart it the requests never reach it again.

I could swear that last week when stopping the NodeJS server I had some traces in the logs saying connection was lost and then when starting it back it worked. But I cannot find any example in my service logs :frowning:

4

Thanks for providing the details, are you using a NodeJS plugin based on one of our samples?

To reproduce this we might need to try an actual implementation. We haven’t experienced any issues with gRPC reconnection in the past.

5

Hi @matiasb,

I tried again today and couldn’t reproduce it anymore…

Now after the error :
transport: http2Client.notifyError got notified that the client transport was broken EOF.
I can see that the gateway tries to reconnect with the gRPC plugin :
grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:5555: getsockopt: connection refused"; Reconnecting to { <nil>}

In any case if the issue appears again, here is my plugin code :

  • package.json

{
“name”: “tyk-grpc”,
“version”: “1.0.0”,
“description”: “”,
“main”: “main.js”,
“scripts”: {
“test”: “echo "Error: no test specified" && exit 1”
},
“author”: “”,
“license”: “ISC”,
“dependencies”: {
“grpc”: “^1.6.6”
}
}

  • manifest.json

{
“custom_middleware”: {
“pre”:[{
“name”: “MyPreMiddleware”,
“require_session”: false
}],
“post”:[{
“name”: “MyPostMiddleware”,
“require_session”: false
}],
“driver”: “grpc”,
“auth_check”: {
“name”: “MyAuthMiddleware”
}
}
}

  • main.js

const grpc = require(‘grpc’),
resolve = require(‘path’).resolve

const tyk = grpc.load({
file: ‘coprocess_object.proto’,
root: resolve(__dirname, ‘tyk-protobuf/proto’)
}).coprocess

const listenAddr = ‘127.0.0.1:5555’,
authHeader = ‘Authorization’
validToken = ‘71f6ac3385ce284152a64208521c592b’

const directusMiddleware = require(‘./directusMiddleware’)

// The dispatch function is called for every hook:
const dispatch = (call, callback) => {
let obj = call.request
console.log(obj.spec.APIID)
switch (obj.spec.APIID) {
case ‘6cf95519e7e348cb44d26c73ee596b34’:
directusMiddleware.dispatch(obj, callback)
break
default:
console.log('no api with id: ’ + obj.spec.APIID)
callback(null, obj)
break
}

}

main = function() {
server = new grpc.Server()

server.addService(tyk.Dispatcher.service, {
    dispatch: dispatch
})
server.bind(listenAddr, grpc.ServerCredentials.createInsecure())
server.start()

}

main()

  • directusMiddleware.js

let directusMiddleware = {
preMiddleware: (obj, callback) => {
var req = obj.request

    // req is the coprocess.MiniRequestObject, we inject a header using the "set_headers" field:
    req.set_headers = {
        'mycustomheader': 'pre'
    }
    // console.log(obj)

    // Use this callback to finish the operation, sending back the modified object:
    callback(null, obj)
},

postMiddleware: (obj, callback) => {
    var req = obj.request

    // console.log(obj)

    // req is the coprocess.MiniRequestObject, we inject a header using the "set_headers" field:
    req.set_headers = {
        'mycustomheader': 'post'
    }

    req.return_overrides = {
        response_code: 200,
        response_error: JSON.stringify({"test":"value"}),
        headers: {
            'myresponseheader': 'value'
        }
    }

    // Use this callback to finish the operation, sending back the modified object:
    callback(null, obj)
},

authMiddleware: (obj, callback) => {
    var req = obj.request

    // We take the value from the "Authorization" header:
    var token = req.headers[authHeader]

    // The token should be attached to the object metadata, this is used internally for key management:
    obj.metadata = {
        token: token
    }

    // If the request token doesn't match the  "validToken" constant we return the call:
    if (token != validToken) {
        callback(null, obj)
        return
    }

    // At this point the token is valid and a session state object is initialized and attached to the coprocess.Object:
    var session = new tyk.SessionState()
    session.id_extractor_deadline = Date.now() + 100000000000
    obj.session = session
    callback(null, obj)
},

dispatch: (obj, callback) => {
    console.log(obj.hook_name)
    // We dispatch the request based on the hook name, we pass obj.request which is the coprocess.Object:
    switch (obj.hook_name) {
        case 'MyPreMiddleware':
            directusMiddleware.preMiddleware(obj, callback)
            break
        case 'MyPostMiddleware':
            directusMiddleware.postMiddleware(obj, callback)
            break
        case 'MyAuthMiddleware':
            directusMiddleware.authMiddleware(obj, callback)
            break
        default:
            callback(null, obj)
            break
    }
}

}

module.exports = directusMiddleware

Basically the only thing I did here was adding a multi-API management in my dispatcher and then use return_overrides. All for the sake of trying to understand how things could workout if I happened to need it for my project.

Spoiler alert, the client decided to go with Tyk Cloud so my experimentation on Tyk middleware is over :wink:

Cheers,
Robert