Unable to distribute system in working state

Using the on-premise Docker Pro Demo documentation on your site and your GitHub pro demo repository as starting points, I am able to quickly get up and running.

When I try to adjust this setup from an all-in-one approach to a distributed installation using 4 VM’s:

  • redis

  • gateway and pump

  • dashboard

  • mongo

I am able to get it running successfully, but some important features do not work. I am unable to find a reason for this.

Specifically, I am following some tutorials on how to create an API with multiple authentication endpoints as seen in this video:

but every time I try to create a key, my installation fails with the below error:

** From Dashboard interface **

Failed to save new session object to Tyk. Attempted administrative access with invalid or missing key!

** From Gateway logs **

time=“Feb 25 03:33:48” level=warning msg=“Attempted administrative access with invalid or missing key!” prefix=main

** From Dashboard Logs **

time=“Feb 25 03:33:43” level=error msg=“Attempted administrative access with invalid or missing key!”

time=“Feb 25 03:33:43” level=warning msg=“Failed create:Attempted administrative access with invalid or missing key!”

time=“Feb 25 03:33:43” level=error msg=“Failed to save new key object to Tyk: Attempted administrative access with invalid or missing key!”

From what I have managed to find online, this is usually caused by the secret in the tyk.conf and tyk_analytics.conf files not matching, however, see the following files which I am using for my installation, these secret values are unchanged from your GitHub repo.

tyk.conf

{

  "listen_address": "",

  "listen_port": 443,

  "secret": "352d20ee67be67f6340b4c0605b044b7",

  "node_secret": "352d20ee67be67f6340b4c0605b044b7",

  "template_path": "./templates",

  "tyk_js_path": "./js/tyk.js",

  "middleware_path": "./middleware",

  "policies": {

    "policy_source": "service",

    "policy_connection_string": "https://api-admin.devel.nz",

    "policy_record_name": "tyk_policies",

    "allow_explicit_policy_id": true

  },

  "use_db_app_configs": true,

  "db_app_conf_options": {

    "connection_string": "https://api-admin.devel.nz",

    "node_is_segmented": false,

    "tags": [

      "test"

    ]

  },

  "disable_dashboard_zeroconf": false,

  "app_path": "./test_apps/",

  "storage": {

    "type": "redis",

    "host": "redis.devel.nz",

    "port": 6379,

    "hosts": null,

    "username": "",

    "password": "",

    "database": 0,

    "optimisation_max_idle": 3000,

    "optimisation_max_active": 5000,

    "enable_cluster": false

  },

  "enable_separate_cache_store": false,

  "enable_analytics": true,

  "analytics_config": {

    "type": "mongo",

    "ignored_ips": [],

    "enable_detailed_recording": true,

    "enable_geo_ip": false,

    "geo_ip_db_path": "./GeoLite2-City.mmdb",

    "normalise_urls": {

      "enabled": true,

      "normalise_uuids": true,

      "normalise_numbers": true,

      "custom_patterns": []

    }

  },

  "health_check": {

    "enable_health_checks": false,

    "health_check_value_timeouts": 0

  },

  "optimisations_use_async_session_write": true,

  "allow_master_keys": false,

  "hash_keys": true,

  "hash_key_function": "murmur64",

  "suppress_redis_signal_reload": false,

  "suppress_default_org_store": false,

  "use_redis_log": true,

  "sentry_code": "",

  "use_sentry": false,

  "use_syslog": false,

  "use_graylog": false,

  "use_logstash": false,

  "graylog_network_addr": "",

  "logstash_network_addr": "",

  "syslog_transport": "",

  "logstash_transport": "",

  "syslog_network_addr": "",

  "enforce_org_data_age": true,

  "enforce_org_data_detail_logging": false,

  "enforce_org_quotas": true,

  "experimental_process_org_off_thread": false,

  "enable_non_transactional_rate_limiter": true,

  "enable_sentinel_rate_limiter": false,

  "Monitor": {

    "enable_trigger_monitors": false,

    "configuration": {

      "method": "",

      "target_path": "",

      "template_path": "",

      "header_map": null,

      "event_timeout": 0

    },

    "global_trigger_limit": 0,

    "monitor_user_keys": false,

    "monitor_org_keys": false

  },

  "oauth_refresh_token_expire": 0,

  "oauth_token_expire": 0,

  "oauth_redirect_uri_separator": ";",

  "slave_options": {

    "use_rpc": false,

    "connection_string": "",

    "rpc_key": "",

    "api_key": "",

    "enable_rpc_cache": false,

    "bind_to_slugs": false,

    "disable_keyspace_sync": false,

    "group_id": ""

  },

  "disable_virtual_path_blobs": false,

  "local_session_cache": {

    "disable_cached_session_state": true,

    "cached_session_timeout": 0,

    "cached_session_eviction": 0

  },

  "http_server_options": {

    "override_defaults": false,

    "read_timeout": 0,

    "write_timeout": 0,

    "use_ssl": true,

    "use_ssl_le": false,

    "enable_websockets": true,

    "certificates": [ 

      {

        "domain_name": "devel.nz",

        "cert_file": "/opt/frontend/certs/devel.nz.cert",

        "key_file": "/opt/frontend/certs/devel.nz.key"

       } 

    ],

    "server_name": "",

    "min_version": 0,

    "flush_interval": 0

  },

  "service_discovery": {

    "default_cache_timeout": 0

  },

  "close_connections": false,

  "auth_override": {

    "force_auth_provider": false,

    "auth_provider": {

      "name": "",

      "storage_engine": "",

      "meta": null

    },

    "force_session_provider": false,

    "session_provider": {

      "name": "",

      "storage_engine": "",

      "meta": null

    }

  },

  "uptime_tests": {

    "disable": true,

    "config": {

      "failure_trigger_sample_size": 1,

      "time_wait": 2,

      "checker_pool_size": 50,

      "enable_uptime_analytics": true

    }

  },

  "hostname": "",

  "enable_api_segregation": false,

  "control_api_hostname": "",

  "enable_custom_domains": true,

  "enable_jsvm": true,

  "coprocess_options": {

    "enable_coprocess": false,

    "coprocess_grpc_server": "tcp://localhost:5555"

  }, 

  "hide_generator_header": false,

  "event_handlers": {

    "events": {}

  },

  "event_trigers_defunct": {},

  "pid_file_location": "./tyk-gateway.pid",

  "allow_insecure_configs": true,

  "public_key_path": "",

  "close_idle_connections": false,

  "allow_remote_config": true,

  "enable_bundle_downloader": false,

  "max_idle_connections_per_host": 500

}

tyk_analytics.conf

{

  "listen_port": 443,

  "tyk_api_config": {

    "Host": "https://api.devel.nz",

    "Port": "443",

    "Secret": "352d20ee67be67f6340b4c0605b044b7"

  },

  "mongo_url": "mongodb://db.devel.nz:27017/tyk_analytics",

  "page_size": 10,

  "admin_secret": "12345",

  "shared_node_secret": "352d20ee67be67f6340b4c0605b044b7",

  "redis_port": 6379,

  "redis_host": "redis.devel.nz",

  "redis_password": "",

  "enable_cluster": false,

  "force_api_defaults": false,

  "notify_on_change": true,

  "license_key": "",

  "redis_database": 0,

  "redis_hosts": null,

  "hash_keys": true,

  "email_backend": {

    "enable_email_notifications": false,

    "code": "sendgrid",

    "settings": {

      "ClientKey": ""

    },

    "default_from_email": "[email protected]",

    "default_from_name": "Some Person"

  },

  "hide_listen_path": false,

  "sentry_code": "",

  "sentry_js_code": "",

  "use_sentry": false,

  "enable_master_keys": false,

  "enable_duplicate_slugs": true,

  "show_org_id": true,

  "host_config": {

    "enable_host_names": true,

    "disable_org_slug_prefix": true,

    "hostname": "api-admin.devel.nz",

    "override_hostname": "api-admin.devel.nz:443",

    "portal_domains": {},

    "portal_root_path": "/portal",

    "generate_secure_paths": false,

    "use_strict_hostmatch": false

  },

  "http_server_options": {

    "use_ssl": true,

    "certificates": [ 

      {

        "domain_name": "devel.nz",

        "cert_file": "/opt/frontend/certs/devel.nz.cert",

        "key_file": "/opt/frontend/certs/devel.nz.key"

       } 

    ],

    "min_version": 0

  },

  "ui": {

    "languages": {

      "Chinese": "cn",

      "English": "en",

      "Korean": "ko"

    },

    "hide_help": true,

    "default_lang": "en",

    "login_page": {},

    "nav": {

      "dont_show_admin_sockets": false,

      "hide_activity_by_api_section": false,

      "hide_geo": false,

      "hide_licenses_section": false,

      "hide_logs": false,

      "hide_tib_section": false

    },

    "uptime": {},

    "portal_section": null,

    "designer": {},

    "dont_show_admin_sockets": false,

    "dont_allow_license_management": false,

    "dont_allow_license_management_view": false

  },

  "home_dir": "/opt/tyk-dashboard",

  "tagging_options": {

    "tag_all_apis_by_org": false

  },

  "use_sharded_analytics": true,

  "enable_aggregate_lookups": true,

  "enable_analytics_cache": false,

  "aggregate_lookup_cutoff": "26/05/2016",

  "maintenance_mode": false,

  "allow_explicit_policy_id": true,

  "private_key_path": "",

  "node_schema_path": "",

  "oauth_redirect_uri_separator": ";",

  "statsd_connection_string": "",

  "statsd_prefix": ""

}

I have configured several things that are non-standard (part of the initial repo):

  • the urls to relevant parts of the system

  • ports for accessing these other parts of the system

  • certificates for ssl support

  • use_ssl value set to true

All other items in the config are as per your demo config.

All parts of the system are being run in docker containers, using Gitlab CI to deploy to the target hosts.

My main goal is to create a usable Proof of Concept that is production-like in its installation on distributed hosts, using ssl to secure traffic, deployed from source control (GitLab), and integrated with some existing internal API’s and identity platform. While a local installation allows me to look at how it works, it does not represent how it will function within our production system, where some questions need answers before the concept is considered proven.

Please help.

Hi Codex, can you confirm if your VMs are all on a flat network or have been configured on the same VLAN/Virtual Switch?

Hi,

I’ve been able to use tyk.conf and tyk_analytics.conf very similar to the ones you’ve given to create a working dashboard/gateway pair.

The error you’ve given “Attempted administrative access with invalid or missing key” comes from the code in the link below and indicates that the shared secret doesn’t match up.

The shared secret in the config files you’ve given does match (tyk_api_config.Secret in tyk_analytics.conf and secret in tyk.conf) so I can’t see anything wrong with your setup. Are you sure what you’ve pasted in here is what you’re running?

Which auth method have you used? can you share your api def? Also, can you please try with a simple case of auth token?

Hi @Yaara, there is no API definition to share yet because this is a Tyk installation issue.
This is a problem with the dashboard communicating with the gateway, which is on a different server.

The problem is only evident when trying to create a Key to secure an API in the dashboard UI.
The definition of that API is verbatim what you can see in the YouTube video I posted in this case, which is what I was following when I encountered the issue.

Hey @ps258_tyk,

To answer your question of whether I am sure the configs currently deployed match the config files I have attached in this case, please see attached screen shot of dashboard and gateway vm’s with config files being shown:

Hopefully you are able to see something I cannot…

Regarding your statement that you could get it working, I have also had this working on a single host, the issue is only when these components are deployed to different VM’s. Were you able to replicate the distributed deployment and have it work?

Hey @Gregor,

Yes the VM’s in this POC are hosted in the same VMWare cluster, on the same VLAN.
The only thing I will say is that the dns CNAME being used by the Gateway to reference the dashboard is not the same as the host name.
The host is: personal.devel.nz
The CNAME in the config is: api-admin.devel.nz

Wouldn’t expect this to make any difference but am happy to change if you believe it could make a difference.

Something else I though may be important is that I have tried to authenticate with the Gateway API using postman with the same outcome.

A colleague and I have looked into the the code of the gateway to figure out what I have done wrong.
We uncovered I was mistakenly setting the TYK_GW_SECRET environment variable in the compose file for the gateway which was a different value to that included in the config files.

General feedback:
We were a little confused by how the code was picking this value over the config file value.
There was also little in the way of guidance regarding if this compose variable was set, then the config value in the tyk_analytics.conf for the gateway secret needed to be the same as that compose value.

Either way, the solution is now working.
@Gregor , @Yaara , and @ps258_tyk, I appreciate your effort and time in helping to figure this out.

Please close this issue as solved.

thanks @codex we’ll look into your feedback and see how we can improve the documentation of these aspects.