Analytics showing APIs from different organisation


#1

Hi ,

I created a new organisation and logged in the dashboard with a user for this newly created organisation. I was surprised to find that all the analytics from the old organisation.

Is that expected? May be something is wrong in my setup? I was expecting that every organisation to completely independent.

No API defined

But heaps of data on the dashboard


#2

@Martin @matiasb

Is there something that I have configured incorrectly?


#3

Can you check if the APIs you have set up are tied to an organisation? It may be the case that you are using an un-bound user.

Can you share an API Definition?


#4

I don’t think I am using an unbound user. Here is an api definition

{ "id": "580aeaf356c02c0ae0000005", "name": "Login-RIB", "slug": "init-auth", "api_id": "15204a4eb380471471d61ed0eea696bb", "org_id": "580a98a056c02c0ae0000003", "use_keyless": false, "use_oauth2": false, "use_openid": false, "openid_options": { "providers": [], "segregate_by_client": false }, "oauth_meta": { "allowed_access_types": [], "allowed_authorize_types": [], "auth_login_redirect": "" }, "auth": { "use_param": false, "param_name": "", "use_cookie": false, "cookie_name": "", "auth_header_name": "x-nab-key" }, "use_basic_auth": false, "enable_jwt": false, "jwt_signing_method": "", "jwt_source": "", "jwt_identity_base_field": "", "jwt_client_base_field": "", "jwt_policy_field_name": "", "notifications": { "shared_secret": "", "oauth_on_keychange_url": "" }, "enable_signature_checking": false, "hmac_allowed_clock_skew": -1, "definition": { "location": "url-param", "key": "v" }, "version_data": { "not_versioned": false, "versions": { "1": { "name": "1", "expires": "", "paths": { "ignored": [], "white_list": [], "black_list": [] }, "use_extended_paths": true, "extended_paths": { "ignored": [], "white_list": [], "black_list": [], "cache": [], "transform": [], "transform_response": [], "transform_headers": [], "transform_response_headers": [], "hard_timeouts": [], "circuit_breakers": [], "url_rewrites": [], "virtual": [], "size_limits": [], "method_transforms": [] }, "global_headers": {}, "global_headers_remove": [], "global_size_limit": 0, "override_target": "" }, "3": { "name": "3", "expires": "", "paths": { "ignored": [], "white_list": [], "black_list": [] }, "use_extended_paths": true, "extended_paths": { "ignored": [], "white_list": [], "black_list": [], "cache": [], "transform": [], "transform_response": [], "transform_headers": [], "transform_response_headers": [], "hard_timeouts": [], "circuit_breakers": [], "url_rewrites": [], "virtual": [], "size_limits": [], "method_transforms": [] }, "global_headers": {}, "global_headers_remove": [], "global_size_limit": 0, "override_target": "" }, "4": { "name": "4", "expires": "", "paths": { "ignored": [], "white_list": [], "black_list": [] }, "use_extended_paths": true, "extended_paths": { "ignored": [], "white_list": [], "black_list": [], "cache": [], "transform": [], "transform_response": [], "transform_headers": [], "transform_response_headers": [], "hard_timeouts": [], "circuit_breakers": [], "url_rewrites": [], "virtual": [], "size_limits": [], "method_transforms": [] }, "global_headers": {}, "global_headers_remove": [], "global_size_limit": 0, "override_target": "" } } }, "uptime_tests": { "check_list": [], "config": { "expire_utime_after": 0, "service_discovery": { "use_discovery_service": false, "query_endpoint": "", "use_nested_query": false, "parent_data_path": "", "data_path": "", "port_data_path": "", "target_path": "", "use_target_list": false, "cache_timeout": 60, "endpoint_returns_list": false }, "recheck_wait": 0 } }, "proxy": { "preserve_host_header": false, "listen_path": "/init/auth", "target_url": "http://192.168.0.22/", "strip_listen_path": false, "enable_load_balancing": false, "target_list": [], "check_host_against_uptime_tests": false, "service_discovery": { "use_discovery_service": false, "query_endpoint": "", "use_nested_query": false, "parent_data_path": "", "data_path": "hostname", "port_data_path": "port", "target_path": "/api-slug", "use_target_list": false, "cache_timeout": 60, "endpoint_returns_list": false } }, "custom_middleware": { "pre": [], "post": [], "response": [] }, "cache_options": { "cache_timeout": 60, "enable_cache": false, "cache_all_safe_requests": false, "cache_response_codes": [], "enable_upstream_cache_control": false }, "session_lifetime": 0, "active": true, "auth_provider": { "name": "", "storage_engine": "", "meta": {} }, "session_provider": { "name": "", "storage_engine": "", "meta": null }, "event_handlers": { "events": {} }, "enable_batch_request_support": false, "enable_ip_whitelisting": false, "allowed_ips": [], "dont_set_quota_on_create": false, "expire_analytics_after": 0, "response_processors": [], "CORS": { "enable": false, "allowed_origins": [], "allowed_methods": [], "allowed_headers": [], "exposed_headers": [], "allow_credentials": false, "max_age": 24, "options_passthrough": false, "debug": false }, "domain": "my-tyk-instance.com", "do_not_track": false, "tags": [], "enable_context_vars": false }


#5

Are the usernames in the accounts different or the same?


#6

Account 1 User name is [email protected]
Account 2 user name is [email protected]


#7

And how we’re these generated?


#8

setup.sh script supplied with docker installation


#9

Have you tried restarting the dashboard application and logging out and in again?


#10

Yes martin, many times.

I was wondering if I can check something in mongodb ? Also this is the setup.sh script that I used.

#! /bin/bash

# This script will set up a full tyk environment on your machine
# and also create a demo user for you with one command

# USAGE
# -----
#
# $> ./setup.sh {IP ADDRESS OF DOCKER VM}

# OSX users will need to specify a virtual IP, linux users can use 127.0.0.1

# Proxy
TYK_DASHBOARD_DOMAIN="my-tyk-instance.com"

# Tyk dashboard settings
TYK_DASHBOARD_USERNAME="[email protected]"
TYK_DASHBOARD_PASSWORD="test123"

# Tyk portal settings
TYK_PORTAL_DOMAIN="my-tyk-portal.com"
TYK_PORTAL_PATH="/portal/"

DOCKER_IP="192.168.0.25"

if [ -n "$DOCKER_HOST" ]
then
		echo "Detected a Docker VM..."
		REMTCP=${DOCKER_HOST#tcp://}
		DOCKER_IP=${REMTCP%:*}
fi

if [ -n "$1" ]
then
		DOCKER_IP=$1
		echo "Docker host address explicitly set."
		echo "Using $DOCKER_IP as Tyk host address."
fi

if [ -n "$2" ]
then
		TYK_PORTAL_DOMAIN=$2
		echo "Docker portal domain address explicitly set."
		echo "Using $TYK_PORTAL_DOMAIN as Tyk host address."
fi

if [ -z "$1" ]
then
        echo "Using $DOCKER_IP as Tyk host address."
        echo "If this is wrong, please specify the instance IP address (e.g. ./setup.sh 192.168.1.1)"
fi

echo "Creating Organisation 1"
ORG_DATA=$(curl --silent --header "admin-auth: 12345" --header "Content-Type:application/json" --data '{"owner_name": "Org1","owner_slug": "org1", "cname_enabled":true}' http://$DOCKER_IP:3000/admin/organisations 2>&1)
ORG_ID=$(echo $ORG_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["Meta"]')
echo "ORG ID: $ORG_ID"

echo "Creating Organisation 2"
ORG_DATA=$(curl --silent --header "admin-auth: 12345" --header "Content-Type:application/json" --data '{"owner_name": "Org2","owner_slug": "org1", "cname_enabled":true}' http://$DOCKER_IP:3000/admin/organisations 2>&1)
ORG_ID=$(echo $ORG_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["Meta"]')
echo "ORG ID: $ORG_ID"

echo "Creating Organisation 3"
ORG_DATA=$(curl --silent --header "admin-auth: 12345" --header "Content-Type:application/json" --data '{"owner_name": "Org3","owner_slug": "org1", "cname_enabled":true}' http://$DOCKER_IP:3000/admin/organisations 2>&1)
ORG_ID=$(echo $ORG_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["Meta"]')
echo "ORG ID: $ORG_ID"


echo "Adding new user"
USER_DATA=$(curl --silent --header "admin-auth: 12345" --header "Content-Type:application/json" --data '{"first_name": "John","last_name": "Smith","email_address": "'$TYK_DASHBOARD_USERNAME'","active": true,"org_id": "'$ORG_ID'"}' http://$DOCKER_IP:3000/admin/users 2>&1)
USER_AUTH=$(echo $USER_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["Message"]')
USER_LIST=$(curl --silent --header "authorization: $USER_AUTH" http://$DOCKER_IP:3000/api/users 2>&1)
USER_ID=$(echo $USER_LIST | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["users"][0]["id"]')
echo "USER AUTH: $USER_AUTH"
echo "USER ID: $USER_ID"

echo "Setting password"
OK=$(curl --silent --header "authorization: $USER_AUTH" --header "Content-Type:application/json" http://$DOCKER_IP:3000/api/users/$USER_ID/actions/reset --data '{"new_password":"'$TYK_DASHBOARD_PASSWORD'"}')

echo "Setting up the portal domain"
OK=$(curl --silent -d "domain="$TYK_PORTAL_DOMAIN"" -H "admin-auth:12345" http://$DOCKER_IP:3000/admin/organisations/$ORG_ID/generate-portals)

echo "Setting up the Portal catalogue"
CATALOGUE_DATA=$(curl --silent --header "Authorization: $USER_AUTH" --header "Content-Type:application/json" --data '{"org_id": "'$ORG_ID'"}' http://$DOCKER_IP:3000/api/portal/catalogue 2>&1)
CATALOGUE_ID=$(echo $CATALOGUE_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["Message"]')
OK=$(curl --silent --header "Authorization: $USER_AUTH" http://$DOCKER_IP:3000/api/portal/catalogue 2>&1)

echo "Setting target URL for Portal APIs"
API_LIST=$(curl --silent --header "Authorization: $USER_AUTH" http://$DOCKER_IP:3000/api/apis 2>&1)
API_PORTAL_DATA=$(echo $API_LIST | python -c 'import json,sys;obj=json.load(sys.stdin);print json.dumps(obj["apis"][2])')
API_PORTAL_DATA=$(echo $API_PORTAL_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);obj["sort_by"]=9;target_url=obj["api_definition"]["proxy"]["target_url"];obj["api_definition"]["proxy"]["target_url"]=target_url.replace("localhost", "'"$TYK_DASHBOARD_DOMAIN"'");print json.dumps(obj)')
API_PORTAL_DATA=$(echo $API_PORTAL_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);obj["api_definition"]["proxy"]["listen_path"]="'$TYK_PORTAL_PATH'";print json.dumps(obj)')
API_PORTAL_ID=$(echo $API_PORTAL_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["api_definition"]["id"]')
API_PORTAL_API_DATA=$(echo $API_LIST | python -c 'import json,sys;obj=json.load(sys.stdin);print json.dumps(obj["apis"][1])')
API_PORTAL_API_DATA=$(echo $API_PORTAL_API_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);target_url=obj["api_definition"]["proxy"]["target_url"];obj["api_definition"]["proxy"]["target_url"]=target_url.replace("localhost", "'"$TYK_DASHBOARD_DOMAIN"'");print json.dumps(obj)')
API_PORTAL_API_DATA=$(echo $API_PORTAL_API_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);obj["api_definition"]["proxy"]["listen_path"]="/portal-api/";print json.dumps(obj)')
API_PORTAL_API_ID=$(echo $API_PORTAL_API_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["api_definition"]["id"]')
API_PORTAL_ASSETS_DATA=$(echo $API_LIST | python -c 'import json,sys;obj=json.load(sys.stdin);print json.dumps(obj["apis"][0])')
API_PORTAL_ASSETS_DATA=$(echo $API_PORTAL_ASSETS_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);target_url=obj["api_definition"]["proxy"]["target_url"];obj["api_definition"]["proxy"]["target_url"]=target_url.replace("localhost", "'"$TYK_DASHBOARD_DOMAIN"'");print json.dumps(obj)')
API_PORTAL_ASSETS_ID=$(echo $API_PORTAL_ASSETS_DATA | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["api_definition"]["id"]')
OK=$(curl --silent --header "Authorization: $USER_AUTH" --header "Content-Type:application/json" --data "$API_PORTAL_DATA" -X PUT http://$DOCKER_IP:3000/api/apis/$API_PORTAL_ID 2>&1)
OK=$(curl --silent --header "Authorization: $USER_AUTH" --header "Content-Type:application/json" --data "$API_PORTAL_API_DATA" -X PUT http://$DOCKER_IP:3000/api/apis/$API_PORTAL_API_ID 2>&1)
OK=$(curl --silent --header "Authorization: $USER_AUTH" --header "Content-Type:application/json" --data "$API_PORTAL_ASSETS_DATA" -X PUT http://$DOCKER_IP:3000/api/apis/$API_PORTAL_ASSETS_ID 2>&1)

echo "Creating the Portal Home page"
OK=$(curl --silent --header "Authorization: $USER_AUTH" --header "Content-Type:application/json" --data '{"is_homepage": true, "template_name":"", "title":"Tyk Developer Portal", "slug":"home", "fields": {"JumboCTATitle": "Tyk Developer Portal", "SubHeading": "Sub Header", "JumboCTALink": "#cta", "JumboCTALinkTitle": "Your awesome APIs, hosted with Tyk!", "PanelOneContent": "Panel 1 content.", "PanelOneLink": "#panel1", "PanelOneLinkTitle": "Panel 1 Button", "PanelOneTitle": "Panel 1 Title", "PanelThereeContent": "", "PanelThreeContent": "Panel 3 content.", "PanelThreeLink": "#panel3", "PanelThreeLinkTitle": "Panel 3 Button", "PanelThreeTitle": "Panel 3 Title", "PanelTwoContent": "Panel 2 content.", "PanelTwoLink": "#panel2", "PanelTwoLinkTitle": "Panel 2 Button", "PanelTwoTitle": "Panel 2 Title"}}' http://$DOCKER_IP:3000/api/portal/pages 2>&1)

echo ""

echo "DONE"
echo "===="
echo "Login at http://$DOCKER_IP:3000/"
echo "Username: $TYK_DASHBOARD_USERNAME"
echo "Password: $TYK_DASHBOARD_PASSWORD"
echo "Portal: http://$TYK_PORTAL_DOMAIN$TYK_PORTAL_PATH"
echo ""

I changed the user manually when running it.


#11

Ok so you could have lead with the fact that you had customised the bootstrap script :wink:

These modifications create three organisations and then attach the user to the last one from the looks of it. So essentially you have every new user added to the last org, but each time you run it you are generating three orgs (two which aren’t used).

Now this is a docker install I’m guessing? So this script is very specific to the docker compose setup and really should only be used once.

If you want to add more orgs and users you should use the REST API (it’s what this script does, but it also does a whole lot more and is designed to stand up a very specific docker demo / QuickStart). I wonder if fiddling with it caused the problem - perhaps assigning a duplicate org ID to your second user when you modified it messed things up.


#12

Hi @Martin,

Sorry to resurrect this thread however we’re having the exact same issue.

We have 5 organiastions, and 5 users.

These have been setup using the Admin API (rather than the bootstrap script). We can see in the mongo database that each of the users has the correct organisationId.

We’ve setup an API in Organisation 1 and it works fine, however when you log into each Organistaion you can see the summary statistics for every API in the dashboard. The one thing you don’t see is any of the details, or if you go into Log Browser any of the requests. However the graphs are showing the stats for the APIs for other orgs.

Again in Mongo we can see that each API we’ve created is linked correctly to the org it’s owned by.

Can you think of what might be causing this issue? Let me know if you want a dump of any configuration.

Thanks,
Richard


#13

Can you share your pump config?


#14

Hi @Martin, I work with @catdevnull, as per your request the /opt/tyk-pump/pump.conf is

{
    "analytics_storage_type": "redis",
    "analytics_storage_config": {
        "type": "redis",
        "host": "*redacted*",
        "port": 6379,
        "hosts": null,
        "username": "",
        "password": "",
        "database": 0,
        "optimisation_max_idle": 100,
        "optimisation_max_active": 0,
        "enable_cluster": false
    },
    "purge_delay": 10,
    "pumps": {
        "mongo": {
            "name": "mongo",
            "meta": {
                "collection_name": "tyk_analytics",
                "mongo_url": "*redacted*"
            }
        },
        "mongo-pump-aggregate": {
            "name": "mongo-pump-aggregate",
            "meta": {
                "mongo_url": "*redacted*",
                "use_mixed_collection": true
            }
        }
    },
    "uptime_pump_config": {
        "collection_name": "tyk_uptime_analytics",
        "mongo_url": "*redacted*"
    },
    "dont_purge_uptime_data": false
}

The redacted Mongo connection string are the same for all three settings.


#15

Ok, those look in order.

The thing is - the gateway associates the analytics records with the org ID of the API that is being proxied, so the analytics records are organised by what is set in the API definition initially.

You can check if it’s working by checking the analytics records themselves in the tyk_analytics collection or one of the aggregate collections (there should be more than one) and see if they are tagged to the right org.

Can you list out the collections in your DB?

One thing to try is to set use_mixed_collection to false.


#16

Hi @Martin we’ve changed use_mixed_collection to false however now we’re not seeing any data for any time since the setting was changed on the graphs. Basically all the data up to the setting change is still displaying on the graphs but nothing for today for example/

I haven’t got the collection list to hand (@reidca should be able to assist) however I believe we have a tyk_analytics_aggregate and then also org specific collections for tyk_analytics_aggegate.


#17

Hi @Martin,

We’ve left the setting use_mixed_collection on false over the weekend and as you can see from the picture below this has definitely broken the aggregate stats. Stats only appear for before we made this change:

Can you advise?

Thanks,
Richard


#18

Are you logged in as a super user (no org, or as an org user?)

Your pump.conf pumps section should look like this:

"pumps": {
        "mongo": {
            "name": "mongo",
            "meta": {
                "collection_name": "tyk_analytics",
                "mongo_url": "mongo-hosts/tyk_analytics"
            }
        },
        "mongo-pump-selective": {
            "name": "mongo-pump-selective",
            "meta": {
                "mongo_url": "mongo-hosts/tyk_analytics"
            }
        },
        "mongo-pump-aggregate": {
            "name": "mongo-pump-aggregate",
            "meta": {
                "mongo_url": "mongo-hosts/tyk_analytics",
                "use_mixed_collection": true
            }
        }
    },
    "uptime_pump_config": {
        "collection_name": "tyk_uptime_analytics",
        "mongo_url": "mongo-hosts/tyk_analytics"
    },
    "dont_purge_uptime_data": false
}

And your tyk_analytics.conf will need:

    "use_sharded_analytics": true,
    "enable_aggregate_lookups": true,
    "aggregate_lookup_cutoff": "01/01/2016",

If these settings do not work for you then there is something else wrong with your configuration. I based the above on a reference configuration which runs thousands of organisations so we know it works :slight_smile:


#19

Hi @Martin

That screenshot was from an org user, rather than the super user. There’s definitely been traffic to APIs attached to that org so stats should be present.

I’ll get @reidca checking out our configuration and get back to you.

Thanks,
Richard


#20

What does the “mongo-hosts/tyk_analytics” refer to in the “mongo_url” key?

Is there a way of declaring the mongo url in a single location and referencing it here? Is it possible that you could you post the full config so I can see the way this is used? Thanks.