{
    "schema_version": "1.0",
    "name": "AI Search Visibility Using SEO Datasets",
    "description": "Snapshot data behind the AI Symantix case study on which crawlers fetch published SEO datasets, how they find the files, what Google Search Console reports for dataset pages, and how AI Symantix search visibility moved over the same period.",
    "url": "https://www.aisymantix.com/article/ai-search-visibility-using-seo-datasets/",
    "license": "https://creativecommons.org/licenses/by/4.0/",
    "generated_at": "2026-10-11T00:00:00+00:00",
    "source": "Digital Karma Data Warehouse (Google Search Console imports and classified server logs)",
    "windows": {
        "server_logs": {
            "start": "2026-09-06",
            "end": "2026-10-10",
            "days": 35
        },
        "gsc_aisymantix": {
            "start": "2026-06-16",
            "end": "2026-10-08"
        },
        "portfolio_sites": 190,
        "before_period": {
            "start": "2026-09-06",
            "end": "2026-09-22",
            "days": 17
        },
        "after_period": {
            "start": "2026-09-23",
            "end": "2026-10-10",
            "days": 18
        },
        "change_event": "Digital Karma Federation v8.2: DataCatalog and Dataset JSON-LD on every rendered page, completed 2026-09-23"
    },
    "definitions": {
        "dataset_layer": "Successful GET requests to /ai/catalog.json, /ai/manifest.json, llms.txt and llm.txt files, /ai/health.json, /ai/federation.json, /ai/karma.json, /ai/rendered-schema.json, any other /ai/*.json, and public JSON files under /data/, /content/ and /datasets/, plus /feeds/.",
        "content_page_requests": "Content-route requests from the warehouse portfolio_request_daily cache for the same 35 days.",
        "unidentified_bot": "Requests classified as a bot that match no active fingerprint in the warehouse.",
        "share": "Dataset-layer fetches divided by dataset-layer fetches plus page fetches for that crawler.",
        "weighted_position": "Average position weighted by impressions."
    },
    "dataset_layer_by_audience": [
        {
            "audience": "Bots that could not be named",
            "requests": 11969,
            "catalog_sites": 176
        },
        {
            "audience": "People",
            "requests": 3390,
            "catalog_sites": 107,
            "llm_guidance_requests": 1853,
            "llm_guidance_distinct_ips": 1469
        },
        {
            "audience": "SEO tools, monitors and other named bots",
            "requests": 3252,
            "catalog_sites": 113
        },
        {
            "audience": "Search engine crawlers",
            "requests": 1463,
            "catalog_sites": 128
        },
        {
            "audience": "AI training crawlers",
            "requests": 1384,
            "catalog_sites": 143
        },
        {
            "audience": "AI search crawlers",
            "requests": 244,
            "catalog_sites": 57
        },
        {
            "audience": "AI tools fetching for a person",
            "requests": 11,
            "catalog_sites": 1
        }
    ],
    "dataset_layer_total_requests": 21713,
    "content_page_requests": 7090304,
    "dataset_layer_share_of_crawl_pct": 0.31,
    "crawler_dataset_share": [
        {
            "crawler": "OAI-SearchBot",
            "owner": "OpenAI",
            "job": "ai_search",
            "dataset_layer_fetches": 222,
            "page_fetches": 5414,
            "share_pct": 3.94,
            "sites_with_dataset_fetch": 62
        },
        {
            "crawler": "Applebot",
            "owner": "Apple",
            "job": "search",
            "dataset_layer_fetches": 141,
            "page_fetches": 9020,
            "share_pct": 1.54,
            "sites_with_dataset_fetch": 52
        },
        {
            "crawler": "Bingbot",
            "owner": "Microsoft",
            "job": "search",
            "dataset_layer_fetches": 294,
            "page_fetches": 20051,
            "share_pct": 1.45,
            "sites_with_dataset_fetch": 67
        },
        {
            "crawler": "CCBot",
            "owner": "Common Crawl",
            "job": "training",
            "dataset_layer_fetches": 25,
            "page_fetches": 2098,
            "share_pct": 1.18,
            "sites_with_dataset_fetch": 10
        },
        {
            "crawler": "PetalBot",
            "owner": "Huawei",
            "job": "search",
            "dataset_layer_fetches": 439,
            "page_fetches": 43125,
            "share_pct": 1.01,
            "sites_with_dataset_fetch": 34
        },
        {
            "crawler": "Googlebot",
            "owner": "Google",
            "job": "search",
            "dataset_layer_fetches": 223,
            "page_fetches": 59219,
            "share_pct": 0.38,
            "sites_with_dataset_fetch": 78
        },
        {
            "crawler": "PerplexityBot",
            "owner": "Perplexity",
            "job": "ai_search",
            "dataset_layer_fetches": 20,
            "page_fetches": 7564,
            "share_pct": 0.26,
            "sites_with_dataset_fetch": 10
        },
        {
            "crawler": "GoogleOther",
            "owner": "Google",
            "job": "search",
            "dataset_layer_fetches": 172,
            "page_fetches": 85170,
            "share_pct": 0.2,
            "sites_with_dataset_fetch": 83
        },
        {
            "crawler": "ChatGPT-User",
            "owner": "OpenAI",
            "job": "user_retrieval",
            "dataset_layer_fetches": 10,
            "page_fetches": 8495,
            "share_pct": 0.12,
            "sites_with_dataset_fetch": 4
        },
        {
            "crawler": "GPTBot",
            "owner": "OpenAI",
            "job": "training",
            "dataset_layer_fetches": 942,
            "page_fetches": 2188904,
            "share_pct": 0.04,
            "sites_with_dataset_fetch": 155
        },
        {
            "crawler": "meta-externalagent",
            "owner": "Meta",
            "job": "training",
            "dataset_layer_fetches": 211,
            "page_fetches": 809146,
            "share_pct": 0.03,
            "sites_with_dataset_fetch": 41
        },
        {
            "crawler": "ClaudeBot",
            "owner": "Anthropic",
            "job": "training",
            "dataset_layer_fetches": 63,
            "page_fetches": 292151,
            "share_pct": 0.02,
            "sites_with_dataset_fetch": 37
        },
        {
            "crawler": "Amazonbot",
            "owner": "Amazon",
            "job": "training",
            "dataset_layer_fetches": 105,
            "page_fetches": 579240,
            "share_pct": 0.02,
            "sites_with_dataset_fetch": 22
        }
    ],
    "catalog_fetch_before_after": {
        "note": "AI crawler GET 2xx requests for /ai/catalog.json, BellyUp city sites excluded because of facet crawl traps in October.",
        "rows": [
            {
                "crawler": "GPTBot",
                "sites_before": 54,
                "sites_after": 87,
                "fetches_before": 105,
                "fetches_after": 127
            },
            {
                "crawler": "OAI-SearchBot",
                "sites_before": 11,
                "sites_after": 43,
                "fetches_before": 14,
                "fetches_after": 44
            },
            {
                "crawler": "meta-externalagent",
                "sites_before": 3,
                "sites_after": 19,
                "fetches_before": 10,
                "fetches_after": 28
            },
            {
                "crawler": "ClaudeBot",
                "sites_before": 5,
                "sites_after": 6,
                "fetches_before": 5,
                "fetches_after": 7
            }
        ],
        "gptbot_daily_catalog_sites_portfolio_wide": [
            {
                "date": "2026-09-18",
                "sites": 11
            },
            {
                "date": "2026-09-22",
                "sites": 7
            },
            {
                "date": "2026-09-23",
                "sites": 7
            },
            {
                "date": "2026-09-24",
                "sites": 28
            },
            {
                "date": "2026-09-25",
                "sites": 30
            },
            {
                "date": "2026-09-28",
                "sites": 27
            },
            {
                "date": "2026-10-06",
                "sites": 20
            },
            {
                "date": "2026-10-08",
                "sites": 17
            }
        ],
        "sites_with_any_ai_catalog_fetch_in_window": {
            "all": 157,
            "excluding_bellyup": 116
        }
    },
    "aisymantix_referer_receipts": [
        {
            "file": "/data/lab/exposure-velocity.json",
            "crawler": "Googlebot",
            "fetches": 18,
            "referer": "/ai-symantix-lab/"
        },
        {
            "file": "/data/lab/exposure-velocity.json",
            "crawler": "GoogleOther",
            "fetches": 2,
            "referer": "/ai-symantix-lab/"
        },
        {
            "file": "/data/lab/exposure-velocity.json",
            "crawler": "Applebot",
            "fetches": 1,
            "referer": "/ai-symantix-lab/"
        },
        {
            "file": "/data/research/search-conversion-baseline.json",
            "crawler": "PetalBot",
            "fetches": 3,
            "referer": "/search-visibility-to-inquiries/"
        },
        {
            "file": "/data/research/ai-bot-visibility-cohort.json",
            "crawler": "GPTBot",
            "fetches": 1,
            "referer": "/ai-bot-visibility-cohort/"
        },
        {
            "file": "/data/research/ai-bot-visibility-cohort.json",
            "crawler": "PetalBot",
            "fetches": 2,
            "referer": "/ai-bot-visibility-cohort/ and /datasets/"
        },
        {
            "file": "/data/research/ai-bot-visibility.json",
            "crawler": "GPTBot",
            "fetches": 1,
            "referer": "/ai-bot-visibility/"
        },
        {
            "file": "/data/research/ai-bot-visibility.json",
            "crawler": "PetalBot",
            "fetches": 1,
            "referer": "/ai-bot-visibility/"
        },
        {
            "file": "/ai/ctr-by-position.json",
            "crawler": "PetalBot",
            "fetches": 2,
            "referer": "/datasets/"
        },
        {
            "file": "/data/visibility-index/index.json",
            "crawler": "GPTBot",
            "fetches": 1,
            "referer": "server preview hostname copy of /visibility-index/"
        },
        {
            "file": "/data/visibility-index/latest.json",
            "crawler": "GPTBot",
            "fetches": 1,
            "referer": "server preview hostname copy of /visibility-index/"
        },
        {
            "file": "/data/research/search-conversion-baseline.json",
            "crawler": "GPTBot",
            "fetches": 1,
            "referer": "https://www.signalarchitectgroup.com",
            "date": "2026-09-12"
        }
    ],
    "aisymantix_dataset_pages_gsc": [
        {
            "page": "/ai-symantix-lab/",
            "first_shown": "2026-08-27",
            "impressions": 66,
            "clicks": 0,
            "weighted_position": 4.7
        },
        {
            "page": "/datasets/ctr-by-position/",
            "first_shown": "2026-09-14",
            "impressions": 54,
            "clicks": 0,
            "weighted_position": 58.4
        },
        {
            "page": "/datasets/",
            "first_shown": "2026-08-10",
            "impressions": 44,
            "clicks": 0,
            "weighted_position": 38.2
        },
        {
            "page": "/ai-bot-visibility/",
            "first_shown": "2026-09-12",
            "impressions": 28,
            "clicks": 0,
            "weighted_position": 52.1
        },
        {
            "page": "/visibility-index/",
            "first_shown": "2026-09-08",
            "impressions": 23,
            "clicks": 0,
            "weighted_position": 19
        },
        {
            "page": "/search-visibility-to-inquiries/",
            "first_shown": "2026-09-12",
            "impressions": 7,
            "clicks": 0,
            "weighted_position": 13.9
        },
        {
            "page": "/ai-bot-visibility-cohort/",
            "first_shown": "2026-09-12",
            "impressions": 4,
            "clicks": 0,
            "weighted_position": 27.5
        }
    ],
    "aisymantix_dataset_page_queries": [
        {
            "page": "/datasets/ctr-by-position/",
            "query": "top queries,clicks,impressions,ctr,position",
            "impressions": 27,
            "clicks": 0,
            "weighted_position": 52.3
        },
        {
            "page": "/ai-bot-visibility/",
            "query": "searchmetrics bot",
            "impressions": 22,
            "clicks": 0,
            "weighted_position": 58.6
        },
        {
            "page": "/datasets/ctr-by-position/",
            "query": "ctr by position",
            "impressions": 14,
            "clicks": 0,
            "weighted_position": 66
        },
        {
            "page": "/datasets/ctr-by-position/",
            "query": "organic ctr by position",
            "impressions": 4,
            "clicks": 0,
            "weighted_position": 94.8
        },
        {
            "page": "/visibility-index/",
            "query": "visibility index",
            "impressions": 3,
            "clicks": 0,
            "weighted_position": 31.7
        }
    ],
    "portfolio_seo_dataset_queries": {
        "page": "https://www.datasetsmaker.com/datasets/seo-keyword-ranking-dataset/",
        "since": "2026-07-13",
        "impressions": 797,
        "clicks": 1,
        "weighted_position": 68.5,
        "queries": [
            {
                "query": "keyword dataset",
                "impressions": 187,
                "weighted_position": 56.4
            },
            {
                "query": "seo datasets",
                "impressions": 171,
                "weighted_position": 62.5
            },
            {
                "query": "seo dataset",
                "impressions": 130,
                "weighted_position": 74.5
            },
            {
                "query": "niche keyword dataset",
                "impressions": 126,
                "weighted_position": 91.5
            },
            {
                "query": "seo dataset for machine learning training",
                "impressions": 25,
                "weighted_position": 89
            }
        ]
    },
    "aisymantix_weekly_gsc": [
        {
            "week_start": "2026-06-16",
            "impressions": 236,
            "clicks": 0,
            "weighted_position": 88.2,
            "queries": 25
        },
        {
            "week_start": "2026-06-22",
            "impressions": 336,
            "clicks": 2,
            "weighted_position": 89.5,
            "queries": 23
        },
        {
            "week_start": "2026-06-29",
            "impressions": 686,
            "clicks": 0,
            "weighted_position": 84.6,
            "queries": 38
        },
        {
            "week_start": "2026-07-06",
            "impressions": 820,
            "clicks": 1,
            "weighted_position": 85.3,
            "queries": 30
        },
        {
            "week_start": "2026-07-13",
            "impressions": 1142,
            "clicks": 0,
            "weighted_position": 81.9,
            "queries": 71
        },
        {
            "week_start": "2026-07-20",
            "impressions": 958,
            "clicks": 0,
            "weighted_position": 76.8,
            "queries": 70
        },
        {
            "week_start": "2026-07-27",
            "impressions": 1233,
            "clicks": 1,
            "weighted_position": 62.9,
            "queries": 86
        },
        {
            "week_start": "2026-08-03",
            "impressions": 1488,
            "clicks": 1,
            "weighted_position": 60.2,
            "queries": 111
        },
        {
            "week_start": "2026-08-10",
            "impressions": 1175,
            "clicks": 1,
            "weighted_position": 62.8,
            "queries": 88
        },
        {
            "week_start": "2026-08-17",
            "impressions": 1273,
            "clicks": 0,
            "weighted_position": 57.5,
            "queries": 63
        },
        {
            "week_start": "2026-08-24",
            "impressions": 1493,
            "clicks": 2,
            "weighted_position": 56.9,
            "queries": 93
        },
        {
            "week_start": "2026-08-31",
            "impressions": 1498,
            "clicks": 1,
            "weighted_position": 56.3,
            "queries": 101
        },
        {
            "week_start": "2026-09-07",
            "impressions": 1600,
            "clicks": 0,
            "weighted_position": 41.1,
            "queries": 96
        },
        {
            "week_start": "2026-09-14",
            "impressions": 1568,
            "clicks": 2,
            "weighted_position": 34.5,
            "queries": 90
        },
        {
            "week_start": "2026-09-21",
            "impressions": 844,
            "clicks": 2,
            "weighted_position": 37.7,
            "queries": 81
        },
        {
            "week_start": "2026-09-28",
            "impressions": 976,
            "clicks": 2,
            "weighted_position": 32.7,
            "queries": 92
        },
        {
            "week_start": "2026-10-05",
            "impressions": 904,
            "clicks": 1,
            "weighted_position": 30.4,
            "queries": 103,
            "partial": true,
            "days": 4
        }
    ],
    "aisymantix_ai_search_visibility_queries_weekly": [
        {
            "week_start": "2026-06-16",
            "impressions": 49,
            "clicks": 0,
            "weighted_position": 89.9,
            "queries": 4
        },
        {
            "week_start": "2026-07-13",
            "impressions": 386,
            "clicks": 0,
            "weighted_position": 83,
            "queries": 13
        },
        {
            "week_start": "2026-08-03",
            "impressions": 382,
            "clicks": 0,
            "weighted_position": 54.3,
            "queries": 17
        },
        {
            "week_start": "2026-09-07",
            "impressions": 82,
            "clicks": 0,
            "weighted_position": 47.1,
            "queries": 7
        },
        {
            "week_start": "2026-09-14",
            "impressions": 68,
            "clicks": 0,
            "weighted_position": 36.1,
            "queries": 8
        },
        {
            "week_start": "2026-09-28",
            "impressions": 31,
            "clicks": 0,
            "weighted_position": 33.3,
            "queries": 5
        },
        {
            "week_start": "2026-10-05",
            "impressions": 28,
            "clicks": 0,
            "weighted_position": 20.9,
            "queries": 9,
            "partial": true,
            "days": 4
        }
    ],
    "aisymantix_ai_crawler_requests_monthly": [
        {
            "month": "2026-06",
            "requests": 362
        },
        {
            "month": "2026-07",
            "requests": 1045
        },
        {
            "month": "2026-08",
            "requests": 1631
        },
        {
            "month": "2026-09",
            "requests": 1754
        },
        {
            "month": "2026-10",
            "requests": 534,
            "partial": true,
            "days": 11
        }
    ],
    "ctr_curve_reference": {
        "file": "/ai/ctr-by-position.json",
        "coverage": "2025-10-07/2026-10-06",
        "properties": 155,
        "ctr_pct_by_bucket": {
            "1": 17.17,
            "2": 14.17,
            "3": 5.49,
            "4": 2.49,
            "5": 1.56,
            "6-10": 0.78,
            "11-20": 0.49,
            "21-50": 0.15,
            "51+": 0.03
        }
    },
    "october_ai_crawler_volume_caveat": {
        "september_requests": 763949,
        "september_sites": 171,
        "october_1_to_11_requests": 3571310,
        "october_sites": 194,
        "october_top_crawlers": [
            {
                "crawler": "GPTBot",
                "requests": 1801870
            },
            {
                "crawler": "meta-externalagent",
                "requests": 783222
            },
            {
                "crawler": "Amazonbot",
                "requests": 644738
            },
            {
                "crawler": "ClaudeBot",
                "requests": 302601
            }
        ],
        "october_top_site": {
            "domain": "bars.bellyuptampa.com",
            "requests": 310575
        },
        "reading": "The top twelve October sites are all BellyUp city finder lenses. The volume is crawlers looping through filter combinations, not interest in content. Raw AI crawler volume is not used as a finding in this case study."
    },
    "user_retrieval_dataset_fetches": {
        "requests": 11,
        "sites": 4,
        "crawlers": [
            "ChatGPT-User",
            "Claude-User"
        ]
    },
    "limits": [
        "A successful request proves a request reached the server. It does not prove indexing, citation, training use or a human reader.",
        "Human counts may include the publisher's own visits.",
        "The before and after comparison is a sequence, not proof of cause. The after window is one day longer and crawler schedules are not observable.",
        "Query rows from Search Console are not additive across pages."
    ]
}