{
  "test": "robots-meta",
  "title": "Robots meta and headers",
  "data_version": "1.0.0",
  "seed": 20261102,
  "page": "/crawl/robots-meta/",
  "needs": "HTTP only",
  "difficulty": "Medium",
  "what_it_tests": "Eight pages marked noindex and nofollow in meta tags, X-Robots-Tag headers and rel=\"nofollow\" links. Index the right ones; follow the right links.",
  "expect": {
    "pages": 8,
    "start_url": "/crawl/robots-meta/",
    "directives": [
      {
        "url": "/crawl/robots-meta/p/1/",
        "meta_robots": null,
        "x_robots_tag": null,
        "linked_from": "the test page",
        "links_to": []
      },
      {
        "url": "/crawl/robots-meta/p/2/",
        "meta_robots": "noindex",
        "x_robots_tag": null,
        "linked_from": "the test page",
        "links_to": [
          "/crawl/robots-meta/p/1/"
        ]
      },
      {
        "url": "/crawl/robots-meta/p/3/",
        "meta_robots": "nofollow",
        "x_robots_tag": null,
        "linked_from": "the test page",
        "links_to": [
          "/crawl/robots-meta/p/8/"
        ]
      },
      {
        "url": "/crawl/robots-meta/p/4/",
        "meta_robots": "noindex, nofollow",
        "x_robots_tag": null,
        "linked_from": "the test page",
        "links_to": [
          "/crawl/robots-meta/p/8/"
        ]
      },
      {
        "url": "/crawl/robots-meta/p/5/",
        "meta_robots": null,
        "x_robots_tag": "noindex",
        "linked_from": "the test page",
        "links_to": [
          "/crawl/robots-meta/p/1/"
        ]
      },
      {
        "url": "/crawl/robots-meta/p/6/",
        "meta_robots": null,
        "x_robots_tag": "nofollow",
        "linked_from": "the test page",
        "links_to": [
          "/crawl/robots-meta/p/8/"
        ]
      },
      {
        "url": "/crawl/robots-meta/p/7/",
        "meta_robots": null,
        "x_robots_tag": null,
        "linked_from": "the test page, by a link marked rel=\"nofollow\"",
        "links_to": []
      },
      {
        "url": "/crawl/robots-meta/p/8/",
        "meta_robots": null,
        "x_robots_tag": null,
        "linked_from": "pages 3, 4 and 6, whose links are not to be followed",
        "links_to": []
      }
    ],
    "polite_crawler": {
      "fetches": [
        "/crawl/robots-meta/p/1/",
        "/crawl/robots-meta/p/2/",
        "/crawl/robots-meta/p/3/",
        "/crawl/robots-meta/p/4/",
        "/crawl/robots-meta/p/5/",
        "/crawl/robots-meta/p/6/"
      ],
      "indexes": [
        "/crawl/robots-meta/p/1/",
        "/crawl/robots-meta/p/3/",
        "/crawl/robots-meta/p/6/"
      ],
      "never_fetches": [
        "/crawl/robots-meta/p/7/",
        "/crawl/robots-meta/p/8/"
      ],
      "note": "Page 7 is linked only with rel=\"nofollow\". Page 8 is linked only from pages whose links are not to be followed. robots.txt allows all of them; these are page-level hints."
    }
  },
  "records": [
    {
      "url": "/crawl/robots-meta/p/1/",
      "code": "ROBOTS-1"
    },
    {
      "url": "/crawl/robots-meta/p/3/",
      "code": "ROBOTS-3"
    },
    {
      "url": "/crawl/robots-meta/p/6/",
      "code": "ROBOTS-6"
    }
  ]
}