User-agent: *
Disallow: /dataset/rate/
Disallow: /revision/
Disallow: /dataset/*/history
Disallow: /api/

# faceted search trap
Disallow: */dataset/*?*groups=
Disallow: */dataset/*?*vocab_region_names_finally=
Disallow: */dataset/*?*vocab_country_names=
Disallow: */dataset/*?*year_updated=
Disallow: */dataset/*?*SpatialAdministrativeArea=
Disallow: */dataset/*?*category=
Disallow: */dataset/*?*tags=
Disallow: */dataset/*?*organization=
Disallow: */dataset/*?*res_format=
Disallow: */dataset/*?*license_id=
Disallow: */dataset/*?*theme=
Disallow: */dataset/*?*openness_score=
Disallow: */dataset/*?*q=
Allow: */dataset/?page=

# another faceted search trap
Disallow: */organization/*?*groups=
Disallow: */organization/*?*vocab_region_names_finally=
Disallow: */organization/*?*vocab_country_names=
Disallow: */organization/*?*year_updated=
Disallow: */organization/*?*SpatialAdministrativeArea=
Disallow: */organization/*?*category=
Disallow: */organization/*?*tags=
Disallow: */organization/*?*organization=
Disallow: */organization/*?*res_format=
Disallow: */organization/*?*license_id=
Disallow: */organization/*?*theme=
Disallow: */organization/*?*openness_score=
Disallow: */organization/*?*q=
Allow: */organization/?page=

Crawl-Delay: 10