# Go Do (go-do.live). Pages are open to search engines, answer engines and AI assistants.
# The listings may not be bulk-copied, scraped or republished: https://go-do.live/terms.html

User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
Allow: /
Disallow: /admin
Disallow: /oauth/
Disallow: /chat/
Disallow: /settings/
Disallow: /edit-profile
Disallow: /notifications

User-agent: *
Allow: /
Disallow: /admin
Disallow: /oauth/
Disallow: /chat/
Disallow: /settings/
Disallow: /edit-profile
Disallow: /notifications
Disallow: /events.json
Disallow: /events-list.json
Disallow: /events-soon.json
Disallow: /event-text/
Disallow: /activities.json
Disallow: /restaurants.json
Disallow: /lodging.json
Disallow: /stock/catalog.json

# Bulk collectors that republish or resell what they collect, or ignore site owners:
#   CCBot: Common Crawl: republishes everything it crawls as a free public dataset
#   Bytespider: ByteDance: widely reported to ignore robots.txt and crawl very heavily
#   Diffbot: sells a knowledge graph and extraction data built from crawled pages
#   omgili: Webz.io (Omgili): sells crawled web data feeds
#   omgilibot: Webz.io (Omgili): sells crawled web data feeds
#   Webzio-Extended: Webz.io: sells crawled web data for AI training
#   ImagesiftBot: Hive ImageSift: builds a reverse-image dataset from crawled pages
#   Timpibot: Timpi: sells its crawled index as datasets
#   img2dataset: a tool for bulk-downloading images into datasets
#   Ai2Bot-Dolma: collects for Dolma, an openly republished training dataset
#   Kangaroo Bot: collects pages to build AI training datasets for others
#   VelenPublicWebCrawler: Velen (Hunter.io): builds datasets for others from crawled pages
#   magpie-crawler: Brandwatch: collects pages into a resold data product
User-agent: CCBot
User-agent: Bytespider
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: Webzio-Extended
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: img2dataset
User-agent: Ai2Bot-Dolma
User-agent: Kangaroo Bot
User-agent: VelenPublicWebCrawler
User-agent: magpie-crawler
Disallow: /

Sitemap: https://go-do.live/sitemap.xml