User-agent: *

Disallow: /application/
Disallow: /development/
Disallow: /externals/
Disallow: /temporary/
sitemap:  /index.php/sitemap?format=xml&rewrite=1

Crawl-delay: 30
#
# disallow adsense bot, as we no longer do adsense.
#
User-agent: Mediapartners-Google
Disallow: /

#
# Yahoo bot is evil.
#
User-agent: Slurp
Disallow: /

#
# Yahoo Pipes is for feeds not web pages.
#
User-agent: Yahoo Pipes 1.0
Disallow: /

User-agent: Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
Disallow: /

User-agent: Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)
Disallow: /

User-agent: MaCoCu
Disallow: /

User-agent: Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)
Disallow: /

User-agent: DataForSeoBot
Disallow: /

#
# disallow barkrowler bot, as we no longer do barkrowler.
#
User-agent: barkrowler
Disallow: /

#
# disallow barkrowler bot, as we no longer do barkrowler.
#
User-agent: Barkrowler
Disallow: /


User-agent: Mozilla/5.0+(compatible;+Barkrowler/0.9;++https://babbar.tech/crawler)
Disallow: /

User-agent: Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)
Disallow: /

User-agent: Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_10_1)+AppleWebKit/600.2.5+(KHTML,+like+Gecko)+Version/8.0.2+Safari/600.2.5+(Amazonbot/0.1;++https://developer.amazon.com/support/amazonbot)            
Disallow: /


User-agent: Mozilla/5.0+(Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15+(KHTML, like Gecko) Version/13.1.1 Safari/605.1.15+(Applebot/0.1; ++http://www.apple.com/go/applebot)
Disallow: /


User-agent: RollCrawl
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Sogou web spider 
Disallow: /

User-agent: Junglee Bot
Disallow: /

User-agent: Pinterestbot
Disallow: /

User-agent: MJ12Bot
Disallow: /

User-agent: SemrushBot-BM
Disallow: /

User-agent: SemrushBot-SA
Disallow: /



User-agent: AhrefsBot
Disallow: /

User-agent: NaverBot
Disallow: /


User-agent: AspiegelBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)
Disallow: /

User-agent: Bingbot
Disallow: /


User-agent: Mozilla/5.0+(compatible;+MegaIndex.ru/2.0;++http://megaindex.com/crawler)
Disallow: /

# Block  MegaIndex.ru

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: megaIndex.ru
Disallow: /


User-agent: CCBot
Disallow: /


User-agent: Applebot
Disallow: /

User-agent: Safari/605.1.15+(compatible;+Applebot/0.1;++http://www.apple.com/go/applebot)
Disallow: /