# robotstxt.org/

User-agent: AhrefsBot
User-agent: Gigabot
User-agent: PiplBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SEOkicks
User-agent: seoscanners.net
User-agent: TurnitinBot
Disallow: /

User-agent: CCBot
User-agent: Linguee
User-agent: TigerBot
Crawl-delay: 10
Disallow: /xml/tei_staged/
Disallow: README
Disallow: /*/rotate/facsimile/*
Disallow: /*/rotate/binding/*
Disallow: /*/change_size/binding/*
Disallow: /*/change_size/facsimile/*
Disallow: /*/change_page/facsimile/*
Disallow: /*/change_page/binding/*
Disallow: /*/preview/
Disallow: /*/send/*
Disallow: /*/download/*
Disallow: /*/list/all/title/*
Disallow: /mobile/*

User-agent: ia_archiver
User-agent: Yandex
Crawl-delay: 5
Disallow: /xml/tei_staged/
Disallow: README
Disallow: /*/rotate/facsimile/*
Disallow: /*/rotate/binding/*
Disallow: /*/change_size/binding/*
Disallow: /*/change_size/facsimile/*
Disallow: /*/change_page/facsimile/*
Disallow: /*/change_page/binding/*
Disallow: /*/preview/
Disallow: /*/send/*
Disallow: /*/download/*
Disallow: /*/list/all/title/*
Disallow: /mobile/*

User-agent: *
Crawl-delay: 3
Disallow: /xml/tei_staged/
Disallow: README
Disallow: /*/rotate/facsimile/*
Disallow: /*/rotate/binding/*
Disallow: /*/change_size/binding/*
Disallow: /*/change_size/facsimile/*
Disallow: /*/change_page/facsimile/*
Disallow: /*/change_page/binding/*
Disallow: /*/preview/
Disallow: /*/send/*
Disallow: /*/download/*
Disallow: /*/list/all/title/*
Disallow: /mobile/*