User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: Web Downloader
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer Pro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: HTTrack Website Copier
Disallow: /

User-agent: Offline Commander
Disallow: /

User-agent: Leech
Disallow: /

User-agent: WebSnake
Disallow: /

User-agent: BlackWidow
Disallow: /

User-agent: HTTP Weazel
Disallow: /

## GENERAL SETTINGS
## Enable robots.txt rules for all crawlers
User-agent: *
Disallow: /identicon.php
## Do not crawl app page
Disallow: /*id*
Disallow: /?*
Disallow: /*page=*
Disallow: /en/
Disallow: /app/
## Do not crawl Magento admin page
Disallow: /admin/
Disallow: /login
Disallow: /register
Disallow: /buy-now-product
Disallow: /forgot-password
Disallow: /*.aspx
Disallow: /mo-to-dien-dia-hinh-4-banh-zp-5118
Disallow: /index.php/admin/
## Do not crawl 2-nd home page copy (example.com/index.php/). Uncomment it only if you activated Magento SEO URLs.
Disallow: /index.php/
Disallow: /cart
Disallow: /thanh-toan
Disallow: /chinh-sach-uu-dai
Disallow: /giay-truot-patin
Disallow: /xe-cho-be
Disallow: /test-1
Disallow: /test
Disallow: /test-2
Disallow: /test-status
Disallow: /ngua-sat
##Disallow: /tin-hot
Disallow: /xe-lac-loai-lon
##Disallow: /van-truot-skateboard
Disallow: /xe-thu-chay-bang-dien
## Xetreem Sitemap 
Sitemap: https://xetreem.com.vn/sitemap.xml