#Welcome to CrawlArabia
#Updated: OCT 2022

#NOENTRY ROAD SIGNS
User-agent: *
Disallow: /wp/wp-admin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /classified/*
Disallow: */amp
Disallow: */amp/
Disallow: */amp/*

#PERMITTED VEHICLES
User-agent: Mediapartners-Google
Allow: /
User-agent: Googlebot-news
Allow: /

#UNWANTED VEHICLES
User-agent: HaoSouSpider
User-agent: Arquivo-web-crawler
User-agent: Baiduspider
User-agent: 360Spider
User-agent: Yisouspider
User-agent: Bytespider
User-agent: Sogou web spider
User-agent: Sogou spider2
User-agent: Sogou inst spider
User-agent: sitecheck.internetseer.com
User-agent: Zealbot
User-agent: MSIECrawler
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebZIP
User-agent: linko
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: Xenu
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Download Ninja
User-agent: WebReaper
User-agent: Screaming Frog SEO Spider
User-agent: Yeti 
Disallow: /

User-agent: grapeshot
Crawl-delay: 1000

Sitemap: http://www.drivearabia.com/sitemap_index.xml