User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Googlebot-Mobile
User-agent: APIs-Google
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-Extended
User-agent: Mediapartners-Google
User-agent: Google-Display-Ads-Bot
User-agent: Storebot-Google
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
User-agent: Bingbot
User-agent: Twitterbot
User-agent: facebookexternalhit
Disallow: /cgi-bin/
Crawl-delay: 1
Allow: /

User-agent: *
Disallow: /

Sitemap: https://mot-history.co.uk/sitemap/vehicles_index.xml.gz