user-agent: *
Allow: /
Allow: */cricket/test/
Allow: */cricketworldcup2019/Test/
Disallow: /toets/
Disallow: */toets/*
Disallow: /test/
Disallow: */_test/*
Disallow: */testpolar/*
Disallow: */test/*
Disallow: /xArchive/Archive/Illegal-liquor-export-20010319
Disallow: /.well-known/
Disallow: /assetlinks.json
Sitemap: https://www.news24.com/sitemap
User-agent: Twitterbot
Allow: /
User-agent: ia_archiver
Disallow: /BreakingNewsSms
User-Agent: AhrefsBot
Disallow: /
User-Agent: MauiBot
Disallow: /
User-agent: SemrushBot
Disallow: /