User-agent:  *
Allow: /

# disallow all files in these directories
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /wp-includes
Disallow: /wp-content
Disallow: /wp-login.php
Disallow: /trackback
Disallow: /comments
Disallow: */trackback
Disallow: */comments


User-agent: Screaming Frog SEO Spider
Allow: /

# disallow all files with ? in url
Disallow: /*?*

User-agent: Mediapartners-Google
Allow: /
 
User-agent: Adsbot-Google
Allow: /
 
User-agent: Googlebot-Image
Allow: /
 
User-agent: Googlebot-Mobile
Allow: /
 
User-agent: ia_archiver-web.archive.org
Disallow: /

Sitemap: http://domain.com/sitemap.xml

Sitemap: https://perniknews.com/sitemap_index.xml