User-agent: Scrapy
Allow: /

User-agent: *
Allow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/plugins/
Allow: /wp-content/uploads/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-includes/images/
Allow: .js
Allow: .css