# robots.txt — remteh.by
# AI navigation map:  https://remteh.by/llms.txt
# Full AI knowledge base: https://remteh.by/llms-full.txt
# Sitemap: https://remteh.by/sitemap.xml

# ============================================================
#  AI assistants & answer engines — explicitly welcome.
#  We WANT to be cited in ChatGPT / Claude / Perplexity / Gemini,
#  so these bots get the same access as everyone else (only admin
#  dirs and discontinued services are blocked). No Crawl-delay:
#  most AI crawlers ignore it and it only throttles the very
#  bots we are trying to attract.
#  Categories:
#    retrieval (cite us live):  OAI-SearchBot, Claude-SearchBot, PerplexityBot
#    on-demand user fetch:      ChatGPT-User, Claude-User, Perplexity-User
#    training / knowledge:      GPTBot, ClaudeBot, Google-Extended,
#                               Applebot-Extended, Meta-ExternalAgent
# ============================================================
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
# (no "Allow: /" — default is allow; listing it before Disallow breaks naive first-match parsers)
Disallow: /administrator/
Disallow: /bin/
Disallow: /cache/
Disallow: /cgi-sys/
Disallow: /cli/
Disallow: /components/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /layouts/
Disallow: /libraries/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /erp/
Disallow: /gadjet.by/
Disallow: /tmp/
# Render-critical assets: CSS/JS должны быть доступны краулерам для корректного рендера
Allow: /cache/com_templates/
Allow: /components/*.css
Allow: /components/*.js
Allow: /modules/*.css
Allow: /modules/*.js
Allow: /plugins/*.css
Allow: /plugins/*.js
# Discontinued services (2026-05-20)
Disallow: /index.php?option=com_sppagebuilder&view=page&id=20
Disallow: /index.php?option=com_sppagebuilder&view=page&id=49
Disallow: /index.php?option=com_sppagebuilder&view=page&id=109
Disallow: /index.php?option=com_sppagebuilder&view=page&id=180
Disallow: /index.php?option=com_sppagebuilder&view=page&id=188

# Amazonbot is intentionally not used for this local-service site.
User-agent: Amazonbot
Disallow: /

User-agent: YandexBot
Disallow: /administrator/
Disallow: /bin/
Disallow: /cache/
Disallow: /cgi-sys/
Disallow: /cli/
Disallow: /components/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /layouts/
Disallow: /libraries/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /erp/
Disallow: /gadjet.by/
Disallow: /tmp/
# Render-critical assets: CSS/JS должны быть доступны краулерам для корректного рендера
Allow: /cache/com_templates/
Allow: /components/*.css
Allow: /components/*.js
Allow: /modules/*.css
Allow: /modules/*.js
Allow: /plugins/*.css
Allow: /plugins/*.js
# Discontinued services (2026-05-20)
Disallow: /index.php?option=com_sppagebuilder&view=page&id=20
Disallow: /index.php?option=com_sppagebuilder&view=page&id=49
Disallow: /index.php?option=com_sppagebuilder&view=page&id=109
Disallow: /index.php?option=com_sppagebuilder&view=page&id=180
Disallow: /index.php?option=com_sppagebuilder&view=page&id=188
Clean-param: etext&ybaip&yclid&yandex-source&gad_source&gclid&fbclid&msclkid&helixMode&NA&SD /
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term&cb /
Clean-param: form-builder-item-[phone]&form-builder-item-[problem]&form_id&addon_id&email_subject&email_template&success_message&failed_message&captcha_type /
# QA-метки от проверок вёрстки (добавлено 2026-07-21: засоряли индекс, 30% дублей)
Clean-param: qa&audit&audit_screen&audit-final-shot&browser-audit&browsercheck&chrome-audit&stick-browser-audit&seo_verify&assert&menu_check /
Clean-param: v&edge&m&lh&fin&final&final-qa&finalqa&finalmobile&finalwide&final_mobile&final_desktop&before&from-qa /
Clean-param: cbv2final&cbv2qa&gpv5&gp7-lh&gp7-shot&gp7-matrix&rtqa&rtmenu-live&rtmenu-final&rtbrandv2_visual&brands_visual&schema_visual /
Clean-param: codex_visual&codex_check&codex_logo_check&codex_pw&codex_ui_check&diag_render&diag_after_width_fix&diag_after_width_class_fix /
Clean-param: visual&visual-qa&visual-nav&visual_after&visual_audit&visual_mobile&visual_nocache&bust&nocache&cachebust /
Clean-param: cbv2matrix&final_v12_3&final_v12_3_contact&gap-before&phone-gap-audit&reference&rtcheck&v13_regression&ved&visualaudit&vw /
Clean-param: r&b&fresh&navqa&compactqa&wideqa&rt-live-design /

User-agent: *
Disallow: /administrator/
Disallow: /bin/
Disallow: /cache/
Disallow: /cgi-sys/
Disallow: /cli/
Disallow: /components/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /layouts/
Disallow: /libraries/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /erp/
Disallow: /gadjet.by/
Disallow: /tmp/
# Render-critical assets: CSS/JS должны быть доступны краулерам для корректного рендера
Allow: /cache/com_templates/
Allow: /components/*.css
Allow: /components/*.js
Allow: /modules/*.css
Allow: /modules/*.js
Allow: /plugins/*.css
Allow: /plugins/*.js
# Discontinued services (2026-05-20)
Disallow: /index.php?option=com_sppagebuilder&view=page&id=20
Disallow: /index.php?option=com_sppagebuilder&view=page&id=49
Disallow: /index.php?option=com_sppagebuilder&view=page&id=109
Disallow: /index.php?option=com_sppagebuilder&view=page&id=180
Disallow: /index.php?option=com_sppagebuilder&view=page&id=188

Sitemap: https://remteh.by/sitemap.xml