# robots.txt — l4rg.com
# Updated: 2026-08-26
# Policy: allow crawling of public URLs by search engines, AI search/retrieval agents,
# and other standards-compliant crawlers.
#
# IMPORTANT: the wildcard group below already permits current, future and unknown bots.
# The explicit groups are included for clarity, compatibility and crawler-policy auditing.
# Some identifiers in the final compatibility section are legacy/community identifiers;
# they are not relied upon for access because User-agent: * already allows them.

User-agent: *
Allow: /

# --- Priority AI / answer-engine / agent crawlers ---
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Google-Agent
User-agent: Google-GeminiNotebook
User-agent: Google-NotebookLM
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: meta-externalfetcher
User-agent: CCBot
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: Bytespider
User-agent: PetalBot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: omgili
User-agent: omgilibot
Allow: /

# --- Major search engines and international discovery crawlers ---
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Googlebot-Mobile
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Mediapartners-Google
User-agent: APIs-Google
User-agent: Storebot-Google
User-agent: Google-InspectionTool
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-Safety
User-agent: FeedFetcher-Google
User-agent: GoogleProducer
User-agent: GoogleMessages
User-agent: Google-Pinpoint
User-agent: Google-CWS
User-agent: google-speakr
User-agent: bingbot
User-agent: BingPreview
User-agent: msnbot
User-agent: msnbot-media
User-agent: adidxbot
User-agent: MicrosoftPreview
User-agent: Slurp
User-agent: YahooSeeker
User-agent: Yahoo-MMCrawler
User-agent: YahooFeedSeeker
User-agent: Yahoo-Blogs
User-agent: Yahoo-VerticalCrawler
User-agent: DuckDuckBot
User-agent: DuckDuckGo-Favicons-Bot
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-video
User-agent: Baiduspider-news
User-agent: Baiduspider-favo
User-agent: Baiduspider-render
User-agent: Baiduspider-cpro
User-agent: YandexBot
User-agent: YandexImages
User-agent: YandexVideo
User-agent: YandexMedia
User-agent: YandexBlogs
User-agent: YandexFavicons
User-agent: YandexWebmaster
User-agent: YandexPagechecker
User-agent: YandexImageResizer
User-agent: YandexDirect
User-agent: YandexDirectDyn
User-agent: YandexCalendar
User-agent: YandexSitelinks
User-agent: YandexMetrika
User-agent: YandexNews
User-agent: YandexOntoDB
User-agent: YandexRCA
User-agent: YandexVertis
User-agent: YandexAccessibilityBot
User-agent: YandexRenderResourcesBot
User-agent: Sogou
User-agent: Sogou-web-spider
User-agent: Sogou-inst-spider
User-agent: Sogou-News-Spider
User-agent: Sogou-Orion-spider
User-agent: Sogou-Pic-Spider
User-agent: 360Spider
User-agent: HaosouSpider
User-agent: QihooBot
User-agent: NaverBot
User-agent: Yeti
User-agent: Daumoa
User-agent: Daum
User-agent: SeznamBot
User-agent: SeznamBot-Image
User-agent: SeznamBot-Video
User-agent: Qwantify
User-agent: Qwantbot
User-agent: Bravebot
User-agent: MojeekBot
User-agent: Exabot
User-agent: ExaleadCloudView
User-agent: Gigabot
User-agent: ia_archiver
User-agent: archive.org_bot
User-agent: Aolbot
User-agent: Teoma
User-agent: Ask
User-agent: Ecosia
User-agent: ecosia-bot
User-agent: swisscows
User-agent: YepBot
User-agent: YisouSpider
User-agent: YodaoBot
User-agent: YodaoBot-Image
User-agent: Ichiro
User-agent: SensisWebCrawler
User-agent: OnetSzukaj
User-agent: Wotbox
User-agent: WebAlta
User-agent: AcoonBot
User-agent: Accoona-AI-Agent
User-agent: Arachmo
User-agent: Aspseek
User-agent: Boitho
User-agent: boitho.com-dc
User-agent: ConveraCrawler
User-agent: Dumbot
User-agent: Eurobot
User-agent: Gaisbot
User-agent: GeonaBot
User-agent: Girafabot
User-agent: GurujiBot
User-agent: holmes
User-agent: IRLbot
User-agent: Jyxobot
User-agent: KaloogaBot
User-agent: LexxeBot
User-agent: Lsearch
User-agent: MLBot
User-agent: Mnogosearch
User-agent: Nusearch
User-agent: OmniExplorer_Bot
User-agent: Panscient
User-agent: Peerbot
User-agent: Pompos
User-agent: Qseero
User-agent: Scrubby
User-agent: ShopWiki
User-agent: Snappy
User-agent: SpeedySpider
User-agent: Steeler
User-agent: SynooBot
User-agent: voyager
User-agent: Wisenutbot
User-agent: Yanga
User-agent: ZyBorg
Allow: /

# --- Extended search, SEO, archive, monitoring and preview crawlers ---
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-SI
User-agent: SemrushBot-BA
User-agent: SemrushBot-CT
User-agent: SemrushBot-SWA
User-agent: SemrushBot-OCOB
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: rogerbot
User-agent: MozDotBot
User-agent: SiteAuditBot
User-agent: DataForSeoBot
User-agent: SEOkicks-Robot
User-agent: SEOdiver
User-agent: linkdexbot
User-agent: spbot
User-agent: LinkpadBot
User-agent: MegaIndex
User-agent: MegaIndex.ru
User-agent: SerpstatBot
User-agent: NetpeakCheckerBot
User-agent: SiteCheckerBot
User-agent: UbersuggestBot
User-agent: SISTRIX
User-agent: RyteBot
User-agent: OnPageBot
User-agent: Cocolyzebot
User-agent: SeobilityBot
User-agent: Seobility
User-agent: WooRankBot
User-agent: SearchmetricsBot
User-agent: BrightEdge
User-agent: ConductorBot
User-agent: seoClarityBot
User-agent: SiteimproveBot
User-agent: Monsidobot
User-agent: ContentKing
User-agent: ContentKingBot
User-agent: Botify
User-agent: OnCrawl
User-agent: DeepCrawl
User-agent: Lumar
User-agent: Sitebulb
User-agent: Prerender
User-agent: Chrome-Lighthouse
User-agent: Lighthouse
User-agent: Pingdom
User-agent: UptimeRobot
User-agent: StatusCake
User-agent: GTmetrix
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterestbot
User-agent: Slackbot
User-agent: Slack-ImgProxy
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: SkypeUriPreview
User-agent: Viber
User-agent: vkShare
User-agent: Mastodon
User-agent: redditbot
User-agent: Tumblr
User-agent: Embedly
User-agent: QuoraLinkPreview
User-agent: heritrix
User-agent: Nutch
User-agent: NutchCVS
User-agent: larbin
User-agent: HTTrack
User-agent: WebCopier
User-agent: Teleport
User-agent: WebZIP
User-agent: WebReaper
User-agent: OfflineExplorer
User-agent: Zealbot
User-agent: TurnitinBot
User-agent: psbot
User-agent: AntBot
User-agent: BSpider
User-agent: Cosmixcrawler
User-agent: Cuil
User-agent: EmeraldShield.com-WebBot
User-agent: Envolk
User-agent: FAST-WebCrawler
User-agent: FyberSpider
User-agent: genieBot
User-agent: gromit
User-agent: ICCrawler
User-agent: Krugle
User-agent: LapozzBot
User-agent: LMQueueBot
User-agent: LWP-Trivial
User-agent: MSRBOT
User-agent: Openbot
User-agent: RufusBot
User-agent: SurveyBot
User-agent: Tempel
User-agent: W3C-checklink
User-agent: W3C_Validator
User-agent: WebCapture
User-agent: WebCollage
User-agent: Websquash
User-agent: Webster
User-agent: Webwombat
User-agent: Zao
User-agent: Arale
User-agent: Arianna
User-agent: Amfibibot
User-agent: Aboundex
User-agent: Abonti
User-agent: FAST-Enterprise-Crawler
User-agent: Silk
User-agent: ScreamingFrogSEOSpider
User-agent: CensysInspect
User-agent: CensysBot
User-agent: NetcraftSurveyAgent
User-agent: ZoominfoBot
User-agent: SeekportBot
User-agent: Cliqzbot
User-agent: Cliqzbot-Nutch
User-agent: ZumBot
User-agent: findlinks
User-agent: ScreenerBot
User-agent: linkfluence
User-agent: Lipperhey-Kaus-Australis
User-agent: ltx71
User-agent: magpie-crawler
User-agent: DomainStatsBot
User-agent: trendictionbot
User-agent: OpenindexSpider
User-agent: SMTBot
User-agent: TinEye-bot
User-agent: MauiBot
User-agent: ZoomBot
User-agent: GrapeshotCrawler
User-agent: GrapeshotCrawler-Partner
User-agent: KomodiaBot
User-agent: LingueeBot
User-agent: Neevabot
User-agent: Seekbot
User-agent: StractBot
User-agent: StractBotBot
User-agent: KagiBot
User-agent: KagiFetcher
User-agent: MarginaliaSearch
User-agent: MarginaliaBot
User-agent: RightDaoBot
Allow: /

# --- Legacy/community compatibility identifiers retained from the previous file ---
User-agent: OpenAI-Python
User-agent: HuggingChat
User-agent: GrokBot
User-agent: CopilotBot
User-agent: CodeWhispererBot
User-agent: JasperBot
User-agent: ChatsonicBot
User-agent: GeminiBot
User-agent: BardBot
User-agent: PiBot
User-agent: PoeBot
User-agent: TabnineBot
User-agent: ElicitBot
User-agent: ContentShakeBot
User-agent: DeepSeek
User-agent: CharacterAI
User-agent: Writesonic
User-agent: AiToolsForEverydayTasks
User-agent: BlackboxBot
Allow: /

# Primary XML sitemap
Sitemap: https://www.l4rg.com/sitemap.xml