diff --git a/docs/robots.txt b/docs/robots.txt index 65eefa9..c26d83b 100644 --- a/docs/robots.txt +++ b/docs/robots.txt @@ -1,111 +1,4 @@ -User-agent: Googlebot-Image -User-agent: msnbot-media -User-agent: GPTBot -User-agent: CCBot -User-agent: ChatGPT-User -User-agent: ia-archiver -User-agent: omgili -User-agent: omgili-bot -User-agent: Omgilibot -User-agent: img2dataset -User-agent: Twitterbot -User-agent: Google-Extended -User-agent: Applebot-Extended -User-agent: anthropic-ai -User-agent: Claude-Web -User-agent: FacebookBot -User-agent: Bytespider -User-agent: magpie-crawler -User-agent: AI2Bot -User-agent: Ai2Bot-Dolma -User-agent: Amazonbot -User-agent: anthropic-ai -User-agent: Applebot -User-agent: Applebot-Extended -User-agent: Bytespider -User-agent: Claude-Web -User-agent: ClaudeBot -User-agent: cohere-ai -User-agent: Diffbot -User-agent: DuckAssistBot -User-agent: FacebookBot -User-agent: FriendlyCrawler -User-agent: Google-Extended -User-agent: iaskspider/2.0 -User-agent: ICC-Crawler -User-agent: ImagesiftBot -User-agent: img2dataset -User-agent: ISSCyberRiskCrawler -User-agent: Kangaroo Bot -User-agent: Meta-ExternalAgent -User-agent: Meta-ExternalFetcher -User-agent: OAI-SearchBot -User-agent: omgili -User-agent: omgilibot -User-agent: PanguBot -User-agent: PerplexityBot -User-agent: PetalBot -User-agent: Scrapy -User-agent: Sidetrade indexer bot -User-agent: Timpibot -User-agent: VelenPublicWebCrawler -User-agent: Webzio-Extended -User-agent: YouBot - Disallow: / -User-Agent: * - Disallow: *.txt - Disallow: *.pdf - Disallow: *.doc - Disallow: *.docx - Disallow: *.odt - Disallow: *.rtf - Disallow: *.tex - Disallow: *.wks - Disallow: *.wpd - Disallow: *.wps - Disallow: *.html - Disallow: *.bmp - Disallow: *.gif - Disallow: *.ico - Disallow: *.jpeg - Disallow: *.jpg - Disallow: *.png - Disallow: *.svg - Disallow: *.tif - Disallow: *.tiff - Disallow: *.webp - Disallow: *.aac - Disallow: *.aiff - Disallow: *.amr - Disallow: *.flac - Disallow: *.m4a - Disallow: *.mp3 - Disallow: *.oga - Disallow: *.opus - Disallow: *.wav - Disallow: *.wma - Disallow: *.mp4 - Disallow: *.webm - Disallow: *.ogg - Disallow: *.avi - Disallow: *.mov - Disallow: *.wmv - Disallow: *.flv - Disallow: *.mkv - Disallow: *.py - Disallow: *.js - Disallow: *.java - Disallow: *.c - Disallow: *.cpp - Disallow: *.cs - Disallow: *.h - Disallow: *.css - Disallow: *.php - Disallow: *.swift - Disallow: *.go - Disallow: *.rb - Disallow: *.pl - Disallow: *.sh - Disallow: *.sql +User-agent: * +Disallow: / Host: https://gugulet.hu