mirror of
https://github.com/jackyzha0/quartz.git
synced 2025-12-24 13:24:05 -06:00
136 lines
4.0 KiB
Plaintext
136 lines
4.0 KiB
Plaintext
############################# ROBOTS.TXT ###############################
|
|
# Updates and informantion can be found at: #
|
|
# https://www.ditig.com/publications/robots-txt-template #
|
|
# This document is licensed with a CC BY-NC-SA 4.0 license. #
|
|
# Last update: 2023-11-28 #
|
|
########################################################################
|
|
# so.com chinese search engine
|
|
User-agent: 360Spider
|
|
User-agent: 360Spider-Image
|
|
User-agent: 360Spider-Video
|
|
# google.com landing page quality checks
|
|
User-agent: AdsBot-Google
|
|
User-agent: AdsBot-Google-Mobile
|
|
# google.com app resource fetcher
|
|
User-agent: AdsBot-Google-Mobile-Apps
|
|
# bing ads bot
|
|
User-agent: adidxbot
|
|
# apple.com search engine
|
|
User-agent: Applebot
|
|
user-agent: AppleNewsBot
|
|
# baidu.com chinese search engine
|
|
User-agent: Baiduspider
|
|
User-agent: Baiduspider-image
|
|
User-agent: Baiduspider-news
|
|
User-agent: Baiduspider-video
|
|
# bing.com international search engine
|
|
User-agent: bingbot
|
|
User-agent: BingPreview
|
|
# bublup.com suggestion/search engine
|
|
User-agent: BublupBot
|
|
# commoncrawl.org open repository of web crawl data
|
|
User-agent: CCBot
|
|
# cliqz.com german in-product search engine
|
|
User-agent: Cliqzbot
|
|
# coccoc.com vietnamese search engine
|
|
User-agent: coccoc
|
|
User-agent: coccocbot-image
|
|
User-agent: coccocbot-web
|
|
# daum.net korean search engine
|
|
User-agent: Daumoa
|
|
# dazoo.fr french search engine
|
|
User-agent: Dazoobot
|
|
# deusu.de german search engine
|
|
User-agent: DeuSu
|
|
# duckduckgo.com international privacy search engine
|
|
User-agent: DuckDuckBot
|
|
User-agent: DuckDuckGo-Favicons-Bot
|
|
# eurip.com european search engine
|
|
User-agent: EuripBot
|
|
# exploratodo.com latin search engine
|
|
User-agent: Exploratodo
|
|
# facebook.com social network
|
|
User-agent: facebookcatalog
|
|
User-agent: facebookexternalhit
|
|
User-agent: Facebot
|
|
# feedly.com feed fetcher
|
|
User-agent: Feedly
|
|
# findx.com european search engine
|
|
User-agent: Findxbot
|
|
# goo.ne.jp japanese search engine
|
|
User-agent: gooblog
|
|
# google.com international search engine
|
|
User-agent: Googlebot
|
|
User-agent: Googlebot-Image
|
|
User-agent: Googlebot-Mobile
|
|
User-agent: Googlebot-News
|
|
User-agent: Googlebot-Video
|
|
# so.com chinese search engine
|
|
User-agent: HaoSouSpider
|
|
# goo.ne.jp japanese search engine
|
|
User-agent: ichiro
|
|
# istella.it italian search engine
|
|
User-agent: istellabot
|
|
# jike.com / chinaso.com chinese search engine
|
|
User-agent: JikeSpider
|
|
# lycos.com & hotbot.com international search engine
|
|
User-agent: Lycos
|
|
# mail.ru russian search engine
|
|
User-agent: Mail.Ru
|
|
# google.com adsense bot
|
|
User-agent: Mediapartners-Google
|
|
# Preview bot for Microsoft products
|
|
User-agent: MicrosoftPreview
|
|
# mojeek.com search engine
|
|
User-agent: MojeekBot
|
|
# bing.com international search engine
|
|
User-agent: msnbot
|
|
User-agent: msnbot-media
|
|
# orange.com international search engine
|
|
User-agent: OrangeBot
|
|
# pinterest.com social networtk
|
|
User-agent: Pinterest
|
|
# botje.nl dutch search engine
|
|
User-agent: Plukkie
|
|
# qwant.com french search engine
|
|
User-agent: Qwantify
|
|
# rambler.ru russian search engine
|
|
User-agent: Rambler
|
|
# seznam.cz czech search engine
|
|
User-agent: SeznamBot
|
|
# soso.com chinese search engine
|
|
User-agent: Sosospider
|
|
# yahoo.com international search engine
|
|
User-agent: Slurp
|
|
# sogou.com chinese search engine
|
|
User-agent: Sogou blog
|
|
User-agent: Sogou inst spider
|
|
User-agent: Sogou News Spider
|
|
User-agent: Sogou Orion spider
|
|
User-agent: Sogou spider2
|
|
User-agent: Sogou web spider
|
|
# twitter.com social media bot
|
|
User-agent: Twitterbot
|
|
# whatsapp.com preview bot
|
|
User-agent: WhatsApp
|
|
# yacy.net p2p search software
|
|
User-agent: yacybot
|
|
# yandex.com russian search engine
|
|
User-agent: Yandex
|
|
User-agent: YandexMobileBot
|
|
# yep.com search engine
|
|
User-agent: YepBot
|
|
# search.naver.com south korean search engine
|
|
user-agent: Yeti
|
|
# yioop.com international search engine
|
|
User-agent: YioopBot
|
|
# yooz.ir iranian search engine
|
|
User-agent: yoozBot
|
|
# youdao.com chinese search engine
|
|
User-agent: YoudaoBot
|
|
# crawling rule(s) for above bots
|
|
Disallow:
|
|
# disallow all other bots
|
|
User-agent: *
|
|
Disallow: /
|