From 91c1e6035b1d17a2180dabb28b26d017b7306dfb Mon Sep 17 00:00:00 2001
From: wangzipai <wangzipaiya@gmail.com>
Date: Wed, 30 Oct 2024 15:32:11 +0800
Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0robots.txt?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 content/robots.txt | 135 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 135 insertions(+)
 create mode 100644 content/robots.txt

diff --git a/content/robots.txt b/content/robots.txt
new file mode 100644
index 000000000..bc88b44fe
--- /dev/null
+++ b/content/robots.txt
@@ -0,0 +1,135 @@
+############################# ROBOTS.TXT ###############################
+# Updates and informantion can be found at:                            #
+# https://www.ditig.com/publications/robots-txt-template               #
+# This document is licensed with a CC BY-NC-SA 4.0 license.            #
+# Last update: 2023-11-28                                              #
+########################################################################
+# so.com chinese search engine
+User-agent: 360Spider
+User-agent: 360Spider-Image
+User-agent: 360Spider-Video
+# google.com landing page quality checks
+User-agent: AdsBot-Google
+User-agent: AdsBot-Google-Mobile
+# google.com app resource fetcher
+User-agent: AdsBot-Google-Mobile-Apps
+# bing ads bot
+User-agent: adidxbot
+# apple.com search engine
+User-agent: Applebot
+user-agent: AppleNewsBot
+# baidu.com chinese search engine
+User-agent: Baiduspider
+User-agent: Baiduspider-image
+User-agent: Baiduspider-news
+User-agent: Baiduspider-video
+# bing.com international search engine
+User-agent: bingbot
+User-agent: BingPreview
+# bublup.com suggestion/search engine
+User-agent: BublupBot
+# commoncrawl.org open repository of web crawl data
+User-agent: CCBot
+# cliqz.com german in-product search engine
+User-agent: Cliqzbot
+# coccoc.com vietnamese search engine
+User-agent: coccoc
+User-agent: coccocbot-image
+User-agent: coccocbot-web
+# daum.net korean search engine
+User-agent: Daumoa
+# dazoo.fr french search engine
+User-agent: Dazoobot
+# deusu.de german search engine
+User-agent: DeuSu
+# duckduckgo.com international privacy search engine
+User-agent: DuckDuckBot
+User-agent: DuckDuckGo-Favicons-Bot
+# eurip.com european search engine
+User-agent: EuripBot
+# exploratodo.com latin search engine
+User-agent: Exploratodo
+# facebook.com social network
+User-agent: facebookcatalog
+User-agent: facebookexternalhit
+User-agent: Facebot
+# feedly.com feed fetcher
+User-agent: Feedly
+# findx.com european search engine
+User-agent: Findxbot
+# goo.ne.jp japanese search engine
+User-agent: gooblog
+# google.com international search engine
+User-agent: Googlebot
+User-agent: Googlebot-Image
+User-agent: Googlebot-Mobile
+User-agent: Googlebot-News
+User-agent: Googlebot-Video
+# so.com chinese search engine
+User-agent: HaoSouSpider
+# goo.ne.jp japanese search engine
+User-agent: ichiro
+# istella.it italian search engine
+User-agent: istellabot
+# jike.com / chinaso.com chinese search engine
+User-agent: JikeSpider
+# lycos.com & hotbot.com international search engine
+User-agent: Lycos
+# mail.ru russian search engine
+User-agent: Mail.Ru
+# google.com adsense bot
+User-agent: Mediapartners-Google
+# Preview bot for Microsoft products
+User-agent: MicrosoftPreview
+# mojeek.com search engine
+User-agent: MojeekBot
+# bing.com international search engine
+User-agent: msnbot
+User-agent: msnbot-media
+# orange.com international search engine
+User-agent: OrangeBot
+# pinterest.com social networtk
+User-agent: Pinterest
+# botje.nl dutch search engine
+User-agent: Plukkie
+# qwant.com french search engine
+User-agent: Qwantify
+# rambler.ru russian search engine
+User-agent: Rambler
+# seznam.cz czech search engine
+User-agent: SeznamBot
+# soso.com chinese search engine
+User-agent: Sosospider
+# yahoo.com international search engine
+User-agent: Slurp
+# sogou.com chinese search engine
+User-agent: Sogou blog
+User-agent: Sogou inst spider
+User-agent: Sogou News Spider
+User-agent: Sogou Orion spider
+User-agent: Sogou spider2
+User-agent: Sogou web spider
+# twitter.com social media bot
+User-agent: Twitterbot
+# whatsapp.com preview bot
+User-agent: WhatsApp
+# yacy.net p2p search software
+User-agent: yacybot
+# yandex.com russian search engine
+User-agent: Yandex
+User-agent: YandexMobileBot
+# yep.com search engine
+User-agent: YepBot
+# search.naver.com south korean search engine
+user-agent: Yeti
+# yioop.com international search engine
+User-agent: YioopBot
+# yooz.ir iranian search engine
+User-agent: yoozBot
+# youdao.com chinese search engine
+User-agent: YoudaoBot
+# crawling rule(s) for above bots
+Disallow:
+# disallow all other bots
+User-agent: *
+Disallow: /