Merhaba Arkadaşlar Örnek Robot Dosyası oluşturarak size göndermek istedim siteye siteleri ziyaret eden zararlı botları engelledim zararlı botları ekleyerek listeye robots dosyasını geliştire bilirsiniz kolay gelsin

Tek yapmanız gereken robots.txt dosyası oluşturup yazdığım kodları içine kopyalayıp ftp adresinize atmak.



User-agent: *




# Verzeichnisse
User-agent: *
Disallow: /admin/
Disallow: /css/
Disallow: /jr/

# Sitemap
#Allow: /sitemap.xml

# Bildersuche
User-agent: Mediapartners-Google*
Disallow: /


# ---------------------------------
# Spider
User-agent: sistrix
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Yahoo-MMCrawler
Disallow: /

User-agent: Yahoo-MMCrawler
Disallow: /

# Ausschluss spiderbots
User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: larbin
Disallow: /

User-agent: b2w/0.1
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: URL_Spider_Pro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver/1.6
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: MIIxpc
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: moget/2.1
Disallow: /

User-agent: WebZip/4.0
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: Mister PiX
Disallow: /

User-agent: WebAuto
Disallow: /

User-agent: TheNomad
Disallow: /

User-agent: WWW-Collector-E
Disallow: /

User-agent: RMA
Disallow: /

User-agent: libWeb/clsHTTP
Disallow: /

User-agent: asterias
Disallow: /

User-agent: httplib
Disallow: /

User-agent: turingos
Disallow: /

User-agent: spanner
Disallow: /

User-agent: InfoNaviRobot
Disallow: /

User-agent: Harvest/1.5
Disallow: /

User-agent: Bullseye/1.0
Disallow: /

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /

User-agent: CherryPickerSE/1.0
Disallow: /

User-agent: CherryPickerElite/1.0
Disallow: /

User-agent: WebBandit/3.50
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: Microsoft URL Control - 5.01.4511
Disallow: /

User-agent: DittoSpyder
Disallow: /

User-agent: Foobot
Disallow: /

User-agent: WebmasterWorldForumBot
Disallow: /

User-agent: SpankBot
Disallow: /

User-agent: BotALot
Disallow: /

User-agent: lwp-trivial/1.34
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: BunnySlippers
Disallow: /

User-agent: Microsoft URL Control - 6.00.8169
Disallow: /

User-agent: URLy Warning
Disallow: /

User-agent: Wget/1.6
Disallow: /

User-agent: Wget/1.5.3
Disallow: /

User-agent: Wget
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: cosmos
Disallow: /

User-agent: moget
Disallow: /

User-agent: hloader
Disallow: /

User-agent: humanlinks
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Mata Hari
Disallow: /

User-agent: LexiBot
Disallow: /

User-agent: Web Image Collector
Disallow: /

User-agent: The Intraformant
Disallow: /

User-agent: True_Robot/1.0
Disallow: /

User-agent: True_Robot
Disallow: /

User-agent: BlowFish/1.0
Disallow: /

User-agent: JennyBot
Disallow: /

User-agent: MIIxpc/4.2
Disallow: /

User-agent: BuiltBotTough
Disallow: /

User-agent: ProPowerBot/2.14
Disallow: /

User-agent: BackDoorBot/1.0
Disallow: /

User-agent: toCrawl/UrlDispatcher
Disallow: /

User-agent: WebEnhancer
Disallow: /

User-agent: suzuran
Disallow: /

User-agent: VCI WebViewer VCI WebViewer Win32
Disallow: /

User-agent: VCI
Disallow: /

User-agent: Szukacz/1.4
Disallow: /

User-agent: QueryN Metasearch
Disallow: /

User-agent: Openfind data gathere
Disallow: /

User-agent: Openfind
Disallow: /

User-agent: Xenu's Link Sleuth 1.1c
Disallow: /

User-agent: Xenu's
Disallow: /

User-agent: Zeus
Disallow: /

User-agent: RepoMonkey Bait & Tackle/v1.01
Disallow: /

User-agent: RepoMonkey
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: Openbot
Disallow: /

User-agent: URL Control
Disallow: /

User-agent: Zeus Link Scout
Disallow: /

User-agent: Zeus 32297 Webster Pro V2.9 Win32
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: EroCrawler
Disallow: /

User-agent: LinkScan/8.1a Unix
Disallow: /

User-agent: Keyword Density/0.9
Disallow: /

User-agent: Kenjin Spider
Disallow: /

User-agent: Iron33/1.0.2
Disallow: /

User-agent: Bookmark search tool
Disallow: /

User-agent: GetRight/4.2
Disallow: /

User-agent: FairAd Client
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: Aqua_Products
Disallow: /

User-agent: Radiation Retriever 1.1
Disallow: /

User-agent: Flaming AttackBot
Disallow: /

User-agent: Oracle Ultra Search
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: PerMan
Disallow: /

User-agent: searchpreview
Disallow: /

User-agent: vscooter # AltaVista Image Search
Disallow: /

User-agent: psbot
Disallow: /

User-agent: DittoSpyder
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: larbin
Disallow: /

User-agent: b2w/0.1
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: URL_Spider_Pro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver/1.6
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: MIIxpc
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: moget/2.1
Disallow: /

User-agent: WebZip/4.0
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: Mister PiX
Disallow: /

User-agent: WebAuto
Disallow: /

User-agent: TheNomad
Disallow: /

User-agent: WWW-Collector-E
Disallow: /

User-agent: RMA
Disallow: /

User-agent: libWeb/clsHTTP
Disallow: /

User-agent: asterias
Disallow: /

User-agent: httplib
Disallow: /

User-agent: turingos
Disallow: /

User-agent: spanner
Disallow: /

User-agent: InfoNaviRobot
Disallow: /

User-agent: Harvest/1.5
Disallow: /

User-agent: Bullseye/1.0
Disallow: /

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /

User-agent: CherryPickerSE/1.0
Disallow: /

User-agent: CherryPickerElite/1.0
Disallow: /

User-agent: WebBandit/3.50
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: Microsoft URL Control - 5.01.4511
Disallow: /

User-agent: DittoSpyder
Disallow: /

User-agent: Foobot
Disallow: /

User-agent: WebmasterWorldForumBot
Disallow: /

User-agent: SpankBot
Disallow: /

User-agent: BotALot
Disallow: /

User-agent: lwp-trivial/1.34
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: BunnySlippers
Disallow: /

User-agent: Microsoft URL Control - 6.00.8169
Disallow: /

User-agent: URLy Warning
Disallow: /

User-agent: Wget/1.6
Disallow: /

User-agent: Wget/1.5.3
Disallow: /

User-agent: Wget
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: cosmos
Disallow: /

User-agent: moget
Disallow: /

User-agent: hloader
Disallow: /

User-agent: humanlinks
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Mata Hari
Disallow: /

User-agent: LexiBot
Disallow: /

User-agent: Web Image Collector
Disallow: /

User-agent: The Intraformant
Disallow: /

User-agent: True_Robot/1.0
Disallow: /

User-agent: True_Robot
Disallow: /

User-agent: BlowFish/1.0
Disallow: /

User-agent: JennyBot
Disallow: /

User-agent: MIIxpc/4.2
Disallow: /

User-agent: BuiltBotTough
Disallow: /

User-agent: ProPowerBot/2.14
Disallow: /

User-agent: BackDoorBot/1.0
Disallow: /

User-agent: toCrawl/UrlDispatcher
Disallow: /

User-agent: WebEnhancer
Disallow: /

User-agent: suzuran
Disallow: /

User-agent: VCI WebViewer VCI WebViewer Win32
Disallow: /

User-agent: VCI
Disallow: /

User-agent: Szukacz/1.4
Disallow: /

User-agent: QueryN Metasearch
Disallow: /

User-agent: Openfind data gathere
Disallow: /

User-agent: Openfind
Disallow: /

User-agent: Xenu's Link Sleuth 1.1c
Disallow: /

User-agent: Xenu's
Disallow: /

User-agent: Zeus
Disallow: /

User-agent: RepoMonkey Bait & Tackle/v1.01
Disallow: /

User-agent: RepoMonkey
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: Openbot
Disallow: /

User-agent: URL Control
Disallow: /

User-agent: Zeus Link Scout
Disallow: /

User-agent: Zeus 32297 Webster Pro V2.9 Win32
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: EroCrawler
Disallow: /

User-agent: LinkScan/8.1a Unix
Disallow: /

User-agent: Keyword Density/0.9
Disallow: /

User-agent: Kenjin Spider
Disallow: /

User-agent: Iron33/1.0.2
Disallow: /

User-agent: Bookmark search tool
Disallow: /

User-agent: GetRight/4.2
Disallow: /

User-agent: FairAd Client
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: Aqua_Products
Disallow: /

User-agent: Radiation Retriever 1.1
Disallow: /

User-agent: Flaming AttackBot
Disallow: /

User-agent: Oracle Ultra Search
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: PerMan
Disallow: /

User-agent: searchpreview
Disallow: /

User-agent: ActiveTouristBot
Disallow: /

User-agent: USyd-NLP-Spider
Disallow: /

User-agent: RAMPyBot
Disallow: /

User-agent: MMCrawler
Disallow: /

User-agent: FDSE
Disallow: /

User-agent: FurlBot
Disallow: /

User-agent: MQbot
Disallow: /

User-agent: envolk
Disallow: /

User-agent: LapozzBot
Disallow: /

User-agent: aipbot
Disallow: /

User-agent: Shim-Crawler
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: MyEngines-Bot
Disallow: /

User-Agent: KFSW-Bot
Disallow: /

User-agent: NimbleCrawler
Disallow: /

User-agent: genieBot
Disallow: /

User-agent: MouseBOT
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: SBIder
Disallow: /

User-agent: stalker
Disallow: /

User-agent: DataSpear
Disallow: /

User-agent: thumbshots-de-bot
Disallow: /

User-agent: LocalcomBot
Disallow: /

User-agent: Findexa
Disallow: /

User-agent: iCCrawler
Disallow: /

User-agent: ObjectsSearch
Disallow: /

User-agent: KnowItAll
Disallow: /

User-agent: yacy
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: dcbspider
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: cfetch
Disallow: /

User-agent: FindLinks
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: gonzo1
Disallow: /

User-agent: gonzo2
Disallow: /

User-agent: gonzo1P
Disallow: /

User-agent: gonzo2P
Disallow: /

User-agent: Moni
Disallow: /

User-agent: Georgios
Disallow: /

User-agent: CydralSpider
Disallow: /

User-agent: HooWWWer
Disallow: /

User-agent: JemmaTheTourist
Disallow: /

User-agent: btbot
Disallow: /

User-agent: Yahoo-MMAudVid
Disallow: /

User-agent: IRLbot
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: Amfibibot
Disallow: /

User-agent: gridBOT
Disallow: /

User-agent: sna
Disallow: /

User-agent: TAMU_CS_IRL_CRAWLER
Disallow: /

User-agent: NPT
Disallow: /

User-agent: BruinBot
Disallow: /

User-agent: ZipppBot
Disallow: /

User-agent: Jetbot
Disallow: /

User-agent: Molbsy
Disallow: /

User-agent: PhpDig
Disallow: /

User-agent: mirago
Disallow:

User-agent: GoForIt.com
Disallow: /

User-agent: GoForIt
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: larbin
Disallow: /

User-agent: appie
Disallow: /

User-agent: Yahoo-MMCrawler
Disallow: /

User-agent: libwww
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: sohu-search
Disallow: /

User-agent: heritrix
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: linko
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: RPT-HTTPClient
Disallow: /

User-agent: Dumbot
Disallow: /

User-agent: Cowbot
Disallow: /

User-agent: SuperGet
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Szukacz
Disallow: /

User-agent: antibot
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: IconSurf
Disallow: /

User-agent: Speedy
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: TutorGig
Disallow: /

User-agent: Searchspider
Disallow: /

User-agent: Lachesis
Disallow: /

User-agent: VSE/1.0
Disallow: /

User-agent: QuepasaCreep
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Haste
Disallow: /

User-agent: NetResearchServer
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: NutchOrg
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: Generic
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: Ultraseek
Disallow: /

User-agent: Spinne
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: :robot
Disallow: /

User-agent: Spider_ Monkey
Disallow: /

User-agent: IXE Crawler
Disallow: /

User-agent: CoolBot
Disallow: /

User-agent: Zao
Disallow: /