# "Any empty [Disallow] value, indicates that all URLs can be retrieved. # At least one Disallow field needs to be present in a record." User-agent: * Crawl-delay: 100 Disallow: /deu.mas-ad.dat Disallow: /deu.mas-g.dat Disallow: /deu.mas-n.dat Disallow: /deu.neu-g.dat Disallow: /deu.neu-nad.dat Disallow: /deu.plu-d.dat Disallow: /deu.plu-nag.dat Disallow: /files/