# Disallow all crawlers access to certain pages.
# One, you should be written Only Disallow-path like "Disallow /cgi-bin/".
# Two, you may write except-Allow-path like "Allow /cgi-bin/searchobject/". Then Allow-path should be written after Disallow-path.

User-agent: *