WebHost: Add robots.txt to WebHost (#3157)

* Add a `robots.txt` file to prevent crawlers from scraping the site

* Added `ASSET_RIGHTS` entry to config.yaml to control whether `/robots.txt` is served or not

* Always import robots.py, determine config in route function

* Finish writing a comment

* Remove unnecessary redundant import and config

docs/webhost configuration sample.yaml

@@ -1,4 +1,4 @@
-# This is a sample configuration for the Web host.  
+# This is a sample configuration for the Web host.
 # If you wish to change any of these,  rename this file to config.yaml
 # Default values are shown here.  Uncomment and change the values as desired.
 
@@ -25,7 +25,7 @@
 
 # Secret key used to determine important things like cookie authentication of room/seed page ownership.
 # If you wish to deploy, uncomment the following line and set it to something not easily guessable.
-# SECRET_KEY: "Your secret key here" 
+# SECRET_KEY: "Your secret key here"
 
 # TODO
 #JOB_THRESHOLD: 2
@@ -38,7 +38,7 @@
 #  provider: "sqlite"
 #  filename: "ap.db3" # This MUST be the ABSOLUTE PATH to the file.
 #  create_db: true
-  
+
 # Maximum number of players that are allowed to be rolled on the server. After this limit, one should roll locally and upload the results.
 #MAX_ROLL: 20
 
@@ -50,3 +50,7 @@
 
 # Host Address.  This is the address encoded into the patch that will be used for client auto-connect.
 #HOST_ADDRESS: archipelago.gg
+
+# Asset redistribution rights.  If true, the host affirms they have been given explicit permission to redistribute
+# the proprietary assets in WebHostLib
+#ASSET_RIGHTS: false