Compare with Previous | Blame | View Log
# forgottenislanderbot -- minimum instructions # not intended to be used as a shell script # download civic addresses from http://www.gov.pe.ca/civicaddress/ wget -O "queens.tsv" "http://www.gov.pe.ca/civicaddress/download/dodownload.php3?county=QUN&street_nm=&downloadformat=tab&downloadfields%5B%5D=street_no&downloadfields%5B%5D=street_nm&downloadfields%5B%5D=comm_nm&downloadfields%5B%5D=apt_no&downloadfields%5B%5D=county&downloadfields%5B%5D=latitude&downloadfields%5B%5D=longitude&downloadfields%5B%5D=police&downloadfields%5B%5D=fire&downloadfields%5B%5D=ems" wget -O "prince.tsv" "http://www.gov.pe.ca/civicaddress/download/dodownload.php3?county=PRN&street_nm=&downloadformat=tab&downloadfields%5B%5D=street_no&downloadfields%5B%5D=street_nm&downloadfields%5B%5D=comm_nm&downloadfields%5B%5D=apt_no&downloadfields%5B%5D=county&downloadfields%5B%5D=latitude&downloadfields%5B%5D=longitude&downloadfields%5B%5D=police&downloadfields%5B%5D=fire&downloadfields%5B%5D=ems" wget -O "kings.tsv" "http://www.gov.pe.ca/civicaddress/download/dodownload.php3?county=KNS&street_nm=&downloadformat=tab&downloadfields%5B%5D=street_no&downloadfields%5B%5D=street_nm&downloadfields%5B%5D=comm_nm&downloadfields%5B%5D=apt_no&downloadfields%5B%5D=county&downloadfields%5B%5D=latitude&downloadfields%5B%5D=longitude&downloadfields%5B%5D=police&downloadfields%5B%5D=fire&downloadfields%5B%5D=ems" # make sqlite database from TSV files python fib-cadb2sql.py database.sqlite queens.tsv kings.tsv prince.tsv # run the bot. This will take 19 to 28 hours. # If they've put in a robots.txt, you may be asked to override it; that is your choice. # This will get every address on PEI. # It will wait one second (the second '1') between single-threaded requests. python fib-crawlbot.py database.sqlite 1 1 n # I assume this won't hurt sync # run it again -- this might get any addresses which failed with errors # the first time. Known addresses won't be checked again. python fib-crawlbot.py database.sqlite 1 1 n # find the totals -- the NODSL row is the interesting one... python fib.py --stats # (new version) if the database has a different name than 'database.sqlite', append --db=file.sqlite # older, deprecated version: python fib-dbutil.py database.sqlite --stats yyyy 1 n # make a few variations of the KML for Google Earth python fib.py --kml # default dslmask=nyyy, default sparsity=40 # use the alternate icons python fib.py --kml --dslmask=nynn --sparsity=1 --icontheme="icons/blob-*.png" --iconsize=0.5 --iconhotspot="16,16" --no-highlight # deprecated older version: python fib-dbutil.py database.sqlite --kml nyyy 100 n map-all-sparse.kml python fib-dbutil.py database.sqlite --kml nyyy 40 n map-all-dense.kml python fib-dbutil.py database.sqlite --kml nynn 100 n map-nodsl-sparse.kml python fib-dbutil.py database.sqlite --kml nynn 40 n map-nodsl-dense.kml # with current Google Maps icons: # EMPTY: white, NODSL: yellow, BASIC: cyan, ULTRA: green # please email me a copy of database.sqlite at this point