Compare with Previous | Blame | View Log
#!/usr/bin/env python # forgottenislanderbot - makes a DSL coverage map from Bell Aliant's website. # Copyright (c) 2010 Art Ortenburger # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # fibmod.py # by Art Ortenburger IV <utrrrongeeb a users , sf , net> # written 2010-01-14 - # # forgottenislanderbot -- shared module functions # # Provides reusable generic functions to # forgottenislanderbot components. # # ToDo: # @77 == att == attn == attention == serious idea, better consider # # O crawl: try blocks everywhere, so won't crash after 18h stretch w/o commit # X crawl: try, and recover from dead httpconnection # O dbutil --list # O dbutil --stats: remove dslflags, or bring them back? # O *: comments, licenseblock, new headers # O *: fix globals, improve code # _ *: write full documentation for users, including howto/tutorial # _ *: write bat/bash wrapper scripts # X *: implement DIP http advice, try gzip (gzip didn't work) # O crawl: put in routine long kill pause, db commit # O *: move dslflags routines to processDB -- many advantages, & floatize delay # _ *: decide on dslflags maxrows mess # _ dbutil: fix KML icons -- small, red for NODSL, blobs? # O *: all @77 notes dealt with? # ^ crawl: fix Linux print status bug [ pending test ] # _ dbutil: NASA World Wind KML output support # _ crawl: end-of-road support & options # ---- # _ dbutil: KMZ output (factor of 20) # _ *: custom SQL, lat/lon rect submaps # _ *: BUG: piping output crashes, maybe from from sys.stdout.flush()? # _ crawl: try taking out hc.connect() under high-speed; maybe make auto? # _ *: deploy 'with' keyword # _ *: make dynamic cmd-line flags with optparse (new 'fib' frontend) # _ *: port to Python 3 # _ confirm all server responses are recognized # _ dbutil --merging # _ dbutil --strip # _ dbutil --diff # import sys import sqlite3 if __name__ == "__main__": print sys.argv[0],"""(forgottenislanderbot shared module) Version 0.1 Copyright (c) 2010 Art Ortenburger. Licensed under the GNU GPL; no warranty.""" def processDB(sqlcur,actionfunc,dslFlags,maxrows,sparsity,delay,excludecities): """ Loops through SQlite database, selecting rows based on parameters, and calling the specified function for each selected row. sqlcur: sqlite3.Cursor to civic address database from fib-cadb2sql.py actionfunc: function to call for selected addresses. Must accept these parameters: sqlite3.Cursor town (string) road (string) civic number (int) seconds to pause between rows (float) dsl status of address (int) longitude (float) latitude (float) dslFlags: 4 y/n charbools in string sequence, OR 4 bools in a list. Whether to call actionfunc() for rows in the EMPTY, NODSL, BASIC, or ULTRA categories. Example: "nyyy". maxrows: bool; whether to keep rigid sparseness based on dslFlags, or not. True is desirable for dbutil --kml, where you want many results, but False is important for the crawler, to maintain high sparseness. sparsity: int; minimum civic-number difference between houses to skip. For every house, enter 1. For about one house per road, enter 10000. delay: float; seconds to pause between selected addresses. Delaying is actually done, if it is, by actionfunc. excludecities: bool; whether to exclude the two major cities from the selected rows or not.""" # first, handle dslflags: dslflags = [ False, False, False, False ] # real dslflags for use dfl = len(dslFlags) # dsl flags length if dfl >= 4: # 4 or more? only use first 4. for i in range(4): dslflags[i] = char2Bool(dslFlags[i]) elif dfl == 3: # hopefully this won't be used -- EMPTY, NODSL, & (BASIC+ULTRA) for i in range(3): dslflags[i] = char2Bool(dslFlags[i]) dslflags[3] = dslflags[2] elif dfl == 2: # hopefully to be ignored -- EMPTY, KNOWN dslflags[0] = char2Bool(dslFlags[0]) dslflags[1] = dslflags[2] = dslflags[3] = char2Bool(dslFlags[1]) elif dfl == 1: # hopefully to be ignored -- EMPTY [KNOWN = True] dslflags = [char2Bool(dslFlags[0]), True, True, True] else: # easy -- all True dslflags = [ True, True, True, True ] sqlcur.execute("select town,road,civic,dsl,long,lat from dsl order by town,road,civic") lasttown = lastroad = town = road = "" lastcivic = civic = 0 # this block will take a lot of ram due to fetchall() rather than fetchone(). # Shouldn't be hard to fix if needed, though. Current-db ram req: 6 MB max? for cca in sqlcur.fetchall(): # current civic address town,road,civic = cca[0],cca[1],int(cca[2])# cast here is importent # skip cities -- currently only Ch'town and Summerside -- if requested if excludecities and ( town == "CHARLOTTETOWN" or town == "SUMMERSIDE" ): continue # apply the dslflags. If status doesn't match, check maxcalls # on whether to set last___ vars, to determine whether a # maximum number of addresses will be gotten, or a regular # sparseness will be maintained. sii = int(cca[3]) # status index int if sii < 0: sii = 0 if not dslflags[sii]: if not maxrows: lasttown,lastroad,lastcivic = cca[0],cca[1],int(cca[2]) continue; try: # loop must not crash # on the same road, make sure to skip houses for sparseness if town == lasttown and road == lastroad: if (civic - lastcivic) >= sparsity: # sparseness check actionfunc(sqlcur,town,road,civic,delay,int(cca[3]),float(cca[4]),float(cca[5])) # if sparseness check failed, don't do anything else: # on a new road, always get the first house actionfunc(sqlcur,town,road,civic,delay,int(cca[3]),float(cca[4]),float(cca[5])) except Exception, errdet: print "> error < :",errdet # not sure how to get the last easily. # @77 Could make last(lat|lon|stat) vars, and a postcheck.... lasttown,lastroad,lastcivic = cca[0],cca[1],int(cca[2]) def char2Bool(inchar): """ Returns a bool from a y/n char param. y:Y: True; anything else: False However, if inchar is a bool, it will be returned as is. param sequence can be any length, but only the first char gets evaluated""" if inchar == True or inchar == False: return inchar elif inchar[0] == 'y' or inchar[0] == 'Y': return True else: return False