Subversion Repositories Misc

#!/usr/bin/env python

# forgottenislanderbot - makes a DSL coverage map from Bell Aliant's website.

# Copyright (c) 2010 Art Ortenburger

#

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; either version 2

# of the License, or any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

#

# fibmod.py

# by Art Ortenburger IV <utrrrongeeb a users , sf , net>

# written 2010-01-14 -

#

# forgottenislanderbot -- shared module functions

#

# Provides reusable generic functions to

# forgottenislanderbot components.

#

# ToDo:

# @77 == att == attn == attention == serious idea, better consider

#

# O crawl: try blocks everywhere, so won't crash after 18h stretch w/o commit

# X crawl: try, and recover from dead httpconnection

# O dbutil --list

# O dbutil --stats: remove dslflags, or bring them back?

# O *: comments, licenseblock, new headers

# O *: fix globals, improve code

# _ *: write full documentation for users, including howto/tutorial

# _ *: write bat/bash wrapper scripts

# X *: implement DIP http advice, try gzip (gzip didn't work)

# O crawl: put in routine long kill pause, db commit

# O *: move dslflags routines to processDB -- many advantages, & floatize delay

# _ *: decide on dslflags maxrows mess

# _ dbutil: fix KML icons -- small, red for NODSL, blobs?

# O *: all @77 notes dealt with?

# ^ crawl: fix Linux print status bug [ pending test ]

# _ dbutil: NASA World Wind KML output support

# _ crawl: end-of-road support & options

# ----

# _ dbutil: KMZ output (factor of 20)

# _ *: custom SQL, lat/lon rect submaps

# _ *: BUG: piping output crashes, maybe from from sys.stdout.flush()?

# _ crawl: try taking out hc.connect() under high-speed; maybe make auto?

# _ *: deploy 'with' keyword

# _ *: make dynamic cmd-line flags with optparse (new 'fib' frontend)

# _ *: port to Python 3

# _ confirm all server responses are recognized

# _ dbutil --merging

# _ dbutil --strip

# _ dbutil --diff

#

import sys

import sqlite3

if __name__ == "__main__":

    print sys.argv[0],"""(forgottenislanderbot shared module)    Version 0.1

Copyright (c) 2010 Art Ortenburger. Licensed under the GNU GPL; no warranty."""

def processDB(sqlcur,actionfunc,dslFlags,maxrows,sparsity,delay,excludecities):

    """ Loops through SQlite database, selecting rows

    based on parameters, and calling the specified function

    for each selected row.

    sqlcur:     sqlite3.Cursor to civic address database from fib-cadb2sql.py

    actionfunc: function to call for selected addresses.

                Must accept these parameters:

                sqlite3.Cursor

                town (string)

                road (string)

                civic number (int)

                seconds to pause between rows (float)

                dsl status of address (int)

                longitude (float)

                latitude (float)

    dslFlags:   4 y/n charbools in string sequence, OR

                4 bools in a list.

                Whether to call actionfunc() for rows

                in the EMPTY, NODSL, BASIC, or ULTRA

                categories. Example: "nyyy".

    maxrows:   bool; whether to keep rigid sparseness based on dslFlags,

                or not. True is desirable for dbutil --kml, where you

                want many results, but False is important for the crawler,

                to maintain high sparseness.

    sparsity:   int; minimum civic-number difference between

                houses to skip. For every house, enter 1.

                For about one house per road, enter 10000.

    delay:      float; seconds to pause between selected addresses.

                Delaying is actually done, if it is, by actionfunc.

    excludecities:  bool; whether to exclude the two major cities

                    from the selected rows or not."""

    # first, handle dslflags:

    dslflags = [ False, False, False, False ] # real dslflags for use

    dfl = len(dslFlags) # dsl flags length

    if dfl >= 4: # 4 or more? only use first 4.

        for i in range(4):

            dslflags[i] = char2Bool(dslFlags[i])

    elif dfl == 3: # hopefully this won't be used -- EMPTY, NODSL, & (BASIC+ULTRA)

        for i in range(3):

            dslflags[i] = char2Bool(dslFlags[i])

            dslflags[3] = dslflags[2]

    elif dfl == 2: # hopefully to be ignored -- EMPTY, KNOWN

        dslflags[0] = char2Bool(dslFlags[0])

        dslflags[1] = dslflags[2] = dslflags[3] = char2Bool(dslFlags[1])

    elif dfl == 1: # hopefully to be ignored -- EMPTY [KNOWN = True]

        dslflags = [char2Bool(dslFlags[0]), True, True, True]

    else: # easy -- all True

        dslflags = [ True, True, True, True ]

    sqlcur.execute("select town,road,civic,dsl,long,lat from dsl order by town,road,civic")

    lasttown = lastroad = town = road = ""

    lastcivic = civic = 0

    # this block will take a lot of ram due to fetchall() rather than fetchone().

    # Shouldn't be hard to fix if needed, though. Current-db ram req: 6 MB max?

    for cca in sqlcur.fetchall(): # current civic address

        town,road,civic = cca[0],cca[1],int(cca[2])# cast here is importent

        # skip cities -- currently only Ch'town and Summerside -- if requested

        if excludecities and ( town == "CHARLOTTETOWN" or town == "SUMMERSIDE" ):

            continue

        # apply the dslflags. If status doesn't match, check maxcalls

        # on whether to set last___ vars, to determine whether a

        # maximum number of addresses will be gotten, or a regular

        # sparseness will be maintained.

        sii = int(cca[3]) # status index int

        if sii < 0:

            sii = 0

        if not dslflags[sii]:

            if not maxrows:

                lasttown,lastroad,lastcivic = cca[0],cca[1],int(cca[2])

            continue;

        try: # loop must not crash

            # on the same road, make sure to skip houses for sparseness

            if town == lasttown and road == lastroad:

                if (civic - lastcivic) >= sparsity: # sparseness check

                    actionfunc(sqlcur,town,road,civic,delay,int(cca[3]),float(cca[4]),float(cca[5]))

                # if sparseness check failed, don't do anything

            else: # on a new road, always get the first house

                actionfunc(sqlcur,town,road,civic,delay,int(cca[3]),float(cca[4]),float(cca[5]))

        except Exception, errdet:

            print "> error < :",errdet

        # not sure how to get the last easily.

        # @77 Could make last(lat|lon|stat) vars, and a postcheck....

        lasttown,lastroad,lastcivic = cca[0],cca[1],int(cca[2])

def char2Bool(inchar):

    """ Returns a bool from a y/n char param.

        y:Y: True; anything else: False

        However, if inchar is a bool, it will be returned as is.

        param sequence can be any length,

        but only the first char gets evaluated"""

    if inchar == True or inchar == False:

        return inchar

    elif inchar[0] == 'y' or inchar[0] == 'Y':

        return True

    else:

        return False