observatory-extension/scripts/generate_fixtures.py

from sqldumpr import Dumpr

def get_tablename_query(column_id, boundary_id, timespan):
    """
        given a column_id, boundary-id (us.census.tiger.block_group), and
        timespan, give back the current table hash from the data observatory
    """
    q = """
        SELECT t.tablename, geoid_ct.colname colname
        FROM obs_table t,
             obs_column_table geoid_ct,
             obs_column_table data_ct
        WHERE
             t.id = geoid_ct.table_id AND
             t.id = data_ct.table_id AND
             geoid_ct.column_id =
        (SELECT source_id
         FROM obs_column_to_column
         WHERE target_id = '{boundary_id}'
         AND reltype = 'geom_ref'
        ) AND
        data_ct.column_id = '{column_id}' AND
        timespan = '{timespan}'
    """.replace('\n','')

    return q.format(column_id=column_id,
                    boundary_id=boundary_id,
                    timespan=timespan)

def select_star(tablename):
    return "SELECT * FROM {}".format(tablename)

cdb = Dumpr('observatory.cartodb.com','')

metadata = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag', 'obs_tag', 'obs_column_to_column']

fixtures = [
    ('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'),
    ('us.census.tiger.block_group', 'us.census.tiger.block_group', '2014'),
    ('us.census.tiger.zcta5', 'us.census.tiger.zcta5', '2014'),
    ('us.census.tiger.county', 'us.census.tiger.county', '2014'),
    ('us.census.acs.B01001001', 'us.census.tiger.census_tract', '2009 - 2013'),
    ('us.census.acs.B01001001_quantile', 'us.census.tiger.census_tract', '2009 - 2013'),
    ('us.census.acs.B01001001', 'us.census.tiger.block_group', '2009 - 2013'),
    ('us.census.acs.B01001001', 'us.census.tiger.block_group', '2010 - 2014'),
    ('us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2009 - 2013'),
    ('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2014-01'),
    ('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2016-03'),
]

unique_tables = set()

for f in fixtures:
    column_id, boundary_id, timespan = f
    tablename_query = get_tablename_query(*f)
    resp = cdb.query(tablename_query).json()['rows'][0]
    tablename = resp['tablename']
    colname = resp['colname']
    table_colname = (tablename, colname, boundary_id, )
    if table_colname not in unique_tables:
        print table_colname
        unique_tables.add(table_colname)

print unique_tables

with open('src/pg/test/fixtures/load_fixtures.sql', 'w') as outfile:
    with open('src/pg/test/fixtures/drop_fixtures.sql', 'w') as dropfiles:
        outfile.write('SET client_min_messages TO WARNING;\n\set ECHO none\n')
        dropfiles.write('SET client_min_messages TO WARNING;\n\set ECHO none\n')
        for tablename in metadata:
            cdb.dump(select_star(tablename), tablename, outfile, schema='observatory')
            dropfiles.write('DROP TABLE IF EXISTS observatory.{};\n'.format(tablename))
            print tablename

        for tablename, colname, boundary_id in unique_tables:
            if 'zcta5' in boundary_id:
                where = '11%'
            else:
                where = '36047%'
            print ' '.join([select_star(tablename), "WHERE {} LIKE '{}'".format(colname, where)])
            cdb.dump(' '.join([select_star(tablename), "WHERE {} LIKE '{}'".format(colname, where)]),
                     tablename, outfile, schema='observatory')
            dropfiles.write('DROP TABLE IF EXISTS observatory.{};\n'.format(tablename))