Merge pull request #479 from CartoDB/improve-prepare_db

Improve prepare db

run_tests.sh

@@ -5,6 +5,7 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 OPT_COVERAGE=no      # run tests with coverage
+OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files
 
 export PGAPPNAME=cartodb_tiler_tester
 
@@ -73,6 +74,10 @@ while [ -n "$1" ]; do
                 OPT_CREATE_REDIS=no
                 shift
                 continue
+        elif test "$1" = "--no-sql-download"; then
+                OPT_DOWNLOAD_SQL=no
+                shift
+                continue
         elif test "$1" = "--with-coverage"; then
                 OPT_COVERAGE=yes
                 shift
@@ -113,6 +118,9 @@ fi
 if test x"$OPT_CREATE_REDIS" != xyes; then
   PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
 fi
+if test x"$OPT_DOWNLOAD_SQL" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --no-sql-download"
+fi
 
 echo "Preparing the environment"
 cd ${BASEDIR}/test/support

test/support/prepare_db.sh

@@ -12,6 +12,7 @@
 
 PREPARE_REDIS=yes
 PREPARE_PGSQL=yes
+DOWNLOAD_SQL_FILES=yes
 
 while [ -n "$1" ]; do
   if test "$1" = "--skip-pg"; then
@@ -20,6 +21,9 @@ while [ -n "$1" ]; do
   elif test "$1" = "--skip-redis"; then
     PREPARE_REDIS=no
     shift; continue
+  elif test "$1" = "--no-sql-download"; then
+    DOWNLOAD_SQL_FILES=no
+    shift; continue
   fi
 done
 
@@ -71,26 +75,30 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  curl -L -s https://raw.githubusercontent.com/CartoDB/camshaft/master/test/fixtures/cdb_analysis_catalog.sql -o sql/cdb_analysis_catalog.sql
-  cat sql/cdb_analysis_catalog.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  LOCAL_SQL_SCRIPTS='_CDB_QueryStatements windshaft.test gadm4 ported/populated_places_simple_reduced'
+  REMOTE_SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_AnalysisCatalog CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
-  cat sql/windshaft.test.sql sql/gadm4.sql |
-    sed "s/:PUBLICUSER/${PUBLICUSER}/" |
-    sed "s/:PUBLICPASS/${PUBLICPASS}/" |
-    sed "s/:TESTUSER/${TESTUSER}/" |
-    sed "s/:TESTPASS/${TESTPASS}/" |
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-
-  cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-
-  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
-  for i in ${SQL_SCRIPTS}
+  CURL_ARGS=""
+  for i in ${REMOTE_SQL_SCRIPTS}
   do
-    curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
-    cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
-        | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+    CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql "
   done
+  if test x"$DOWNLOAD_SQL_FILES" = xyes; then
+    echo "Downloading and updating: ${REMOTE_SQL_SCRIPTS}"
+    echo ${CURL_ARGS} | xargs curl -L -s
+  fi
 
+  ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}"
+  for i in ${ALL_SQL_SCRIPTS}
+  do
+    cat sql/${i}.sql |
+      sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" |
+      sed "s/:PUBLICUSER/${PUBLICUSER}/" |
+      sed "s/:PUBLICPASS/${PUBLICPASS}/" |
+      sed "s/:TESTUSER/${TESTUSER}/" |
+      sed "s/:TESTPASS/${TESTPASS}/" |
+      PGOPTIONS='--client-min-messages=WARNING' psql -q -v ON_ERROR_STOP=1 ${TEST_DB} > /dev/null || exit 1
+  done
 fi
 
 if test x"$PREPARE_REDIS" = xyes; then
@@ -122,17 +130,3 @@ EOF
 fi
 
 echo "Finished preparing data. Ready to run tests"
-
-
-############################ WINDSHAFT TESTS ############################
-
-echo "...Configuring Windshaft test database"
-
-cat sql/ported/populated_places_simple_reduced.sql |
-    sed "s/:PUBLICUSER/${PUBLICUSER}/" |
-    sed "s/:PUBLICPASS/${PUBLICPASS}/" |
-    sed "s/:TESTUSER/${TESTUSER}/" |
-    sed "s/:TESTPASS/${TESTPASS}/" |
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-
-echo "...Test database configuration complete"