From 86e8cedfab557fbe7cf994983cf5751f10ccc267 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 31 May 2016 15:17:41 +0200 Subject: [PATCH 1/4] All remote sql files together --- test/support/prepare_db.sh | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index ceac7c78..ed93e021 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -71,19 +71,7 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" - curl -L -s https://raw.githubusercontent.com/CartoDB/camshaft/master/test/fixtures/cdb_analysis_catalog.sql -o sql/cdb_analysis_catalog.sql - cat sql/cdb_analysis_catalog.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - - cat sql/windshaft.test.sql sql/gadm4.sql | - sed "s/:PUBLICUSER/${PUBLICUSER}/" | - sed "s/:PUBLICPASS/${PUBLICPASS}/" | - sed "s/:TESTUSER/${TESTUSER}/" | - sed "s/:TESTPASS/${TESTPASS}/" | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - - cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - - SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_AnalysisCatalog CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' for i in ${SQL_SCRIPTS} do curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql @@ -91,6 +79,12 @@ if test x"$PREPARE_PGSQL" = xyes; then | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 done + cat sql/_CDB_QueryStatements.sql sql/windshaft.test.sql sql/gadm4.sql sql/ported/populated_places_simple_reduced.sql | + sed "s/:PUBLICUSER/${PUBLICUSER}/" | + sed "s/:PUBLICPASS/${PUBLICPASS}/" | + sed "s/:TESTUSER/${TESTUSER}/" | + sed "s/:TESTPASS/${TESTPASS}/" | + psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 fi if test x"$PREPARE_REDIS" = xyes; then @@ -122,17 +116,3 @@ EOF fi echo "Finished preparing data. Ready to run tests" - - -############################ WINDSHAFT TESTS ############################ - -echo "...Configuring Windshaft test database" - -cat sql/ported/populated_places_simple_reduced.sql | - sed "s/:PUBLICUSER/${PUBLICUSER}/" | - sed "s/:PUBLICPASS/${PUBLICPASS}/" | - sed "s/:TESTUSER/${TESTUSER}/" | - sed "s/:TESTPASS/${TESTPASS}/" | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - -echo "...Test database configuration complete" From 989df4a8a47c912895a49ef8f717f4a5147a8cd4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 31 May 2016 16:42:42 +0200 Subject: [PATCH 2/4] curl over all remote files at the same time --- test/support/prepare_db.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index ed93e021..67ab5958 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -71,20 +71,28 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" - SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_AnalysisCatalog CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' - for i in ${SQL_SCRIPTS} + echo '\c' > .remote-sql-urls.txt + REMOTE_SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_AnalysisCatalog CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + for i in ${REMOTE_SQL_SCRIPTS} do - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql - cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \ - | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + echo "\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql" >> .remote-sql-urls.txt done + cat .remote-sql-urls.txt | xargs curl -L -s + rm .remote-sql-urls.txt - cat sql/_CDB_QueryStatements.sql sql/windshaft.test.sql sql/gadm4.sql sql/ported/populated_places_simple_reduced.sql | - sed "s/:PUBLICUSER/${PUBLICUSER}/" | - sed "s/:PUBLICPASS/${PUBLICPASS}/" | - sed "s/:TESTUSER/${TESTUSER}/" | - sed "s/:TESTPASS/${TESTPASS}/" | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + LOCAL_SQL_SCRIPTS='_CDB_QueryStatements windshaft.test gadm4 ported/populated_places_simple_reduced' + + ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" + for i in ${ALL_SQL_SCRIPTS} + do + cat sql/${i}.sql | + sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | + sed "s/:PUBLICUSER/${PUBLICUSER}/" | + sed "s/:PUBLICPASS/${PUBLICPASS}/" | + sed "s/:TESTUSER/${TESTUSER}/" | + sed "s/:TESTPASS/${TESTPASS}/" | + PGOPTIONS='--client-min-messages=WARNING' psql -q -v ON_ERROR_STOP=1 ${TEST_DB} > /dev/null || exit 1 + done fi if test x"$PREPARE_REDIS" = xyes; then From 6644711969a99c50e55b61dda0d4fad44074e3b1 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 31 May 2016 16:46:57 +0200 Subject: [PATCH 3/4] Use a variable instead of a tmp file --- test/support/prepare_db.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 67ab5958..a0cfa621 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -71,16 +71,15 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" - echo '\c' > .remote-sql-urls.txt + LOCAL_SQL_SCRIPTS='_CDB_QueryStatements windshaft.test gadm4 ported/populated_places_simple_reduced' REMOTE_SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_AnalysisCatalog CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + + CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} do - echo "\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql" >> .remote-sql-urls.txt + CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql " done - cat .remote-sql-urls.txt | xargs curl -L -s - rm .remote-sql-urls.txt - - LOCAL_SQL_SCRIPTS='_CDB_QueryStatements windshaft.test gadm4 ported/populated_places_simple_reduced' + echo ${CURL_ARGS} | xargs curl -L -s ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" for i in ${ALL_SQL_SCRIPTS} From 22f3a54fbf12fd3fd3690f4cdeab69e7b552c4d8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 31 May 2016 16:57:28 +0200 Subject: [PATCH 4/4] Option to skip sql files download --- run_tests.sh | 8 ++++++++ test/support/prepare_db.sh | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index e63a0492..92e8621b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,6 +5,7 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment OPT_DROP_REDIS=yes # drop the redis test environment OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment OPT_COVERAGE=no # run tests with coverage +OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files export PGAPPNAME=cartodb_tiler_tester @@ -73,6 +74,10 @@ while [ -n "$1" ]; do OPT_CREATE_REDIS=no shift continue + elif test "$1" = "--no-sql-download"; then + OPT_DOWNLOAD_SQL=no + shift + continue elif test "$1" = "--with-coverage"; then OPT_COVERAGE=yes shift @@ -113,6 +118,9 @@ fi if test x"$OPT_CREATE_REDIS" != xyes; then PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis" fi +if test x"$OPT_DOWNLOAD_SQL" != xyes; then + PREPARE_DB_OPTS="$PREPARE_DB_OPTS --no-sql-download" +fi echo "Preparing the environment" cd ${BASEDIR}/test/support diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index a0cfa621..dbf27d35 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -12,6 +12,7 @@ PREPARE_REDIS=yes PREPARE_PGSQL=yes +DOWNLOAD_SQL_FILES=yes while [ -n "$1" ]; do if test "$1" = "--skip-pg"; then @@ -20,6 +21,9 @@ while [ -n "$1" ]; do elif test "$1" = "--skip-redis"; then PREPARE_REDIS=no shift; continue + elif test "$1" = "--no-sql-download"; then + DOWNLOAD_SQL_FILES=no + shift; continue fi done @@ -79,7 +83,10 @@ if test x"$PREPARE_PGSQL" = xyes; then do CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql " done - echo ${CURL_ARGS} | xargs curl -L -s + if test x"$DOWNLOAD_SQL_FILES" = xyes; then + echo "Downloading and updating: ${REMOTE_SQL_SCRIPTS}" + echo ${CURL_ARGS} | xargs curl -L -s + fi ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" for i in ${ALL_SQL_SCRIPTS}