From 2d8286c5df9207c0eb79c6e12846fd06301daef4 Mon Sep 17 00:00:00 2001
From: Stefan Verhoeven <stefan.verhoeven@gmail.com>
Date: Thu, 13 Apr 2017 23:54:00 +0200
Subject: [PATCH] Able to fill internal geocoder tables, but get 'too many
 source rows' error during georeference analysis of countries.

---
 Dockerfile            |  1 +
 README.md             |  5 ++++-
 create_dev_user       |  7 -------
 geocoder.sh           | 17 +++++++++++++++--
 geocoder_server.sql   |  9 ++++++++-
 setup_organization.sh | 12 ------------
 6 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 19e537f..3b78cd8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -220,6 +220,7 @@ ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
 ADD ./config/varnish.vcl /etc/varnish.vcl
 ADD ./geocoder.sh /cartodb/script/geocoder.sh
 ADD ./geocoder_server.sql /cartodb/script/geocoder_server.sql
+ADD ./fill_geocoder_server.sh /cartodb/script/fill_geocoder_server.sh
 ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \
     /opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \
diff --git a/README.md b/README.md
index 8a58712..327d49b 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,9 @@ The chosen hostname should also resolve to an IP adress of the machine where the
 Geocoder
 --------
 
+The external geocoders like heremaps or mapzen have dummy api keys and do not work.
+No attempts haven been made or will be made in this Docker image to get the external geocoders to work.
+
 The internal geocoder is configured, but contains no data inside the image.
 
 To fill the internal geocoder run
@@ -58,4 +61,4 @@ docker exec -ti <carto docker container id> bash -c /cartodb/script/fill_geocode
 ```
 
 This will run the scripts described at https://github.com/CartoDB/data-services/tree/master/geocoder
-It will use at least require 10Gb of diskspace to download the dumps and import them.
+It will use at least require 5.7+7.8Gb of diskspace to download the dumps and import them.
diff --git a/create_dev_user b/create_dev_user
index 46229e1..1ea574a 100755
--- a/create_dev_user
+++ b/create_dev_user
@@ -33,10 +33,3 @@ rake cartodb:db:set_user_account_type["${SUBDOMAIN}",'[DEDICATED]']
 
 # Set dataservices server
 rake cartodb:db:configure_geocoder_extension_for_non_org_users[$SUBDOMAIN]
-
-USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
-psql -U postgres $USER_DB < /cartodb/script/geocoder_client.sql
-echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
-echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
-GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -A -U postgres -t carto_db_development`
-echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $USER_DB
diff --git a/geocoder.sh b/geocoder.sh
index 2c1cbec..29b4dfd 100644
--- a/geocoder.sh
+++ b/geocoder.sh
@@ -1,3 +1,4 @@
+cd /cartodb
 
 rake cartodb:db:create_user --trace SUBDOMAIN="geocoder" \
 	PASSWORD="pass1234" ADMIN_PASSWORD="pass1234" \
@@ -11,6 +12,18 @@ rake cartodb:db:set_user_quota[geocoder,102400]
 echo "--- Allowing unlimited tables creation"
 rake cartodb:db:set_unlimited_table_quota[geocoder]
 
-
-GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`    
+GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
 psql -U postgres $GEOCODER_DB < /cartodb/script/geocoder_server.sql
+
+# Setup dataservices client
+# dev user
+USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
+echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
+echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
+echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $USER_DB
+
+# example organization
+ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
+echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
+echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
+echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB
diff --git a/geocoder_server.sql b/geocoder_server.sql
index a4c80be..0305891 100644
--- a/geocoder_server.sql
+++ b/geocoder_server.sql
@@ -21,4 +21,11 @@ SELECT CDB_Conf_SetConf(
 SELECT CDB_Conf_SetConf(
     'server_conf',
     '{"environment": "development"}'
-);
\ No newline at end of file
+);
+
+SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}');
+
+-- dummy conf from https://github.com/CartoDB/dataservices-api/blob/master/server/extension/test/sql/00_install_test.sql
+SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
diff --git a/setup_organization.sh b/setup_organization.sh
index 04e3d18..1f1f0c4 100644
--- a/setup_organization.sh
+++ b/setup_organization.sh
@@ -1,6 +1,3 @@
-cd /cartodb
-source /usr/local/rvm/scripts/rvm
-
 ORGANIZATION_NAME="example"
 USERNAME="admin4example"
 EMAIL="admin@example.com"
@@ -11,12 +8,3 @@ rake cartodb:db:set_unlimited_table_quota["${USERNAME}"]
 rake cartodb:db:create_new_organization_with_owner ORGANIZATION_NAME="${ORGANIZATION_NAME}" USERNAME="${USERNAME}" ORGANIZATION_SEATS=100 ORGANIZATION_QUOTA=102400 ORGANIZATION_DISPLAY_NAME="${ORGANIZATION_NAME}"
 rake cartodb:db:set_organization_quota[$ORGANIZATION_NAME,5000]
 rake cartodb:db:configure_geocoder_extension_for_organizations[$ORGANIZATION_NAME]
-
-# Set dataservices server
-ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
-psql -U postgres $ORGANIZATION_DB < /cartodb/script/geocoder_client.sql
-echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
-echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
-GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -A -U postgres -t carto_db_development`
-echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB
-