cartodb/docker-cartojw
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Able to fill internal geocoder tables, but get 'too many source rows' error during georeference analysis of countries.

Browse Source
This commit is contained in:
Stefan Verhoeven 2017-04-13 23:54:00 +02:00
parent 723dc41d65
commit 2d8286c5df
6 changed files with 28 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@ -220,6 +220,7 @@ ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
ADD ./config/varnish.vcl /etc/varnish.vcl
ADD ./geocoder.sh /cartodb/script/geocoder.sh
ADD ./geocoder_server.sql /cartodb/script/geocoder_server.sql
ADD ./fill_geocoder_server.sh /cartodb/script/fill_geocoder_server.sh
ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \
/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \

5
README.md
View File

@ -50,6 +50,9 @@ The chosen hostname should also resolve to an IP adress of the machine where the
Geocoder
--------
The external geocoders like heremaps or mapzen have dummy api keys and do not work.
No attempts haven been made or will be made in this Docker image to get the external geocoders to work.
The internal geocoder is configured, but contains no data inside the image.
To fill the internal geocoder run
@ -58,4 +61,4 @@ docker exec -ti <carto docker container id> bash -c /cartodb/script/fill_geocode
```
This will run the scripts described at https://github.com/CartoDB/data-services/tree/master/geocoder
It will use at least require 10Gb of diskspace to download the dumps and import them.
It will use at least require 5.7+7.8Gb of diskspace to download the dumps and import them.

7
create_dev_user
View File

@ -33,10 +33,3 @@ rake cartodb:db:set_user_account_type["${SUBDOMAIN}",'[DEDICATED]']
# Set dataservices server
rake cartodb:db:configure_geocoder_extension_for_non_org_users[$SUBDOMAIN]
USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
psql -U postgres $USER_DB < /cartodb/script/geocoder_client.sql
echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -A -U postgres -t carto_db_development`
echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $USER_DB

17
geocoder.sh
View File

@ -1,3 +1,4 @@
cd /cartodb
rake cartodb:db:create_user --trace SUBDOMAIN="geocoder" \
PASSWORD="pass1234" ADMIN_PASSWORD="pass1234" \
@ -11,6 +12,18 @@ rake cartodb:db:set_user_quota[geocoder,102400]
echo "--- Allowing unlimited tables creation"
rake cartodb:db:set_unlimited_table_quota[geocoder]
GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
psql -U postgres $GEOCODER_DB < /cartodb/script/geocoder_server.sql
# Setup dataservices client
# dev user
USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $USER_DB
# example organization
ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB

9
geocoder_server.sql
View File

@ -21,4 +21,11 @@ SELECT CDB_Conf_SetConf(
SELECT CDB_Conf_SetConf(
'server_conf',
'{"environment": "development"}'
);
);
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}');
-- dummy conf from https://github.com/CartoDB/dataservices-api/blob/master/server/extension/test/sql/00_install_test.sql
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');

12
setup_organization.sh
View File

@ -1,6 +1,3 @@
cd /cartodb
source /usr/local/rvm/scripts/rvm
ORGANIZATION_NAME="example"
USERNAME="admin4example"
EMAIL="admin@example.com"
@ -11,12 +8,3 @@ rake cartodb:db:set_unlimited_table_quota["${USERNAME}"]
rake cartodb:db:create_new_organization_with_owner ORGANIZATION_NAME="${ORGANIZATION_NAME}" USERNAME="${USERNAME}" ORGANIZATION_SEATS=100 ORGANIZATION_QUOTA=102400 ORGANIZATION_DISPLAY_NAME="${ORGANIZATION_NAME}"
rake cartodb:db:set_organization_quota[$ORGANIZATION_NAME,5000]
rake cartodb:db:configure_geocoder_extension_for_organizations[$ORGANIZATION_NAME]
# Set dataservices server
ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
psql -U postgres $ORGANIZATION_DB < /cartodb/script/geocoder_client.sql
echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -A -U postgres -t carto_db_development`
echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB