From 29438076289b5e0f88eae4608bddd376f8b857e2 Mon Sep 17 00:00:00 2001
From: zhongjin <zhongjin@onenetcom.cn>
Date: Thu, 11 May 2023 13:16:31 +0800
Subject: [PATCH] new

---
 cartodb_pgsql.sh       |  7 +++++++
 create_dev_user        | 42 ++++++++++++++++++++++++++++++++++++++++++
 fill_geocoder.sh       | 13 +++++++++++++
 geocoder.sh            | 40 ++++++++++++++++++++++++++++++++++++++++
 geocoder_server.sql    | 36 ++++++++++++++++++++++++++++++++++++
 setup_fdw.sh           | 20 ++++++++++++++++++++
 setup_organization.sh  | 19 +++++++++++++++++++
 startup.sh             | 35 +++++++++++++++++++++++++++++++++++
 sync_tables_trigger.sh |  8 ++++++++
 template_postgis.sh    | 18 ++++++++++++++++++
 10 files changed, 238 insertions(+)
 create mode 100755 cartodb_pgsql.sh
 create mode 100755 create_dev_user
 create mode 100644 fill_geocoder.sh
 create mode 100644 geocoder.sh
 create mode 100644 geocoder_server.sql
 create mode 100644 setup_fdw.sh
 create mode 100644 setup_organization.sh
 create mode 100755 startup.sh
 create mode 100644 sync_tables_trigger.sh
 create mode 100755 template_postgis.sh

diff --git a/cartodb_pgsql.sh b/cartodb_pgsql.sh
new file mode 100755
index 0000000..5a2d716
--- /dev/null
+++ b/cartodb_pgsql.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+#
+# Init script to success tests.
+#
+
+psql -c "CREATE EXTENSION postgis;"
+psql -c "CREATE EXTENSION cartodb;"
diff --git a/create_dev_user b/create_dev_user
new file mode 100755
index 0000000..6543e35
--- /dev/null
+++ b/create_dev_user
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+
+SUBDOMAIN="dev"
+PASSWORD="pass1234"
+ADMIN_PASSWORD="pass1234"
+EMAIL="dev@contoso.com"
+
+echo "--- Creating databases"
+bundle exec rake cartodb:db:setup
+
+echo "--- Create '${SUBDOMAIN}' user"
+bundle exec  rake cartodb:db:create_user --trace SUBDOMAIN="${SUBDOMAIN}" \
+	PASSWORD="${PASSWORD}" ADMIN_PASSWORD="${ADMIN_PASSWORD}" \
+	EMAIL="${EMAIL}"
+
+# # Update your quota to 100GB
+echo "--- Updating quota to 100GB"
+bundle exec  rake cartodb:db:set_user_quota["${SUBDOMAIN}",102400]
+
+# # Allow unlimited tables to be created
+echo "--- Allowing unlimited tables creation"
+bundle exec  rake cartodb:db:set_unlimited_table_quota["${SUBDOMAIN}"]
+
+# # Allow user to create private tables in addition to public
+echo "--- Allowing private tables creation"
+bundle exec  rake cartodb:db:set_user_private_tables_enabled["${SUBDOMAIN}",'true']
+
+# # Set the account type
+echo "--- Setting cartodb account type"
+bundle exec  rake cartodb:db:set_user_account_type["${SUBDOMAIN}",'[DEDICATED]']
+
+# Set dataservices server
+bundle exec  rake cartodb:db:configure_geocoder_extension_for_non_org_users[$SUBDOMAIN]
+
+# Set import limits
+bundle exec rake cartodb:set_custom_limits_for_user["${SUBDOMAIN}",10240000000,100000000,1]
+
+
+# Enable sync tables
+echo "UPDATE users SET sync_tables_enabled=true WHERE username='${SUBDOMAIN}'" | psql -U postgres -t carto_db_development
diff --git a/fill_geocoder.sh b/fill_geocoder.sh
new file mode 100644
index 0000000..8edd7d1
--- /dev/null
+++ b/fill_geocoder.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# See https://github.com/CartoDB/data-services/issues/228#issuecomment-280037353
+# Not run during Docker build phase as it would make the image too big
+cd /data-services/geocoder
+./geocoder_download_dumps
+GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
+./geocoder_restore_dump postgres $GEOCODER_DB db_dumps/*.sql
+rm -r db_dumps
+chmod +x geocoder_download_patches.sh geocoder_apply_patches.sh
+./geocoder_download_patches.sh
+./geocoder_apply_patches.sh postgres $GEOCODER_DB data_patches/*.sql
+rm -r data_patches
diff --git a/geocoder.sh b/geocoder.sh
new file mode 100644
index 0000000..7a711a7
--- /dev/null
+++ b/geocoder.sh
@@ -0,0 +1,40 @@
+cd /cartodb
+
+bundle exec  rake cartodb:db:create_user --trace SUBDOMAIN="geocoder" \
+	PASSWORD="pass1234" ADMIN_PASSWORD="pass1234" \
+	EMAIL="geocoder@contoso.com"
+
+# # Update your quota to 100GB
+echo "--- Updating quota to 100GB"
+bundle exec  rake cartodb:db:set_user_quota[geocoder,102400]
+
+# # Allow unlimited tables to be created
+echo "--- Allowing unlimited tables creation"
+bundle exec  rake cartodb:db:set_unlimited_table_quota[geocoder]
+
+GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
+psql -U postgres $GEOCODER_DB < /cartodb/script/geocoder_server.sql
+
+# Import observatory test dataset
+psql -U postgres -d $GEOCODER_DB -f /observatory-extension/src/pg/test/fixtures/load_fixtures.sql
+# Setup permissions for observatory
+psql -U postgres -d $GEOCODER_DB -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory VERSION 'dev'; COMMIT" -e
+psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO geocoder; COMMIT" -e
+psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder; COMMIT" -e
+psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO geocoder; COMMIT" -e
+psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA observatory TO geocoder; COMMIT" -e
+
+# Setup dataservices client
+# dev user
+USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
+echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
+echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
+echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB# } user=postgres\"}');" | psql -U postgres $USER_DB
+bundle exec  rake cartodb:services:set_user_quota['dev',geocoding,100000]
+
+# example organization
+ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
+echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
+echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
+echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB# } user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB
+bundle exec  rake cartodb:services:set_org_quota['example',geocoding,100000]
diff --git a/geocoder_server.sql b/geocoder_server.sql
new file mode 100644
index 0000000..5faa480
--- /dev/null
+++ b/geocoder_server.sql
@@ -0,0 +1,36 @@
+create extension cdb_geocoder;
+create extension plproxy;
+create extension observatory;
+create extension cdb_dataservices_server;
+create extension cdb_dataservices_client;
+
+SELECT CDB_Conf_SetConf(
+    'redis_metadata_config',
+    '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'
+);
+SELECT CDB_Conf_SetConf(
+    'redis_metrics_config',
+    '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'
+);
+
+SELECT CDB_Conf_SetConf(
+    'user_config',
+    '{"is_organization": false, "entity_name": "geocoder"}'
+);
+
+SELECT CDB_Conf_SetConf(
+    'server_conf',
+    '{"environment": "development"}'
+);
+
+SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}');
+
+-- dummy conf from https://github.com/CartoDB/dataservices-api/blob/master/server/extension/test/sql/00_install_test.sql
+SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
+SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
+SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
diff --git a/setup_fdw.sh b/setup_fdw.sh
new file mode 100644
index 0000000..4b72833
--- /dev/null
+++ b/setup_fdw.sh
@@ -0,0 +1,20 @@
+# Create foreign data wrapper to another postgresql database
+#
+ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -U postgres -t carto_db_development`
+
+echo "CREATE EXTENSION postgres_fdw;" | psql -U postgres $ORGANIZATION_DB
+echo "CREATE SCHEMA gps;" | psql -U postgres $ORGANIZATION_DB
+echo "CREATE SERVER remotedb FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host '10.0.0.1', port '5432', dbname 'somedb');" | psql -U postgres $ORGANIZATION_DB
+echo "CREATE FOREIGN TABLE gps.places (id integer NOT NULL, location geometry) SERVER remotedb OPTIONS (schema_name 'gps', table_name 'places');"
+
+for user in `echo "SELECT 'development_cartodb_user_' || id FROM users WHERE organization_id = (SELECT id FROM organizations WHERE name='example')" | psql -U postgres -t carto_db_development`
+do
+  echo "GRANT USAGE ON SCHEMA gps TO ${user};" | psql -U postgres $ORGANIZATION_DB
+  echo "GRANT SELECT ON gps.places TO ${user};" | psql -U postgres $ORGANIZATION_DB
+  echo "GRANT USAGE ON FOREIGN SERVER remotedb TO ${user}" | psql -U postgres $ORGANIZATION_DB
+done
+
+# User should be able to creat his/her own mapping with
+# CREATE USER MAPPING FOR bob SERVER remotedb (user 'bob', password 'secret');
+# Then in create empty table in CartoDB and run something like
+# SELECT row_number() OVER(ORDER BY id), location the_geom, ST_Transform(location, 3857) AS the_geom_webmercator FROM gps.places;
diff --git a/setup_organization.sh b/setup_organization.sh
new file mode 100644
index 0000000..39a25e5
--- /dev/null
+++ b/setup_organization.sh
@@ -0,0 +1,19 @@
+ORGANIZATION_NAME="example"
+USERNAME="admin4example"
+EMAIL="admin@contoso.com"
+PASSWORD="pass1234"
+
+bundle exec rake cartodb:db:create_user EMAIL="${EMAIL}" PASSWORD="${PASSWORD}" SUBDOMAIN="${USERNAME}"
+bundle exec rake cartodb:db:set_unlimited_table_quota["${USERNAME}"]
+bundle exec rake cartodb:db:set_user_quota["${USERNAME}",102400]
+bundle exec rake cartodb:db:create_new_organization_with_owner ORGANIZATION_NAME="${ORGANIZATION_NAME}" USERNAME="${USERNAME}" ORGANIZATION_SEATS=100 ORGANIZATION_QUOTA=102400 ORGANIZATION_DISPLAY_NAME="${ORGANIZATION_NAME}"
+bundle exec rake cartodb:db:set_organization_quota[$ORGANIZATION_NAME,5000]
+bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations[$ORGANIZATION_NAME]
+bundle exec rake cartodb:set_custom_limits_for_user["${USERNAME}",10240000000,100000000,1]
+
+# Enable sync tables
+echo "UPDATE users SET sync_tables_enabled=true WHERE username='${USERNAME}'" | psql -U postgres -t carto_db_development
+# Enable private maps
+echo "UPDATE users SET private_maps_enabled = 't'" | psql -U postgres -t carto_db_development
+
+bundle exec rake cartodb:features:enable_feature_for_all_users["new_dashboard"]
diff --git a/startup.sh b/startup.sh
new file mode 100755
index 0000000..c40ac39
--- /dev/null
+++ b/startup.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+export CARTO_HOSTNAME=${CARTO_HOSTNAME:=$HOSTNAME}
+
+perl -pi -e 's/cartodb\.localhost/$ENV{"CARTO_HOSTNAME"}/g' /etc/nginx/nginx.conf /cartodb/config/app_config.yml /Windshaft-cartodb/config/environments/development.js
+
+PGDATA=/var/lib/postgresql
+if [ "$(stat -c %U $PGDATA)" != "postgres" ]; then
+(>&2 echo "${PGDATA} not owned by postgres, updating permissions")
+chown -R postgres $PGDATA
+chmod 700 $PGDATA
+fi
+
+service postgresql start
+service redis-server start
+/opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl
+service nginx start
+
+cd /Windshaft-cartodb
+node app.js development &
+
+cd /CartoDB-SQL-API
+node app.js development &
+
+cd /cartodb
+bundle exec script/restore_redis
+bundle exec script/resque > resque.log 2>&1 &
+script/sync_tables_trigger.sh &
+
+# Recreate api keys in db and redis, so sql api is authenticated
+echo 'delete from api_keys' | psql -U postgres -t carto_db_development
+bundle exec rake carto:api_key:create_default
+
+# bundle exec rake carto:api_key:create_default
+bundle exec thin start --threaded -p 3000 --threadpool-size 5
diff --git a/sync_tables_trigger.sh b/sync_tables_trigger.sh
new file mode 100644
index 0000000..8c01fc5
--- /dev/null
+++ b/sync_tables_trigger.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+while :
+do
+sleep $SYNC_TABLES_INTERVAL
+cd /cartodb
+bundle exec rake cartodb:sync_tables[true]
+done
diff --git a/template_postgis.sh b/template_postgis.sh
new file mode 100755
index 0000000..6b3986f
--- /dev/null
+++ b/template_postgis.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Init script for template postgis
+#
+
+POSTGIS_SQL_PATH=`pg_config --sharedir`/contrib/postgis-2.2;
+createdb -E UTF8 template_postgis;
+createlang -d template_postgis plpgsql;
+psql -d postgres -c "UPDATE pg_database SET datistemplate='true' \
+  WHERE datname='template_postgis'"
+psql -d template_postgis -c "CREATE EXTENSION postgis;"
+psql -d template_postgis -c "CREATE EXTENSION postgis_topology;"
+psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"
+psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"
+psql -c "CREATE EXTENSION plpythonu;"
+psql -d template_postgis -c "CREATE EXTENSION plpythonu;"
+psql -d template_postgis -c "CREATE EXTENSION crankshaft VERSION 'dev';"
+psql -d template_postgis -c "CREATE EXTENSION plproxy;"