Merge pull request #76 from nballenger/update_caching

Update caching

Dockerfile

@@ -221,7 +221,7 @@ ADD ./config/app_config.yml /cartodb/config/app_config.yml
 ADD ./config/database.yml /cartodb/config/database.yml
 ADD ./create_dev_user /cartodb/script/create_dev_user
 ADD ./setup_organization.sh /cartodb/script/setup_organization.sh
-ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
+ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/nginx.conf
 ADD ./config/varnish.vcl /etc/varnish.vcl
 ADD ./geocoder.sh /cartodb/script/geocoder.sh
 ADD ./geocoder_server.sql /cartodb/script/geocoder_server.sql

config/app_config.yml

@@ -163,14 +163,14 @@ defaults: &defaults
     timeout: 5
     # 'warning' or 'error'
     trigger_verbose: true
-  invalidation_service:
+#  invalidation_service:
-    enabled: false
+#    enabled: false
-    host: '127.0.0.1'
+#    host: '127.0.0.1'
-    port: 3142
+#    port: 3142
-    retries: 5 # number of retries before considering failure
+#    retries: 5 # number of retries before considering failure
-    critical: false # either the failure is considered an error or a warning
+#    critical: false # either the failure is considered an error or a warning
-    timeout: 5 # socket timeout
+#    timeout: 5 # socket timeout
-    trigger_verbose: true
+#    trigger_verbose: true
   redis:
     host: '127.0.0.1'
     port: 6379

config/cartodb.nginx.proxy.conf

@@ -1,6 +1,33 @@
-server {
+user www-data;
-  server_name cartodb.localhost *.cartodb.localhost;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
 
+events {
+    worker_connections 768;
+}
+
+http {
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    ssl_protocols               TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+    ssl_prefer_server_ciphers   on;
+
+    access_log  /var/log/nginx/access.log;
+    error_log   /var/log/nginx/error.log;
+
+    log_format main '[$time_local] $status REQUEST: "$request" REFERER: "$http_referer" FWD_FOR "$http_x_forwarded_for" PROXY_HOST: "$proxy_host" UPSTREAM_ADDR: "$upstream_addr"';
+
+    gzip on;
+
+    server {
+      server_name           cartodb.localhost *.cartodb.localhost;
       client_max_body_size  0;
 
       location ~* /(user/.*/)?api/v1/maps {
@@ -16,17 +43,23 @@ server {
         proxy_set_header        X-Real-IP $remote_addr;
         proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header        X-Forwarded-Proto $scheme;
-    proxy_pass http://127.0.0.1:8181;
+        proxy_set_header        X-Carto-Service windshaft; # tell varnish what backend
+        proxy_no_cache          true;           # Make sure nginx doesn't cache
+        proxy_cache_bypass      true;           # Make sure nginx doesn't cache
+        proxy_pass              http://127.0.0.1:6081;  # hand off to Varnish
       }
 
-  location ~* /(user/.*)?/api/v2/sql {
+      location ~* /(user/.*/)?api/v2/sql {
         # RedHog: Hack to work around bug in cartodb local hosting but using cdn for js libs
         rewrite /(user/.*)?/api/v2/sql(.*) /$1/api/v2/sql$2  break;
         proxy_set_header        Host $host;
         proxy_set_header        X-Real-IP $remote_addr;
         proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header        X-Forwarded-Proto $scheme;
-    proxy_pass http://127.0.0.1:8080;
+        proxy_set_header        X-Carto-Service sqlapi; # tell varnish what backend
+        proxy_no_cache          true;           # make sure nginx doesn't cache
+        proxy_cache_bypass      true;           # make sure nginx doesn't cache
+        proxy_pass              http://127.0.0.1:6081;  # hand off to Varnish
       }
 
       location ^~ /assets {
@@ -42,5 +75,6 @@ server {
       }
 
       error_log /var/log/nginx/cartodb_error.log;
-  access_log /var/log/nginx/cartodb_access.log;
+      access_log /var/log/nginx/cartodb_access.log main;
+    }
 }

config/varnish.vcl

@@ -1,4 +1,48 @@
-backend default {
+acl purge {
+    "localhost";
+    "127.0.0.1";
+}
+
+backend sqlapi {
     .host = "127.0.0.1";
     .port = "8080";
 }
+
+backend windshaft {
+    .host = "127.0.0.1";
+    .port = "8181";
+}
+
+sub vcl_recv {
+    # Allowing PURGE from localhost
+    if (req.request == "PURGE") {
+        if (!client.ip ~ purge) {
+            error 405 "Not allowed.";
+        }
+        return (lookup);
+    }
+
+    # Routing request to backend based on X-Carto-Service header from nginx
+    if (req.http.X-Carto-Service == "sqlapi") {
+        set req.backend = sqlapi;
+        remove req.http.X-Carto-Service;
+    }
+    if (req.http.X-Carto-Service == "windshaft") {
+        set req.backend = windshaft;
+        remove req.http.X-Carto-Service;
+    }
+}
+
+sub vcl_hit {
+    if (req.request == "PURGE") {
+        purge;
+        error 200 "Purged.";
+    }
+}
+
+sub vcl_miss {
+    if (req.request == "PURGE") {
+        purge;
+        error 200 "Purged.";
+    }
+}

startup.sh

@@ -2,7 +2,7 @@
 
 export CARTO_HOSTNAME=${CARTO_HOSTNAME:=$HOSTNAME}
 
-perl -pi -e 's/cartodb\.localhost/$ENV{"CARTO_HOSTNAME"}/g' /etc/nginx/sites-enabled/default /cartodb/config/app_config.yml /Windshaft-cartodb/config/environments/development.js
+perl -pi -e 's/cartodb\.localhost/$ENV{"CARTO_HOSTNAME"}/g' /etc/nginx/nginx.conf /cartodb/config/app_config.yml /Windshaft-cartodb/config/environments/development.js
 
 PGDATA=/var/lib/postgresql
 if [ "$(stat -c %U $PGDATA)" != "postgres" ]; then