Updated robots.txt and fixed base url

lib/assets/javascripts/new-dashboard/utils/catalog/base-url.js

@@ -4,6 +4,6 @@ export function getBaseURL (state) {
     const host = state.config.account_host;
     return `https://${username}.${host}/api/v4`;
   } else {
-    return 'https://public.carto.com/api/v4/';
+    return 'https://public.carto.com/api/v4';
   }
 }

public/robots.txt

@@ -2,16 +2,11 @@
 # while trying to access our API or whatever
 
 User-agent: Googlebot
-Disallow: /api/
-
 User-Agent: Slurp
-Disallow: /api/
-
 User-agent: Yandex
-Disallow: /api/
-
 User-Agent: msnbot
-Disallow: /api/
-
 User-agent: baiduspider
 Disallow: /api/
+
+User-agent: Googlebot
+Allow: /api/v4/data/observatory