CDB-1788 started to make gdrive code have shape. commit to change branch

Gemfile

@@ -62,6 +62,7 @@ gem 'dbf',                     '2.0.6'
 
 # Sync tables
 gem 'faraday',                 '0.9.0'
+gem 'retriable',               '1.4.1'
 gem 'google-api-client',       '0.7.0'
 gem 'dropbox-sdk',             '1.6.3'
 

Gemfile.lock

@@ -277,6 +277,7 @@ GEM
       redis-namespace (~> 1.0)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
+    retriable (1.4.1)
     rgeo (0.3.2)
     rgeo-geojson (0.2.1)
       rgeo (>= 0.2.8)
@@ -425,6 +426,7 @@ DEPENDENCIES
   rchardet19 (= 1.3.5)
   redis (= 2.2.2)
   resque (= 1.23.0)
+  retriable (= 1.4.1)
   rgeo (= 0.3.2)
   rgeo-geojson (= 0.2.1)
   rollbar (= 0.9.6)

Makefile

@@ -28,6 +28,7 @@ WORKING_SPECS = \
   spec/requests/api/user_layers_spec.rb \
   spec/requests/api/map_layers_spec.rb \
   spec/requests/api/records_spec.rb \
+  services/synchronizer/spec/unit/fileproviders_gdrive_spec.rb \
   $(NULL)
 
 CDB_PATH=lib/assets/javascripts/cdb

services/synchronizer/lib/synchronizer/file-providers/base.rb

@@ -3,7 +3,7 @@
 module CartoDB
   module Synchronizer
     module FileProviders
-      class Base
+      class BaseProvider
 
         FORMAT_CSV = 'csv'
         FORMAT_EXCEL = 'xls'

services/synchronizer/lib/synchronizer/file-providers/gdrive.rb

@@ -1,11 +1,12 @@
 # encoding: utf-8
 
+require_relative 'base'
 require 'google/api_client'
 
 module CartoDB
   module Synchronizer
     module FileProviders
-      class GDrive
+      class GDrive < BaseProvider
 
         # Required for all providers
         SERVICE = 'gdrive'
@@ -20,73 +21,102 @@ module CartoDB
             FORMAT_SVG => %W( image/svg+xml )
         }
 
-        def get_new(config)
-          return GDrive.new(config)
-        end
+        def self.get_new(config)
+          return new(config)
+        end #get_new
 
         def initialize(config)
           @service_name = SERVICE
 
           @client = Google::APIClient.new ({
-              application_name: config.fetch('application_name')
+              application_name: config.fetch(:application_name)
           })
           @drive = @client.discovered_api('drive', 'v2')
 
-          @client.authorization.client_id = config.fetch('client_id')
-          @client.authorization.client_secret = config.fetch('client_secret')
+          @client.authorization.client_id = config.fetch(:client_id)
+          @client.authorization.client_secret = config.fetch(:client_secret)
           @client.authorization.scope = oauth_scope
           @client.authorization.redirect_uri = redirect_uri
-        end
+        end #initialize
 
         def get_auth_url
           return @client.authorization.authorization_uri
-        end
+        end #get_auth_url
 
-        def validate_auth_token(token)
-          @client.authorization.code = token
+        def validate_auth_code(auth_code)
+          @client.authorization.code = auth_code
           @client.authorization.fetch_access_token!
-        end
+        end #validate_auth_code
 
         def get_files_list(formats_filter={})
-          if formats_filter.empty?
-            @formats = SUPPORTED_FORMATS
-          else
-            @formats = []
-            formats_filter.each do |id, mime_types|
-              @formats = @formats.push(mime_types)
+          all_results = []
+          setup_formats_filter(formats_filter)
+
+          batch_request = Google::APIClient::BatchRequest.new do |result|
+            if result.status == 200
+              data = result.data.to_hash
+              if data.include? 'items'
+                data['items'].each do |item|
+                  all_results.push(item)
+                end
+              end
+            else
+              # TODO: Proper log (inject at instantiation)
+              puts "An error occurred: #{result.data['error']['message']}"
             end
           end
 
-          # [ 'service', 'id', 'title' ]
-          raise 'To be implemented in child classes'
-        end
+          @formats.each do |mime_type|
+            batch_request.add(
+              :api_method => @drive.files.list,
+              :parameters => {
+                'trashed' => 'false',
+                'q' => "mime_type = '#{mime_type}'",
+                'fields' => fields_to_retrieve
+              }
+            )
+          end
+          @client.execute(batch_request)
+
+          # TODO: Format results [ 'service', 'id', 'title' ]
+          all_results
+        end #get_files_list
+
 
         def store_chosen_files(id, service, sync_type)
-          raise 'To be implemented in child classes'
-        end
+          raise 'Pending implementation'
+        end #store_chosen_files
 
         def download_file(service, id)
-          raise 'To be implemented in child classes'
-        end
+          raise 'Pending implementation'
+        end #download_file
+
+        def setup_formats_filter(filter=[])
+          @formats = []
+          FORMATS_TO_MIME_TYPES.each do |id, mime_types|
+            if (filter.empty? || filter.include?(id))
+              mime_types.each do |mime_type|
+                @formats = @formats.push(mime_type)
+              end
+            end
+          end
+        end #setup_formats_filter
+
+        attr_reader :formats
 
         private
 
         def oauth_scope
           'https://www.googleapis.com/auth/drive'
-        end
+        end #oauth_scope
 
         def redirect_uri
           'urn:ietf:wg:oauth:2.0:oob'
-        end
+        end #redirect_uri
 
         def fields_to_retrieve
           'items(downloadUrl,etag,exportLinks,fileExtension,id,mimeType,modifiedDate,originalFilename,title)'
-        end
-
-        def test_method
-
-        end
-
+        end #fields_to_retrieve
 
       end #GDrive
     end #FileProviders

services/synchronizer/spec/unit/fileproviders_gdrive_spec.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+
+require_relative '../../lib/synchronizer/file-providers/gdrive'
+
+include CartoDB::Synchronizer::FileProviders
+
+describe GDrive do
+
+  def get_config
+    {
+      application_name: '',
+      client_id: '',
+      client_secret: ''
+    }
+  end #get_config
+
+  describe '#filters' do
+    it 'test that filter options work correctly' do
+      gdrive_provider = GDrive.get_new(get_config)
+
+      # No filter = all formats allowed
+      expected_formats = []
+      GDrive::FORMATS_TO_MIME_TYPES.each do |id, mime_types|
+        mime_types.each do |mime_type|
+          expected_formats = expected_formats.push(mime_type)
+        end
+      end
+      gdrive_provider.setup_formats_filter()
+      gdrive_provider.formats.should eq expected_formats
+
+      # Filter to 'documents'
+      expected_formats = []
+      format_ids = [ GDrive::FORMAT_CSV, GDrive::FORMAT_EXCEL ]
+      GDrive::FORMATS_TO_MIME_TYPES.each do |id, mime_types|
+        if format_ids.include?(id)
+          mime_types.each do |mime_type|
+            expected_formats = expected_formats.push(mime_type)
+          end
+        end
+
+      end
+      gdrive_provider.setup_formats_filter(format_ids)
+      gdrive_provider.formats.should eq expected_formats
+    end
+  end #run
+
+  describe '#manual_test' do
+    it 'with user interaction, tests the full oauth flow' do
+      pending('This test requires manual run, opening the url in a browser, grabbing the code and setting input to it')
+
+      gdrive_provider = GDrive.get_new(get_config)
+
+      gdrive_provider.setup_formats_filter()
+      puts gdrive_provider.get_auth_url
+      input = ''
+      debugger
+      gdrive_provider.validate_auth_code(input)
+      data = gdrive_provider.get_files_list()
+      puts data
+    end
+  end
+
+end
+