zhongjin
/
cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update resque and change importer jobs signature

Browse Source 
* Update Resque to the latest version in order to fix redis connection errors
  on resque workers
* Update sequel and vizzuality-sequel-rails, remove the activerecord
  requirement
* Change importer job signature to ease debugging

I tried resque-status, but we need to get a list of pending jobs from redis
and is not possible to get that with resque-status, so we'll have to stick
with resque-job-tracking for the time being
2.0
David Arango 12 years ago
parent e882cf2b09
commit 8fe233d8ca
6 changed files with 52 additions and 53 deletions
Show Stats Download Patch File Download Diff File

13
Gemfile
Unescape View File

@ -4,16 +4,10 @@ gem "rails", "3.2.2"
gem "rake", "0.9.2.2"
gem "pg", "0.11"
gem "sequel", "3.37.0"
gem "sequel", "3.42.0"
gem "sequel_pg", "1.2.0", :require => "sequel"
# TODO remove this when everything works
# gem "sequel_column_type_array", "0.0.4"
# gem "sequel-rails", :git => "git://github.com/tokumine/sequel-rails.git", :tag => '0.3.6'
# gem "sequel-rails-cartodb", "~> 0.1.7", :require => "sequel-rails"
# gem "text-hyphen", "1.2.0"
gem "vizzuality-sequel-rails", "0.3.5"
gem "vizzuality-sequel-rails", "0.3.6", :git => 'https://github.com/Vizzuality/sequel-rails.git'
gem "rails_warden", "0.5.2" # Auth via the Warden Rack framework
gem "oauth", "0.4.5"
@ -25,7 +19,8 @@ gem "rgeo", "0.3.2" # Geospatial data library
gem "rgeo-geojson", "0.2.1", :require => "rgeo/geo_json"
gem "redis", "2.2.2"
gem "resque", "1.19.0"
gem "resque", "1.23.0"
gem "resque-meta", "1.0.3"
gem "resque-job-tracking", "0.0.4"
gem "yajl-ruby", "1.1.0", :require => "yajl"
gem "airbrake", "3.0.9"

50
Gemfile.lock
Unescape View File

@ -1,3 +1,11 @@
GIT
remote: https://github.com/Vizzuality/sequel-rails.git
revision: 09ad75f865bfd488da5712ffacc62ba4503530ab
specs:
vizzuality-sequel-rails (0.3.6)
railties (~> 3.2.0)
sequel (~> 3.28)
GIT
remote: https://github.com/josh/mustache-trimmer.git
revision: b67722e3eb8d35238c14f7910981cdc0cdc033a2
@ -71,7 +79,7 @@ GEM
debugger-ruby_core_source (~> 1.1.3)
debugger-linecache (1.1.2)
debugger-ruby_core_source (>= 1.1.1)
debugger-ruby_core_source (1.1.4)
debugger-ruby_core_source (1.1.5)
diff-lcs (1.1.3)
ejs (1.1.1)
email_spec (1.2.1)
@ -90,7 +98,7 @@ GEM
multipart-post (~> 1.1)
faye-websocket (0.4.6)
eventmachine (>= 0.12.0)
ffi (1.1.5)
ffi (1.2.0)
foreman (0.46.0)
thor (>= 0.13.6)
fssm (0.2.9)
@ -113,8 +121,9 @@ GEM
launchy (2.1.0)
addressable (~> 2.2.6)
libv8 (3.3.10.4)
libwebsocket (0.1.5)
libwebsocket (0.1.7.1)
addressable
websocket
log4r (1.1.10)
mail (2.4.4)
i18n (>= 0.4.0)
@ -125,7 +134,7 @@ GEM
minitest (2.0.2)
mocha (0.10.5)
metaclass (~> 0.0.1)
multi_json (1.3.6)
multi_json (1.5.0)
multi_xml (0.5.1)
multipart-post (1.1.5)
mustache (0.99.4)
@ -142,7 +151,7 @@ GEM
oily_png (1.0.2)
chunky_png (~> 1.2.1)
pg (0.11.0)
poltergeist (1.0.1)
poltergeist (1.0.2)
capybara (~> 1.1)
childprocess (~> 0.3)
faye-websocket (~> 0.4, >= 0.4.4)
@ -152,7 +161,7 @@ GEM
rack (1.4.1)
rack-cache (1.2)
rack (>= 0.4)
rack-protection (1.2.0)
rack-protection (1.3.2)
rack
rack-reverse-proxy (0.4.4)
rack (>= 1.0.0)
@ -182,11 +191,11 @@ GEM
rdoc (3.12)
json (~> 1.4)
redis (2.2.2)
redis-namespace (1.0.3)
redis-namespace (1.1.0)
redis (< 3.0.0)
resque (1.19.0)
resque (1.23.0)
multi_json (~> 1.0)
redis-namespace (~> 1.0.2)
redis-namespace (~> 1.0)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
resque-job-tracking (0.0.4)
@ -218,17 +227,17 @@ GEM
rspec (~> 2.10.0)
ruby-ole (1.2.11.3)
rubyzip (0.9.9)
sass (3.2.1)
sass (3.2.3)
sass-rails (3.2.5)
railties (~> 3.2.0)
sass (>= 3.1.10)
tilt (~> 1.3)
selenium-webdriver (2.25.0)
selenium-webdriver (2.27.2)
childprocess (>= 0.2.5)
libwebsocket (~> 0.1.3)
multi_json (~> 1.0)
rubyzip
sequel (3.37.0)
sequel (3.42.0)
sequel_pg (1.2.0)
pg (>= 0.8.0)
sequel (>= 3.29.0)
@ -257,26 +266,24 @@ GEM
thor (0.14.6)
tilt (1.3.3)
timecop (0.3.5)
todonotes (0.1.0)
todonotes (0.1.1)
log4r
treetop (1.4.11)
treetop (1.4.12)
polyglot
polyglot (>= 0.3.1)
turbo-sprockets-rails3 (0.1.16)
railties (>= 3.1.0)
sprockets (>= 2.0.0)
tzinfo (0.3.33)
tzinfo (0.3.35)
uglifier (1.3.0)
execjs (>= 0.3.0)
multi_json (~> 1.0, >= 1.0.2)
uuidtools (2.1.3)
vegas (0.1.11)
rack (>= 1.0.0)
vizzuality-sequel-rails (0.3.5)
rails (~> 3.2.0)
sequel (~> 3.28)
warden (1.2.1)
rack (>= 1.0)
websocket (1.0.4)
xpath (0.1.4)
nokogiri (~> 1.3)
yajl-ruby (1.1.0)
@ -317,8 +324,9 @@ DEPENDENCIES
rake (= 0.9.2.2)
rchardet19 (= 1.3.5)
redis (= 2.2.2)
resque (= 1.19.0)
resque (= 1.23.0)
resque-job-tracking (= 0.0.4)
resque-meta (= 1.0.3)
rgeo (= 0.3.2)
rgeo-geojson (= 0.2.1)
roo (= 1.9.7)
@ -327,7 +335,7 @@ DEPENDENCIES
rubyzip (= 0.9.9)
sass-rails (~> 3.2.3)
selenium-webdriver (>= 2.5.0)
sequel (= 3.37.0)
sequel (= 3.42.0)
sequel_pg (= 1.2.0)
simplecov (= 0.6.4)
spin (= 0.5.3)
@ -339,5 +347,5 @@ DEPENDENCIES
timecop (= 0.3.5)
turbo-sprockets-rails3 (= 0.1.16)
uglifier (~> 1.3.0)
vizzuality-sequel-rails (= 0.3.5)
vizzuality-sequel-rails (= 0.3.6)!
yajl-ruby (= 1.1.0)

16
app/controllers/api/json/imports_controller.rb
Unescape View File

@ -16,18 +16,20 @@ class Api::Json::ImportsController < Api::ApplicationController
end
def create
file_uri = params[:url].present? ? params[:url] : _upload_file
table_name = params[:table_name].presence
table_id = params[:table_id].presence
append = params[:append].presence == 'true'
from_query = params[:sql].presence
table_copy = params[:table_copy].presence
file_uri = params[:url].present? ? params[:url] : _upload_file
if synchronous_import?
#@data_import = Resque::ImporterJobs.process(current_user[:id], params[:table_name], file_uri)
#render :json => {:item_queue_id => job_meta.meta_id, :success => true}
else
job = Resque::ImporterJobs.enqueue(current_user[:id], table_name, file_uri, table_id, append, nil, table_copy, from_query)
job = Resque::ImporterJobs.enqueue(:user_id => current_user[:id],
:table_name => params[:table_name].presence,
:data_source => file_uri.presence,
:table_id => params[:table_id].presence,
:append => (params[:append].presence == 'true'),
:table_copy => params[:table_copy].presence,
:from_query => params[:sql].presence)
render_jsonp({:item_queue_id => job.meta_id, :success => true})
end
#rescue => e

1
app/models/user.rb
Unescape View File

@ -1,5 +1,4 @@
# coding: UTF-8
class User < Sequel::Model
include CartoDB::MiniSequel

2
config/initializers/load_resque.rb
Unescape View File

@ -2,4 +2,4 @@ require 'resque'
# Load automatically all resque files from lib/resque
Dir[Rails.root.join("lib/resque/*.rb")].each {|f| require f}
Resque.redis = "#{Cartodb.config[:redis]['host']}:#{Cartodb.config[:redis]['port']}"
Resque.redis = "#{Cartodb.config[:redis]['host']}:#{Cartodb.config[:redis]['port']}"

23
lib/resque/importer_jobs.rb
Unescape View File

@ -6,22 +6,17 @@ module Resque
@queue = :import_jobs
def self.track(user_id, table_name, data_source, table_id = nil, append = false, migrate_table = nil, table_copy = nil, from_query = nil)
[User.where(:id => user_id).first.job_tracking_identifier]
def self.track(options)
[User.where(:id => options.symbolize_keys[:user_id]).first.job_tracking_identifier]
end
def self.perform(meta_id, user_id, table_name, data_source, table_id = nil, append = false, migrate_table = nil, table_copy = nil, from_query = nil)
DataImport.create( :queue_id => meta_id,
:user_id => user_id,
:table_id => table_id,
:table_name => table_name,
:data_source => data_source,
:updated_at => Time.now,
:append => append,
:migrate_table => migrate_table,
:table_copy => table_copy,
:from_query => from_query )
def self.perform(meta_id, options)
begin
DataImport.create options.symbolize_keys.merge(:updated_at => Time.now, :queue_id => meta_id)
rescue => e
# TODO handle some common exceptions here
raise e
end
end
end
end

Write Preview
Loading…
Cancel
Save