cartodb/app/helpers/file_upload.rb

require 'aws-sdk-s3'

module CartoDB
  class FileUpload

    DEFAULT_UPLOADS_PATH = 'public/uploads'
    FILE_ENCODING = 'utf-8'
    MAX_SYNC_UPLOAD_S3_FILE_SIZE = 52428800 # bytes

    def initialize(uploads_path = nil)
      @uploads_path = uploads_path || DEFAULT_UPLOADS_PATH
      @uploads_path = if @uploads_path[0] == "/"
                        Pathname.new(@uploads_path)
                      else
                        Rails.root.join(@uploads_path)
                      end
    end

    def get_uploads_path
      @uploads_path
    end

    # force_s3_upload uploads file to S3 even if size is greater than `MAX_SYNC_UPLOAD_S3_FILE_SIZE`
    def upload_file_to_storage(filename_param: nil,
                               file_param: nil,
                               request_body: nil,
                               s3_config: nil,
                               timestamp: Time.now,
                               allow_spaces: false,
                               force_s3_upload: false,
                               random_token: nil)
      results = {
        file_uri: nil,
        enqueue:  true
      }

      load_file_from_request_body = false
      case
      when filename_param.present? && request_body.present?
        filename = filename_param.original_filename rescue filename_param.to_s
        begin
          filepath = filename_param.path
        rescue
          filepath = filename_param.to_s
          load_file_from_request_body = true
        end
      when file_param.present?
        filename = file_param.original_filename rescue file_param.to_s
        filepath = file_param.path rescue ''
      else
        return results
      end

      filename = filename.tr(' ', '_') unless allow_spaces

      random_token ||= Digest::SHA2.hexdigest("#{timestamp.utc}--#{filename.object_id}").first(20)

      use_s3 = !s3_config.nil? && s3_config['access_key_id'].present? && s3_config['secret_access_key'].present? &&
               s3_config['bucket_name'].present? && s3_config['url_ttl'].present?

      file = nil
      if load_file_from_request_body
        file = filedata_from_params(filename_param, file_param, request_body, random_token, filename)
        filepath = file.path
      end

      do_long_upload = s3_config && s3_config['async_long_uploads'].present? && s3_config['async_long_uploads'] &&
        File.size(filepath) > MAX_SYNC_UPLOAD_S3_FILE_SIZE

      if use_s3 && (!do_long_upload || force_s3_upload)
        file_url = upload_file_to_s3(filepath, filename, random_token, s3_config)

        if load_file_from_request_body
          File.delete(file.path)
        end

        results[:file_uri] = file_url
      else
        unless load_file_from_request_body
          file = filedata_from_params(filename_param, file_param, request_body, random_token, filename)
        end

        results[:file_path] = file.path

        if use_s3 && do_long_upload
          results[:file_uri] = file.path[/(\/uploads\/.*)/, 1]
          results[:enqueue] = false
        else
          results[:file_uri] = file.path[/(\/uploads\/.*)/, 1]
        end
      end

      results
    end

    def upload_file_to_s3(filepath, filename, token, s3_config)
      s3_config_hash = {
        access_key_id: s3_config['access_key_id'],
        secret_access_key: s3_config['secret_access_key'],
        http_proxy: s3_config['proxy_uri'].present? ? s3_config['proxy_uri'] : nil,
        region: s3_config['region']
      }
      # This allows to override the S3 endpoint in case a non AWS compatible
      # S3 storage service is being used
      # WARNING: This attribute may not work in some aws-sdk v1 versions newer
      # than 1.8.5 (http://lists.basho.com/pipermail/riak-users_lists.basho.com/2013-May/011984.html)
      s3_config_hash[:endpoint] = s3_config['s3_endpoint'] if s3_config['s3_endpoint'].present?
      Aws.config = s3_config_hash

      s3 = Aws::S3::Resource.new
      obj = s3.bucket(s3_config['bucket_name']).object("#{token}/#{File.basename(filename)}")

      obj.upload_file(filepath, acl: 'authenticated-read')

      options = { expires_in: s3_config['url_ttl'] }
      content_disposition = s3_config['content-disposition']
      options[:response_content_disposition] = content_disposition if content_disposition.present?
      obj.presigned_url(:get, options)
    end

    private

    def filedata_from_params(filename_param, file_param, request_body, random_token, filename)
      case
      when filename_param.present? && request_body.present?
        filedata = filename_param
      when file_param.present?
        filedata = file_param
      else
        return
      end

      FileUtils.mkdir_p(get_uploads_path.join(random_token))

      if filedata.respond_to?(:tempfile)
        save_using_streaming(filedata, random_token, filename)
      else
        begin
          data = filedata.read.force_encoding(FILE_ENCODING)
        rescue
          data = request_body.read.force_encoding(FILE_ENCODING)
        end
        save(data, random_token, filename)
      end
    end

    def save(filedata, random_token, filename)
      file = File.new(get_uploads_path.join(random_token).join(File.basename(filename)), "w:#{FILE_ENCODING}")
      file.write filedata
      file.close
      file
    end

    def save_using_streaming(filedata, random_token, filename)
      src = File.open(filedata.tempfile.path, "r:UTF-8")
      file = File.new(get_uploads_path.join(random_token).join(File.basename(filename)), 'w:UTF-8')
      IO.copy_stream(src, file)
      file.close
      src.close
      file
    end
  end

  # CartoDB::FileUpload was originally designed for Rails' UploadedFile.
  # In order to avoid this need this class provides the needed interface.
  class FileUploadFile
    attr_reader :path, :original_filename, :filename, :tempfile

    def initialize(filepath)
      @path = filepath
      @original_filename = File.basename(filepath)
      @filename = @original_filename
      @tempfile = File.new(filepath)
    end
  end
end