cartodb-4.42/services/datasources/spec/unit/arcgis_spec.rb

523 lines
17 KiB
Ruby
Raw Normal View History

2024-04-06 13:25:13 +08:00
require 'active_support'
require 'active_support/core_ext'
require_relative '../../lib/datasources'
require_relative '../doubles/user'
include CartoDB::Datasources
describe Url::ArcGIS do
before(:all) do
@url = 'http://myserver/arcgis/rest/services/MyFakeService/featurename'
@invalid_url = 'http://myserver/mysite/rest/myfakefolder/MyFakeService/featurename'
@user = CartoDB::Datasources::Doubles::User.new
end
before(:each) do
Typhoeus::Expectation.clear
end
describe '#set_data_from' do
it 'tests preparing the correct url from the one given from the UI' do
invalid_1 = 'http://myserver/services/MyFakeService/featurename/MapServer'
invalid_2 = 'myserver/services/MyFakeService/featurename/MapServer'
test1 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer'
test2 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/'
test3 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0'
test4 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0?'
test5 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0?q=blablabla'
test6 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer?q=blablabla'
valid_map = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer'
valid_map_trailing_slash = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/'
valid_layer = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/MapServer/0'
# Should be treated as ok
test7 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/2314/'
valid_7 = 'http://myserver/arcgis/rest/services/MyFakeService/featurename/2314/'
arcgis = Url::ArcGIS.get_new(@user)
expect {
arcgis.send(:sanitize_id, invalid_1)
}.to raise_error InvalidInputDataError
expect {
arcgis.send(:sanitize_id, invalid_2)
}.to raise_error InvalidInputDataError
arcgis.send(:sanitize_id, test1).should eq valid_map
arcgis.send(:sanitize_id, test2).should eq valid_map_trailing_slash
arcgis.send(:sanitize_id, test3).should eq valid_layer
arcgis.send(:sanitize_id, test4).should eq valid_layer
arcgis.send(:sanitize_id, test5).should eq valid_layer
arcgis.send(:sanitize_id, test6).should eq valid_map
arcgis.send(:sanitize_id, test7).should eq valid_7
end
end
describe '#get_resource_metadata' do
it 'tests error scenarios' do
arcgis = Url::ArcGIS.get_new(@user)
sub_id = '0'
# 'general http error (non-200)'
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do
Typhoeus::Response.new(
code: 400,
headers: { 'Content-Type' => 'application/json' },
body: ''
)
end
expect {
arcgis.get_resource_metadata(@url)
}.to raise_error DataDownloadError
# Stub layers request (so now works)
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_layers.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(::JSON.parse(body))
)
end
layers_data = arcgis.get_resource_metadata(@url)
layers_data_expected = {
id: @url,
subresources: [{
id: "#{@url}/#{sub_id}",
title: 'first layer'
}]
}
layers_data.should eq layers_data_expected
# 'fields' part
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
body = ::JSON.parse(body)
body.delete('fields')
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(body)
)
end
expect {
arcgis.send(:get_subresource_metadata, @url, sub_id)
}.to raise_error ResponseError
# Another required field
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
body = ::JSON.parse(body)
body.delete('name')
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(body)
)
end
expect {
arcgis.send(:get_subresource_metadata, @url, sub_id)
}.to raise_error ResponseError
# Invalid ArcGIS version
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
body = ::JSON.parse(body)
body['currentVersion'] = 9.0
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(body)
)
end
expect {
arcgis.send(:get_subresource_metadata, @url, sub_id)
}.to raise_error InvalidServiceError
# Invalid ArcGIS URL
expect {
arcgis.send(:get_resource_metadata, @invalid_url)
}.to raise_error InvalidInputDataError
end
it 'tests metadata retrieval' do
arcgis = Url::ArcGIS.get_new(@user)
# Stub layers request (so now works)
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_layers.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(::JSON.parse(body))
)
end
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0/) do |request|
accept = (request.options[:headers]||{})['Accept'] || 'application/json'
format = accept.split(',').first
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => format },
body: body
)
end
expected_metadata = {
:arcgis_version=>10.22,
:name=>"Test Feature",
:description=>"Sample metadata payload",
:type=>"Feature Layer",
:geometry_type=>"esriGeometryPolygon",
:copyright=>"CartoDB",
:fields=>[
{
:name => "OBJECTID",
:type => "esriFieldTypeOID"
},
{
:name => "NAME",
:type => "esriFieldTypeString"
}
],
:max_records_per_query=>1000,
:supported_formats=>["JSON", "AMF"],
:advanced_queries_supported=>true
}
expected_metadata_response = {
id: @url + '/0',
title: 'Test Feature',
url: nil,
service: Url::ArcGIS::DATASOURCE_NAME,
checksum: nil,
size: 0,
filename: 'test_feature.json'
}
# Multi-resource scenario already tested above
response = arcgis.get_resource_metadata(@url + '/0')
response.nil?.should be false
arcgis.metadata.should eq expected_metadata
response.should eq expected_metadata_response
end
end
describe '#get_resource' do
it 'tests the get_ids_list() private method with error scenarios' do
arcgis = Url::ArcGIS.get_new(@user)
id = arcgis.send(:sanitize_id, @url)
# 'general http error (non-200)'
Typhoeus.stub(/\/arcgis\/rest\//) do
Typhoeus::Response.new(
code: 400,
headers: { 'Content-Type' => 'application/json' },
body: ''
)
end
expect {
arcgis.send(:get_ids_list, id)
}.to raise_error DataDownloadError
# 'objectIds' not present
Typhoeus::Expectation.clear
Typhoeus.stub(/\/arcgis\/rest\//) do
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list.json"))
body = ::JSON.parse(body)
body.delete('objectIds')
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(body)
)
end
expect {
arcgis.send(:get_ids_list, id)
}.to raise_error ResponseError
# 'objectIds' empty
Typhoeus::Expectation.clear
Typhoeus.stub(/\/arcgis\/rest\//) do
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list.json"))
body = ::JSON.parse(body)
body['objectIds'] = []
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(body)
)
end
expect {
arcgis.send(:get_ids_list, id)
}.to raise_error ResponseError
end
it 'tests the get_ids_list() private method' do
arcgis = Url::ArcGIS.get_new(@user)
id = arcgis.send(:sanitize_id, @url)
Typhoeus.stub(/\/arcgis\/rest\//) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: body
)
end
expected_ids = [1,2,3,4,5,6,7,8,9,10]
respose_ids = arcgis.send(:get_ids_list, id)
respose_ids.nil?.should be false
respose_ids.should eq expected_ids
end
it 'tests the get_ids_list() private method on out-of-order ids' do
arcgis = Url::ArcGIS.get_new(@user)
id = arcgis.send(:sanitize_id, @url)
Typhoeus.stub(/\/arcgis\/rest\//) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_unordered_ids_list.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: body
)
end
expected_ids = [1,2,3,4,5,6,7,8,9,10]
respose_ids = arcgis.send(:get_ids_list, id)
respose_ids.nil?.should be false
respose_ids.should eq expected_ids
end
it 'tests the get_by_ids() private method with error scenarios' do
arcgis = Url::ArcGIS.get_new(@user)
id = arcgis.send(:sanitize_id, @url)
# Empty ids
expect {
arcgis.send(:get_by_ids, id, [], [{ key: 'value' }])
}.to raise_error InvalidInputDataError
# Empty fields
expect {
arcgis.send(:get_by_ids, id, [1], [])
}.to raise_error InvalidInputDataError
# 'general http error (non-200)'
Typhoeus.stub(/\/arcgis\/rest\//) do
Typhoeus::Response.new(
code: 400,
headers: { 'Content-Type' => 'application/json' },
body: ''
)
end
expect {
arcgis.send(:get_by_ids, id, [1], [{ key: 'value' }])
}.to raise_error DataDownloadError
end
it 'tests the get_by_ids() private method' do
arcgis = Url::ArcGIS.get_new(@user)
id = arcgis.send(:sanitize_id, @url)
Typhoeus.stub(/\/arcgis\/rest\//) do
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_data_01.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: body
)
end
expected_response_data = {
geometryType: "esriGeometryPolygon",
spatialReference: {
"wkid" => 4326,
"latestWkid" => 4326
},
fields: [
{
"name" => "OBJECTID",
"type" => "esriFieldTypeOID",
"alias" => "OBJECTID"
},
{
"name" => "WDPAID",
"type" => "esriFieldTypeInteger",
"alias" => "WDPAID"
},
{
"name" => "NAME",
"type" => "esriFieldTypeString",
"alias" => "NAME",
"length" => 254
}
],
features: [
{"attributes"=>{"OBJECTID"=>1, "NAME"=>"Name of object 1"}, "geometry"=>{"fake"=>"geom"}},
{"attributes"=>{"OBJECTID"=>2, "NAME"=>"Name of object 2"}, "geometry"=>{"fake"=>"geom"}},
{"attributes"=>{"OBJECTID"=>3, "NAME"=>"Name of object 3"}, "geometry"=>{"fake"=>"geom"}}
]
}
ids_to_retrieve = [1,2,3]
# WDPAID also present, but left on purpose untouched
fields_to_retrieve = [{
name: 'OBJECTID',
type: 'esriFieldTypeOID'
},
{
name: 'NAME',
type: 'esriFieldTypeString'
}]
response_data = arcgis.send(:get_by_ids, id, ids_to_retrieve, fields_to_retrieve)
response_data.nil?.should eq false
response_data[:features].length.should eq 3
response_data.should eq expected_response_data
end
it 'tests retrieval of data' do
arcgis = Url::ArcGIS.get_new(@user)
feature_names = []
# Layers request
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/layers/) do |request|
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_layers.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(::JSON.parse(body))
)
end
# Metadata of a layer
Typhoeus.stub(/\/arcgis\/rest\/services\/MyFakeService\/featurename\/0\?f=json/) do
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_metadata_minimal.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: body
)
end
# IDs list of a layer
Typhoeus.stub(/\/arcgis\/rest\/(.*)query\?where=/) do
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_ids_list_01.json"))
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: body
)
end
Typhoeus.stub(/\/arcgis\/rest\/(.*)query$/) do |response|
if response.options[:body][:objectIds].to_i == 1
# First item fetch of a layer
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_data_01.json"))
body = ::JSON.parse(body)
feature_names.push body['features'][0]['attributes']['NAME']
body['features'] = [ body['features'][0] ]
else
# Remaining items fetch of a layer, will not use :objectIds
body = File.read(File.join(File.dirname(__FILE__), "../fixtures/arcgis_data_01.json"))
body = ::JSON.parse(body)
feature_names.push body['features'][1]['attributes']['NAME']
feature_names.push body['features'][2]['attributes']['NAME']
body['features'] = [ body['features'][1], body['features'][2] ]
end
Typhoeus::Response.new(
code: 200,
headers: { 'Content-Type' => 'application/json' },
body: ::JSON.dump(body)
)
end
# 1) Retrieve lists of layers
metadata = arcgis.get_resource_metadata(@url)
# 2) Retrieve metadata of a specific layer (doesn't adds much, but to replicate flow)
item_metadata = arcgis.get_resource_metadata(metadata[:subresources].first[:id])
item_metadata.nil?.should eq false
# No more checks as will fail later if missing something
spatial_ref_expectation = {"wkid"=>4326, "latestWkid"=>4326}
initial_stream_data = arcgis.initial_stream(item_metadata[:id])
initial_stream_data.nil?.should eq false
initial_stream_data = ::JSON.parse(initial_stream_data)
initial_stream_data['geometryType'].should eq 'esriGeometryPolygon'
initial_stream_data['spatialReference'].should eq spatial_ref_expectation
initial_stream_data['fields'].count.should eq 3
initial_stream_data['fields'][0]['name'].should eq 'OBJECTID'
initial_stream_data['fields'][1]['name'].should eq 'WDPAID'
initial_stream_data['fields'][2]['name'].should eq 'NAME'
initial_stream_data['features'].count.should eq 1
initial_stream_data['features'][0]['attributes'].nil?.should eq false
initial_stream_data['features'][0]['attributes']['NAME'].should eq feature_names[0]
initial_stream_data['features'][0]['geometry'].nil?.should eq false
streamed_data = arcgis.stream_resource(item_metadata[:id])
streamed_data.nil?.should eq false
streamed_data = ::JSON.parse(streamed_data)
streamed_data['geometryType'].should eq 'esriGeometryPolygon'
streamed_data['spatialReference'].should eq spatial_ref_expectation
streamed_data['fields'].count.should eq 3
streamed_data['fields'][0]['name'].should eq 'OBJECTID'
streamed_data['fields'][1]['name'].should eq 'WDPAID'
streamed_data['fields'][2]['name'].should eq 'NAME'
streamed_data['features'].count.should eq 2
streamed_data['features'][0]['attributes']['NAME'].should eq feature_names[1]
streamed_data['features'][1]['attributes']['NAME'].should eq feature_names[2]
end
end
end