1
2
3
4
5
6 package org.tailormap.api.persistence.helper;
7
8 import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
9 import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILESET3D;
10 import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
11 import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
12
13 import java.io.IOException;
14 import java.lang.invoke.MethodHandles;
15 import java.net.URI;
16 import java.net.URISyntaxException;
17 import java.net.URL;
18 import java.nio.charset.StandardCharsets;
19 import java.time.Instant;
20 import java.util.ArrayList;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Objects;
25 import java.util.Optional;
26 import java.util.Set;
27 import java.util.function.Predicate;
28 import java.util.stream.Collectors;
29 import org.apache.commons.lang3.StringUtils;
30 import org.geotools.api.data.ServiceInfo;
31 import org.geotools.data.ows.AbstractOpenWebService;
32 import org.geotools.data.ows.Capabilities;
33 import org.geotools.data.ows.OperationType;
34 import org.geotools.http.HTTPClientFinder;
35 import org.geotools.ows.wms.Layer;
36 import org.geotools.ows.wms.WMSCapabilities;
37 import org.geotools.ows.wms.WebMapServer;
38 import org.geotools.ows.wmts.WebMapTileServer;
39 import org.geotools.ows.wmts.model.WMTSLayer;
40 import org.slf4j.Logger;
41 import org.slf4j.LoggerFactory;
42 import org.springframework.beans.factory.annotation.Autowired;
43 import org.springframework.http.MediaType;
44 import org.springframework.stereotype.Service;
45 import org.springframework.util.CollectionUtils;
46 import org.tailormap.api.configuration.TailormapConfig;
47 import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
48 import org.tailormap.api.geotools.WMSServiceExceptionUtil;
49 import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
50 import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
51 import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
52 import org.tailormap.api.persistence.GeoService;
53 import org.tailormap.api.persistence.TMFeatureSource;
54 import org.tailormap.api.persistence.json.GeoServiceLayer;
55 import org.tailormap.api.persistence.json.ServiceAuthentication;
56 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
57 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
58 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
59 import org.tailormap.api.persistence.json.TMServiceCaps;
60 import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
61 import org.tailormap.api.persistence.json.TMServiceInfo;
62 import org.tailormap.api.persistence.json.WMSStyle;
63 import org.tailormap.api.repository.FeatureSourceRepository;
64
65 @Service
66 public class GeoServiceHelper {
67
68 private static final Logger logger =
69 LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
70 private final TailormapConfig tailormapConfig;
71 private final FeatureSourceRepository featureSourceRepository;
72
73 @Autowired
74 public GeoServiceHelper(TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
75 this.tailormapConfig = tailormapConfig;
76 this.featureSourceRepository = featureSourceRepository;
77 }
78
79 public org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
80
81 if (StringUtils.isBlank(url)) {
82 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
83 }
84 if (url.contains("/arcgis/")) {
85 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
86 }
87 if (url.contains("/geoserver/")) {
88 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
89 }
90 if (url.contains("/mapserv")) {
91 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
92 }
93 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
94 }
95
96 public void loadServiceCapabilities(GeoService geoService) throws Exception {
97
98 if (geoService.getProtocol() == XYZ) {
99 setXyzCapabilities(geoService);
100 return;
101 }
102
103 if (geoService.getProtocol() == TILESET3D) {
104 setTileset3DCapabilities(geoService);
105 return;
106 }
107
108 ResponseTeeingHTTPClient client = new ResponseTeeingHTTPClient(
109 HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
110
111 ServiceAuthentication auth = geoService.getAuthentication();
112 if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
113 client.setUser(auth.getUsername());
114 client.setPassword(auth.getPassword());
115 }
116
117 client.setReadTimeout(this.tailormapConfig.getTimeout());
118 client.setConnectTimeout(this.tailormapConfig.getTimeout());
119 client.setTryGzip(true);
120
121 logger.info(
122 "Get capabilities for {} {} from URL {}",
123 geoService.getProtocol(),
124 geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
125 geoService.getUrl());
126
127
128
129 switch (geoService.getProtocol()) {
130 case WMS:
131 loadWMSCapabilities(geoService, client);
132 break;
133 case WMTS:
134 loadWMTSCapabilities(geoService, client);
135 break;
136 default:
137 throw new UnsupportedOperationException(
138 "Unsupported geo service protocol: " + geoService.getProtocol());
139 }
140
141 if (geoService.getTitle() == null) {
142 geoService.setTitle(Optional.ofNullable(geoService.getServiceCapabilities())
143 .map(TMServiceCaps::getServiceInfo)
144 .map(TMServiceInfo::getTitle)
145 .orElse(null));
146 }
147
148 if (logger.isDebugEnabled()) {
149 logger.debug("Loaded service layers: {}", geoService.getLayers());
150 } else {
151 logger.info(
152 "Loaded service layers: {}",
153 geoService.getLayers().stream()
154 .filter(Predicate.not(GeoServiceLayer::getVirtual))
155 .map(GeoServiceLayer::getName)
156 .collect(Collectors.toList()));
157 }
158 }
159
160 private static void setXyzCapabilities(GeoService geoService) {
161 geoService.setLayers(List.of(new GeoServiceLayer()
162 .id("0")
163 .root(true)
164 .name("xyz")
165 .title(geoService.getTitle())
166 .crs(Set.of(geoService.getSettings().getXyzCrs()))
167 .virtual(false)
168 .queryable(false)));
169 }
170
171 private static void setTileset3DCapabilities(GeoService geoService) {
172 geoService.setLayers(List.of(new GeoServiceLayer()
173 .id("0")
174 .root(true)
175 .name("Tileset3D")
176 .title(geoService.getTitle())
177 .virtual(false)
178 .queryable(false)));
179 }
180
181 private void setServiceInfo(
182 GeoService geoService,
183 ResponseTeeingHTTPClient client,
184 AbstractOpenWebService<? extends Capabilities, Layer> ows) {
185 geoService.setCapabilities(client.getLatestResponseCopy());
186 geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
187
188 geoService.setCapabilitiesFetched(Instant.now());
189
190 ServiceInfo info = ows.getInfo();
191
192 TMServiceCaps caps = new TMServiceCaps();
193 geoService.setServiceCapabilities(caps);
194
195 caps.setCorsAllowOrigin(client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
196
197 if (info != null) {
198 if (StringUtils.isBlank(geoService.getTitle())) {
199 geoService.setTitle(info.getTitle());
200 }
201
202 caps.serviceInfo(new TMServiceInfo()
203 .keywords(info.getKeywords())
204 .description(info.getDescription())
205 .title(info.getTitle())
206 .publisher(info.getPublisher())
207 .schema(info.getSchema())
208 .source(info.getSource()));
209
210 geoService.setAdvertisedUrl(info.getSource().toString());
211 } else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
212 org.geotools.data.ows.Service service = ows.getCapabilities().getService();
213
214 if (StringUtils.isBlank(geoService.getTitle())) {
215 geoService.setTitle(service.getTitle());
216 }
217 caps.setServiceInfo(new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
218 }
219 }
220
221 private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
222 return new GeoServiceLayer()
223 .id(String.valueOf(layers.indexOf(l)))
224 .name(l.getName())
225 .root(l.getParent() == null)
226 .title(l.getTitle())
227 .maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
228 .minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
229 .virtual(l.getName() == null)
230 .crs(l.getSrs())
231 .latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
232 .styles(l.getStyles().stream()
233 .map(gtStyle -> {
234 WMSStyle style = new WMSStyle()
235 .name(gtStyle.getName())
236 .title(Optional.ofNullable(gtStyle.getTitle())
237 .map(Objects::toString)
238 .orElse(null))
239 .abstractText(Optional.ofNullable(gtStyle.getAbstract())
240 .map(Objects::toString)
241 .orElse(null));
242 try {
243 List<?> legendURLs = gtStyle.getLegendURLs();
244
245 if (legendURLs != null && !legendURLs.isEmpty() && legendURLs.get(0) != null) {
246 style.legendURL(new URI((String) legendURLs.get(0)));
247 }
248 } catch (URISyntaxException ignored) {
249
250
251 }
252 return style;
253 })
254 .collect(Collectors.toList()))
255 .queryable(l.isQueryable())
256 .abstractText(l.get_abstract())
257 .children(l.getLayerChildren().stream()
258 .map(layers::indexOf)
259 .map(String::valueOf)
260 .collect(Collectors.toList()));
261 }
262
263 private void addLayerRecursive(
264 GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
265 GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
266
267
268
269 geoServiceLayer.getCrs().removeAll(parentCrs);
270 geoService.getLayers().add(geoServiceLayer);
271 for (Layer l : layer.getLayerChildren()) {
272 addLayerRecursive(geoService, layers, l, layer.getSrs());
273 }
274 }
275
276 void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
277 WebMapServer wms;
278 try {
279 wms = new WebMapServer(new URL(geoService.getUrl()), client);
280 } catch (ClassCastException | IllegalStateException e) {
281
282
283
284
285
286
287
288
289
290 String contentType = client.getLatestResponse().getContentType();
291 if (contentType != null && contentType.contains("text/xml")) {
292 String wmsException =
293 WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
294 throw new Exception("Error loading WMS capabilities: "
295 + (wmsException != null
296 ? wmsException
297 : new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
298 } else {
299 throw e;
300 }
301 } catch (IOException e) {
302
303
304
305 if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
306 throw new Exception(
307 "Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
308 } else {
309 throw e;
310 }
311 }
312
313 OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
314 OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
315
316 if (getMap == null) {
317 throw new Exception("Service does not support GetMap");
318 }
319
320 setServiceInfo(geoService, client, wms);
321
322 WMSCapabilities wmsCapabilities = wms.getCapabilities();
323
324
325
326 geoService
327 .getServiceCapabilities()
328 .capabilities(new TMServiceCapsCapabilities()
329 .version(wmsCapabilities.getVersion())
330 .updateSequence(wmsCapabilities.getUpdateSequence())
331 .abstractText(wmsCapabilities.getService().get_abstract())
332 .request(new TMServiceCapabilitiesRequest()
333 .getMap(new TMServiceCapabilitiesRequestGetMap()
334 .formats(Set.copyOf(getMap.getFormats())))
335 .getFeatureInfo(
336 getFeatureInfo == null
337 ? null
338 : new TMServiceCapabilitiesRequestGetFeatureInfo()
339 .formats(Set.copyOf(getFeatureInfo.getFormats())))
340 .describeLayer(
341 wms.getCapabilities().getRequest().getDescribeLayer() != null)));
342
343 if (logger.isDebugEnabled()) {
344 logger.debug("Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
345 } else {
346 logger.info(
347 "Loaded capabilities from \"{}\", title: \"{}\"",
348 geoService.getUrl(),
349 geoService.getServiceCapabilities() != null
350 && geoService.getServiceCapabilities().getServiceInfo() != null
351 ? geoService
352 .getServiceCapabilities()
353 .getServiceInfo()
354 .getTitle()
355 : "(none)");
356 }
357 geoService.setLayers(new ArrayList<>());
358 addLayerRecursive(
359 geoService,
360 wms.getCapabilities().getLayerList(),
361 wms.getCapabilities().getLayer(),
362 Collections.emptySet());
363 }
364
365 void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
366 WebMapTileServer wmts = new WebMapTileServer(new URL(geoService.getUrl()), client);
367 setServiceInfo(geoService, client, wmts);
368
369
370
371 List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
372 geoService.setLayers(
373 layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
374 }
375
376 public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
377
378
379 if (CollectionUtils.isEmpty(geoService.getLayers())) {
380 return Collections.emptyMap();
381 }
382
383
384
385
386
387
388
389 List<String> layers = geoService.getLayers().stream()
390 .filter(l -> !l.getVirtual())
391 .map(GeoServiceLayer::getName)
392 .filter(n -> {
393
394 boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
395 if (!noWhitespace) {
396 logger.warn(
397 "Not doing WFS DescribeLayer request for layer name with space: \"{}\" of WMS {}",
398 n,
399 geoService.getUrl());
400 }
401 return noWhitespace;
402 })
403 .collect(Collectors.toList());
404
405
406 Map<String, SimpleWFSLayerDescription> descriptions =
407 SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
408
409 for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
410 String layerName = entry.getKey();
411 SimpleWFSLayerDescription description = entry.getValue();
412 if (description.typeNames().size() == 1 && layerName.equals(description.getFirstTypeName())) {
413 logger.info(
414 "layer \"{}\" linked to feature type with same name of WFS {}",
415 layerName,
416 description.wfsUrl());
417 } else {
418 logger.info(
419 "layer \"{}\" -> feature type(s) {} of WFS {}",
420 layerName,
421 description.typeNames(),
422 description.wfsUrl());
423 }
424 }
425 return descriptions;
426 }
427
428 public void findAndSaveRelatedWFS(GeoService geoService) {
429 if (geoService.getProtocol() != WMS) {
430 throw new IllegalArgumentException();
431 }
432
433
434
435 Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
436
437 wfsByLayer.values().stream()
438 .map(SimpleWFSLayerDescription::wfsUrl)
439 .distinct()
440 .forEach(url -> {
441 TMFeatureSource fs = featureSourceRepository.findByUrl(url);
442 if (fs == null) {
443 fs = new TMFeatureSource()
444 .setProtocol(WFS)
445 .setUrl(url)
446 .setTitle("WFS for " + geoService.getTitle())
447 .setLinkedService(geoService);
448 try {
449 new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
450 } catch (IOException e) {
451 String msg = String.format(
452 "Error loading WFS from URL %s: %s: %s", url, e.getClass(), e.getMessage());
453 if (logger.isTraceEnabled()) {
454 logger.error(msg, e);
455 } else {
456 logger.error(msg);
457 }
458 }
459 featureSourceRepository.save(fs);
460 }
461 });
462 }
463
464
465
466
467
468
469
470
471
472
473
474 public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
475 if (serviceLayer.getRoot()) {
476
477
478 return serviceLayer.getStyles().stream()
479 .findFirst()
480 .map(WMSStyle::getLegendURL)
481 .orElse(null);
482 }
483
484 final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
485 if (allOurLayersStyles.size() == 1) {
486 return allOurLayersStyles.get(0).getLegendURL();
487 }
488
489 service.getLayers().stream()
490 .filter(layer -> !layer.equals(serviceLayer))
491 .forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
492
493 return allOurLayersStyles.stream()
494 .findFirst()
495 .map(WMSStyle::getLegendURL)
496 .orElse(null);
497 }
498 }