1
2
3
4
5
6 package org.tailormap.api.persistence.helper;
7
8 import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
9 import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILESET3D;
10 import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
11 import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
12
13 import java.io.IOException;
14 import java.lang.invoke.MethodHandles;
15 import java.net.URI;
16 import java.net.URISyntaxException;
17 import java.net.URL;
18 import java.nio.charset.StandardCharsets;
19 import java.time.Instant;
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.Collections;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Objects;
26 import java.util.Optional;
27 import java.util.Set;
28 import java.util.function.Predicate;
29 import java.util.stream.Collectors;
30 import org.apache.commons.lang3.StringUtils;
31 import org.geotools.api.data.ServiceInfo;
32 import org.geotools.data.ows.AbstractOpenWebService;
33 import org.geotools.data.ows.Capabilities;
34 import org.geotools.data.ows.OperationType;
35 import org.geotools.http.HTTPClientFinder;
36 import org.geotools.ows.wms.Layer;
37 import org.geotools.ows.wms.WMSCapabilities;
38 import org.geotools.ows.wms.WebMapServer;
39 import org.geotools.ows.wmts.WebMapTileServer;
40 import org.geotools.ows.wmts.model.WMTSLayer;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43 import org.springframework.beans.factory.annotation.Autowired;
44 import org.springframework.http.MediaType;
45 import org.springframework.stereotype.Service;
46 import org.springframework.util.CollectionUtils;
47 import org.tailormap.api.configuration.TailormapConfig;
48 import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
49 import org.tailormap.api.geotools.WMSServiceExceptionUtil;
50 import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
51 import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
52 import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
53 import org.tailormap.api.persistence.GeoService;
54 import org.tailormap.api.persistence.TMFeatureSource;
55 import org.tailormap.api.persistence.json.GeoServiceLayer;
56 import org.tailormap.api.persistence.json.ServiceAuthentication;
57 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
58 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
59 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
60 import org.tailormap.api.persistence.json.TMServiceCaps;
61 import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
62 import org.tailormap.api.persistence.json.TMServiceInfo;
63 import org.tailormap.api.persistence.json.WMSStyle;
64 import org.tailormap.api.repository.FeatureSourceRepository;
65
66 @Service
67 public class GeoServiceHelper {
68
69 private static final Logger logger =
70 LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
71 private final TailormapConfig tailormapConfig;
72 private final FeatureSourceRepository featureSourceRepository;
73
74 @Autowired
75 public GeoServiceHelper(
76 TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
77 this.tailormapConfig = tailormapConfig;
78 this.featureSourceRepository = featureSourceRepository;
79 }
80
81 public org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
82
83 if (StringUtils.isBlank(url)) {
84 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
85 }
86 if (url.contains("/arcgis/")) {
87 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
88 }
89 if (url.contains("/geoserver/")) {
90 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
91 }
92 if (url.contains("/mapserv")) {
93 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
94 }
95 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
96 }
97
98 public void loadServiceCapabilities(GeoService geoService) throws Exception {
99
100 if (geoService.getProtocol() == XYZ) {
101 setXyzCapabilities(geoService);
102 return;
103 }
104
105 if (geoService.getProtocol() == TILESET3D) {
106 setTileset3DCapabilities(geoService);
107 return;
108 }
109
110 ResponseTeeingHTTPClient client =
111 new ResponseTeeingHTTPClient(
112 HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
113
114 ServiceAuthentication auth = geoService.getAuthentication();
115 if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
116 client.setUser(auth.getUsername());
117 client.setPassword(auth.getPassword());
118 }
119
120 client.setReadTimeout(this.tailormapConfig.getTimeout());
121 client.setConnectTimeout(this.tailormapConfig.getTimeout());
122 client.setTryGzip(true);
123
124 logger.info(
125 "Get capabilities for {} {} from URL {}",
126 geoService.getProtocol(),
127 geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
128 geoService.getUrl());
129
130
131
132 switch (geoService.getProtocol()) {
133 case WMS:
134 loadWMSCapabilities(geoService, client);
135 break;
136 case WMTS:
137 loadWMTSCapabilities(geoService, client);
138 break;
139 default:
140 throw new UnsupportedOperationException(
141 "Unsupported geo service protocol: " + geoService.getProtocol());
142 }
143
144 if (geoService.getTitle() == null) {
145 geoService.setTitle(
146 Optional.ofNullable(geoService.getServiceCapabilities())
147 .map(TMServiceCaps::getServiceInfo)
148 .map(TMServiceInfo::getTitle)
149 .orElse(null));
150 }
151
152 if (logger.isDebugEnabled()) {
153 logger.debug("Loaded service layers: {}", geoService.getLayers());
154 } else {
155 logger.info(
156 "Loaded service layers: {}",
157 geoService.getLayers().stream()
158 .filter(Predicate.not(GeoServiceLayer::getVirtual))
159 .map(GeoServiceLayer::getName)
160 .collect(Collectors.toList()));
161 }
162 }
163
164 private static void setXyzCapabilities(GeoService geoService) {
165 geoService.setLayers(
166 List.of(
167 new GeoServiceLayer()
168 .id("0")
169 .root(true)
170 .name("xyz")
171 .title(geoService.getTitle())
172 .crs(Set.of(geoService.getSettings().getXyzCrs()))
173 .virtual(false)
174 .queryable(false)));
175 }
176
177 private static void setTileset3DCapabilities(GeoService geoService) {
178 geoService.setLayers(
179 List.of(
180 new GeoServiceLayer()
181 .id("0")
182 .root(true)
183 .name("Tileset3D")
184 .title(geoService.getTitle())
185 .virtual(false)
186 .queryable(false)));
187 }
188
189 private void setServiceInfo(
190 GeoService geoService,
191 ResponseTeeingHTTPClient client,
192 AbstractOpenWebService<? extends Capabilities, Layer> ows) {
193 geoService.setCapabilities(client.getLatestResponseCopy());
194 geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
195
196 geoService.setCapabilitiesFetched(Instant.now());
197
198 ServiceInfo info = ows.getInfo();
199
200 TMServiceCaps caps = new TMServiceCaps();
201 geoService.setServiceCapabilities(caps);
202
203 caps.setCorsAllowOrigin(
204 client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
205
206 if (info != null) {
207 if (StringUtils.isBlank(geoService.getTitle())) {
208 geoService.setTitle(info.getTitle());
209 }
210
211 caps.serviceInfo(
212 new TMServiceInfo()
213 .keywords(info.getKeywords())
214 .description(info.getDescription())
215 .title(info.getTitle())
216 .publisher(info.getPublisher())
217 .schema(info.getSchema())
218 .source(info.getSource()));
219
220 geoService.setAdvertisedUrl(info.getSource().toString());
221 } else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
222 org.geotools.data.ows.Service service = ows.getCapabilities().getService();
223
224 if (StringUtils.isBlank(geoService.getTitle())) {
225 geoService.setTitle(service.getTitle());
226 }
227 caps.setServiceInfo(
228 new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
229 }
230 }
231
232 private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
233 return new GeoServiceLayer()
234 .id(String.valueOf(layers.indexOf(l)))
235 .name(l.getName())
236 .root(l.getParent() == null)
237 .title(l.getTitle())
238 .maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
239 .minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
240 .virtual(l.getName() == null)
241 .crs(l.getSrs())
242 .latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
243 .styles(
244 l.getStyles().stream()
245 .map(
246 gtStyle -> {
247 WMSStyle style =
248 new WMSStyle()
249 .name(gtStyle.getName())
250 .title(
251 Optional.ofNullable(gtStyle.getTitle())
252 .map(Objects::toString)
253 .orElse(null))
254 .abstractText(
255 Optional.ofNullable(gtStyle.getAbstract())
256 .map(Objects::toString)
257 .orElse(null));
258 try {
259 @SuppressWarnings("unchecked")
260 List legendURLs = gtStyle.getLegendURLs();
261
262 if (legendURLs != null
263 && !legendURLs.isEmpty()
264 && legendURLs.get(0) != null) {
265 style.legendURL(new URI((String) legendURLs.get(0)));
266 }
267 } catch (URISyntaxException ignored) {
268
269
270 }
271 return style;
272 })
273 .collect(Collectors.toList()))
274 .queryable(l.isQueryable())
275 .abstractText(l.get_abstract())
276 .children(
277 l.getLayerChildren().stream()
278 .map(layers::indexOf)
279 .map(String::valueOf)
280 .collect(Collectors.toList()));
281 }
282
283 private void addLayerRecursive(
284 GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
285 GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
286
287
288
289 geoServiceLayer.getCrs().removeAll(parentCrs);
290 geoService.getLayers().add(geoServiceLayer);
291 for (Layer l : layer.getLayerChildren()) {
292 addLayerRecursive(geoService, layers, l, layer.getSrs());
293 }
294 }
295
296 void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client)
297 throws Exception {
298 WebMapServer wms;
299 try {
300 wms = new WebMapServer(new URL(geoService.getUrl()), client);
301 } catch (ClassCastException | IllegalStateException e) {
302
303
304
305
306
307
308
309
310
311 String contentType = client.getLatestResponse().getContentType();
312 if (contentType != null && contentType.contains("text/xml")) {
313 String wmsException =
314 WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
315 throw new Exception(
316 "Error loading WMS capabilities: "
317 + (wmsException != null
318 ? wmsException
319 : new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
320 } else {
321 throw e;
322 }
323 } catch (IOException e) {
324
325
326
327 if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
328 throw new Exception(
329 "Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
330 } else {
331 throw e;
332 }
333 }
334
335 OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
336 OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
337
338 if (getMap == null) {
339 throw new Exception("Service does not support GetMap");
340 }
341
342 setServiceInfo(geoService, client, wms);
343
344 WMSCapabilities wmsCapabilities = wms.getCapabilities();
345
346
347
348 geoService
349 .getServiceCapabilities()
350 .capabilities(
351 new TMServiceCapsCapabilities()
352 .version(wmsCapabilities.getVersion())
353 .updateSequence(wmsCapabilities.getUpdateSequence())
354 .abstractText(wmsCapabilities.getService().get_abstract())
355 .request(
356 new TMServiceCapabilitiesRequest()
357 .getMap(
358 new TMServiceCapabilitiesRequestGetMap()
359 .formats(Set.copyOf(getMap.getFormats())))
360 .getFeatureInfo(
361 getFeatureInfo == null
362 ? null
363 : new TMServiceCapabilitiesRequestGetFeatureInfo()
364 .formats(Set.copyOf(getFeatureInfo.getFormats())))
365 .describeLayer(
366 wms.getCapabilities().getRequest().getDescribeLayer() != null)));
367
368 if (logger.isDebugEnabled()) {
369 logger.debug(
370 "Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
371 } else {
372 logger.info(
373 "Loaded capabilities from \"{}\", title: \"{}\"",
374 geoService.getUrl(),
375 geoService.getServiceCapabilities() != null
376 && geoService.getServiceCapabilities().getServiceInfo() != null
377 ? geoService.getServiceCapabilities().getServiceInfo().getTitle()
378 : "(none)");
379 }
380 geoService.setLayers(new ArrayList<>());
381 addLayerRecursive(
382 geoService,
383 wms.getCapabilities().getLayerList(),
384 wms.getCapabilities().getLayer(),
385 Collections.emptySet());
386 }
387
388 void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client)
389 throws Exception {
390 WebMapTileServer wmts = new WebMapTileServer(new URL(geoService.getUrl()), client);
391 setServiceInfo(geoService, client, wmts);
392
393
394
395 List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
396 geoService.setLayers(
397 layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
398 }
399
400 public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
401
402
403 if (CollectionUtils.isEmpty(geoService.getLayers())) {
404 return Collections.emptyMap();
405 }
406
407
408
409
410
411
412
413 List<String> layers =
414 geoService.getLayers().stream()
415 .filter(l -> !l.getVirtual())
416 .map(GeoServiceLayer::getName)
417 .filter(
418 n -> {
419
420 boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
421 if (!noWhitespace) {
422 logger.warn(
423 String.format(
424 "Not doing WFS DescribeLayer request for layer name with space: \"%s\" of WMS %s",
425 n, geoService.getUrl()));
426 }
427 return noWhitespace;
428 })
429 .collect(Collectors.toList());
430
431
432 Map<String, SimpleWFSLayerDescription> descriptions =
433 SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
434
435 for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
436 String layerName = entry.getKey();
437 SimpleWFSLayerDescription description = entry.getValue();
438 if (description.getTypeNames().length == 1
439 && layerName.equals(description.getFirstTypeName())) {
440 logger.info(
441 String.format(
442 "layer \"%s\" linked to feature type with same name of WFS %s",
443 layerName, description.getWfsUrl()));
444 } else {
445 logger.info(
446 String.format(
447 "layer \"%s\" -> feature type(s) %s of WFS %s",
448 layerName, Arrays.toString(description.getTypeNames()), description.getWfsUrl()));
449 }
450 }
451 return descriptions;
452 }
453
454 public void findAndSaveRelatedWFS(GeoService geoService) {
455 if (geoService.getProtocol() != WMS) {
456 throw new IllegalArgumentException();
457 }
458
459
460
461 Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
462
463 wfsByLayer.values().stream()
464 .map(SimpleWFSLayerDescription::getWfsUrl)
465 .distinct()
466 .forEach(
467 url -> {
468 TMFeatureSource fs = featureSourceRepository.findByUrl(url);
469 if (fs == null) {
470 fs =
471 new TMFeatureSource()
472 .setProtocol(WFS)
473 .setUrl(url)
474 .setTitle("WFS for " + geoService.getTitle())
475 .setLinkedService(geoService);
476 try {
477 new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
478 } catch (IOException e) {
479 String msg =
480 String.format(
481 "Error loading WFS from URL %s: %s: %s",
482 url, e.getClass(), e.getMessage());
483 if (logger.isTraceEnabled()) {
484 logger.error(msg, e);
485 } else {
486 logger.error(msg);
487 }
488 }
489 featureSourceRepository.save(fs);
490 }
491 });
492 }
493
494
495
496
497
498
499
500
501
502
503
504 public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
505 if (serviceLayer.getRoot()) {
506
507
508 return serviceLayer.getStyles().stream().findFirst().map(WMSStyle::getLegendURL).orElse(null);
509 }
510
511 final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
512 if (allOurLayersStyles.size() == 1) {
513 return allOurLayersStyles.get(0).getLegendURL();
514 }
515
516 service.getLayers().stream()
517 .filter(layer -> !layer.equals(serviceLayer))
518 .forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
519
520 return allOurLayersStyles.stream().findFirst().map(WMSStyle::getLegendURL).orElse(null);
521 }
522 }