1
2
3
4
5
6 package org.tailormap.api.persistence.helper;
7
8 import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
9 import static org.tailormap.api.persistence.json.GeoServiceProtocol.QUANTIZEDMESH;
10 import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILES3D;
11 import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
12 import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
13
14 import java.io.IOException;
15 import java.lang.invoke.MethodHandles;
16 import java.net.URI;
17 import java.net.URISyntaxException;
18 import java.nio.charset.StandardCharsets;
19 import java.time.Instant;
20 import java.util.ArrayList;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Objects;
25 import java.util.Optional;
26 import java.util.Set;
27 import java.util.function.Predicate;
28 import java.util.stream.Collectors;
29 import org.apache.commons.lang3.StringUtils;
30 import org.geotools.api.data.ServiceInfo;
31 import org.geotools.data.ows.AbstractOpenWebService;
32 import org.geotools.data.ows.Capabilities;
33 import org.geotools.data.ows.OperationType;
34 import org.geotools.http.HTTPClientFinder;
35 import org.geotools.ows.wms.Layer;
36 import org.geotools.ows.wms.WMSCapabilities;
37 import org.geotools.ows.wms.WebMapServer;
38 import org.geotools.ows.wmts.WebMapTileServer;
39 import org.geotools.ows.wmts.model.WMTSLayer;
40 import org.slf4j.Logger;
41 import org.slf4j.LoggerFactory;
42 import org.springframework.beans.factory.annotation.Autowired;
43 import org.springframework.http.MediaType;
44 import org.springframework.stereotype.Service;
45 import org.springframework.util.CollectionUtils;
46 import org.springframework.web.util.UriComponentsBuilder;
47 import org.tailormap.api.configuration.TailormapConfig;
48 import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
49 import org.tailormap.api.geotools.WMSServiceExceptionUtil;
50 import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
51 import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
52 import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
53 import org.tailormap.api.persistence.GeoService;
54 import org.tailormap.api.persistence.TMFeatureSource;
55 import org.tailormap.api.persistence.json.GeoServiceLayer;
56 import org.tailormap.api.persistence.json.ServiceAuthentication;
57 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
58 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
59 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
60 import org.tailormap.api.persistence.json.TMServiceCaps;
61 import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
62 import org.tailormap.api.persistence.json.TMServiceInfo;
63 import org.tailormap.api.persistence.json.WMSStyle;
64 import org.tailormap.api.repository.FeatureSourceRepository;
65
66 @Service
67 public class GeoServiceHelper {
68
69 private static final Logger logger =
70 LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
71 private final TailormapConfig tailormapConfig;
72 private final FeatureSourceRepository featureSourceRepository;
73
74 @Autowired
75 public GeoServiceHelper(TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
76 this.tailormapConfig = tailormapConfig;
77 this.featureSourceRepository = featureSourceRepository;
78 }
79
80 public static org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
81
82 if (StringUtils.isBlank(url)) {
83 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
84 }
85 if (url.contains("/arcgis/")) {
86 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
87 }
88 if (url.contains("/geoserver/")) {
89 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
90 }
91 if (url.contains("/mapserv")) {
92 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
93 }
94 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
95 }
96
97 public static String getWmsRequest(String uri) {
98 return getWmsRequest(uri == null ? null : URI.create(uri));
99 }
100
101
102
103
104
105
106
107 public static String getWmsRequest(URI uri) {
108 if (uri == null || uri.getQuery() == null) {
109 return null;
110 }
111 return UriComponentsBuilder.fromUri(uri).build().getQueryParams().entrySet().stream()
112 .filter(entry -> "request".equalsIgnoreCase(entry.getKey()))
113 .map(entry -> entry.getValue().getFirst())
114 .findFirst()
115 .orElse(null);
116 }
117
118 public void loadServiceCapabilities(GeoService geoService) throws Exception {
119
120 if (geoService.getProtocol() == XYZ) {
121 setXyzCapabilities(geoService);
122 return;
123 }
124
125 if (geoService.getProtocol() == TILES3D) {
126 set3DTilesCapabilities(geoService);
127 return;
128 }
129
130 if (geoService.getProtocol() == QUANTIZEDMESH) {
131 setQuantizedMeshCapabilities(geoService);
132 return;
133 }
134
135 ResponseTeeingHTTPClient client = new ResponseTeeingHTTPClient(
136 HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
137
138 ServiceAuthentication auth = geoService.getAuthentication();
139 if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
140 client.setUser(auth.getUsername());
141 client.setPassword(auth.getPassword());
142 }
143
144 client.setReadTimeout(this.tailormapConfig.getTimeout());
145 client.setConnectTimeout(this.tailormapConfig.getTimeout());
146 client.setTryGzip(true);
147
148 logger.info(
149 "Get capabilities for {} {} from URL {}",
150 geoService.getProtocol(),
151 geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
152 geoService.getUrl());
153
154
155
156 switch (geoService.getProtocol()) {
157 case WMS -> loadWMSCapabilities(geoService, client);
158 case WMTS -> loadWMTSCapabilities(geoService, client);
159 default ->
160 throw new UnsupportedOperationException(
161 "Unsupported geo service protocol: " + geoService.getProtocol());
162 }
163
164 if (geoService.getTitle() == null) {
165 geoService.setTitle(Optional.ofNullable(geoService.getServiceCapabilities())
166 .map(TMServiceCaps::getServiceInfo)
167 .map(TMServiceInfo::getTitle)
168 .orElse(null));
169 }
170
171 if (logger.isDebugEnabled()) {
172 logger.debug("Loaded service layers: {}", geoService.getLayers());
173 } else {
174 logger.info(
175 "Loaded service layers: {}",
176 geoService.getLayers().stream()
177 .filter(Predicate.not(GeoServiceLayer::getVirtual))
178 .map(GeoServiceLayer::getName)
179 .collect(Collectors.toList()));
180 }
181 }
182
183 private static void setXyzCapabilities(GeoService geoService) {
184 geoService.setLayers(List.of(new GeoServiceLayer()
185 .id("0")
186 .root(true)
187 .name("xyz")
188 .title(geoService.getTitle())
189 .crs(Set.of(geoService.getSettings().getXyzCrs()))
190 .virtual(false)
191 .queryable(false)));
192 }
193
194 private static void set3DTilesCapabilities(GeoService geoService) {
195 geoService.setLayers(List.of(new GeoServiceLayer()
196 .id("0")
197 .root(true)
198 .name("tiles3d")
199 .title(geoService.getTitle())
200 .virtual(false)
201 .queryable(false)));
202 }
203
204 private static void setQuantizedMeshCapabilities(GeoService geoService) {
205 geoService.setLayers(List.of(new GeoServiceLayer()
206 .id("0")
207 .root(true)
208 .name("quantizedmesh")
209 .title(geoService.getTitle())
210 .virtual(false)
211 .queryable(false)));
212 }
213
214 private void setServiceInfo(
215 GeoService geoService,
216 ResponseTeeingHTTPClient client,
217 AbstractOpenWebService<? extends Capabilities, Layer> ows) {
218 geoService.setCapabilities(client.getLatestResponseCopy());
219 geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
220
221 geoService.setCapabilitiesFetched(Instant.now());
222
223 ServiceInfo info = ows.getInfo();
224
225 TMServiceCaps caps = new TMServiceCaps();
226 geoService.setServiceCapabilities(caps);
227
228 caps.setCorsAllowOrigin(client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
229
230 if (info != null) {
231 if (StringUtils.isBlank(geoService.getTitle())) {
232 geoService.setTitle(info.getTitle());
233 }
234
235 caps.serviceInfo(new TMServiceInfo()
236 .keywords(info.getKeywords())
237 .description(info.getDescription())
238 .title(info.getTitle())
239 .publisher(info.getPublisher())
240 .schema(info.getSchema())
241 .source(info.getSource()));
242
243 geoService.setAdvertisedUrl(info.getSource().toString());
244 } else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
245 org.geotools.data.ows.Service service = ows.getCapabilities().getService();
246
247 if (StringUtils.isBlank(geoService.getTitle())) {
248 geoService.setTitle(service.getTitle());
249 }
250 caps.setServiceInfo(new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
251 }
252 }
253
254 private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
255 return new GeoServiceLayer()
256 .id(String.valueOf(layers.indexOf(l)))
257 .name(l.getName())
258 .root(l.getParent() == null)
259 .title(l.getTitle())
260 .maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
261 .minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
262 .virtual(l.getName() == null)
263 .crs(l.getSrs())
264 .latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
265 .styles(l.getStyles().stream()
266 .map(gtStyle -> {
267 WMSStyle style = new WMSStyle()
268 .name(gtStyle.getName())
269 .title(Optional.ofNullable(gtStyle.getTitle())
270 .map(Objects::toString)
271 .orElse(null))
272 .abstractText(Optional.ofNullable(gtStyle.getAbstract())
273 .map(Objects::toString)
274 .orElse(null));
275 try {
276 List<?> legendURLs = gtStyle.getLegendURLs();
277
278 if (legendURLs != null && !legendURLs.isEmpty() && legendURLs.getFirst() != null) {
279 style.legendURL(new URI((String) legendURLs.getFirst()));
280 }
281 } catch (URISyntaxException ignored) {
282
283
284 }
285 return style;
286 })
287 .collect(Collectors.toList()))
288 .queryable(l.isQueryable())
289 .abstractText(l.get_abstract())
290 .children(l.getLayerChildren().stream()
291 .map(layers::indexOf)
292 .map(String::valueOf)
293 .collect(Collectors.toList()));
294 }
295
296 private void addLayerRecursive(
297 GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
298 GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
299
300
301
302 geoServiceLayer.getCrs().removeAll(parentCrs);
303 geoService.getLayers().add(geoServiceLayer);
304 for (Layer l : layer.getLayerChildren()) {
305 addLayerRecursive(geoService, layers, l, layer.getSrs());
306 }
307 }
308
309 void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
310 WebMapServer wms;
311 try {
312 wms = new WebMapServer(new URI(geoService.getUrl()).toURL(), client);
313 } catch (ClassCastException | IllegalStateException e) {
314
315
316
317
318
319
320
321
322
323 String contentType = client.getLatestResponse().getContentType();
324 if (contentType != null && contentType.contains("text/xml")) {
325 String wmsException =
326 WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
327 throw new Exception("Error loading WMS capabilities: "
328 + (wmsException != null
329 ? wmsException
330 : new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
331 } else {
332 throw e;
333 }
334 } catch (IOException e) {
335
336
337
338 if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
339 throw new Exception(
340 "Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
341 } else {
342 throw e;
343 }
344 }
345
346 OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
347 OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
348
349 if (getMap == null) {
350 throw new Exception("Service does not support GetMap");
351 }
352
353 setServiceInfo(geoService, client, wms);
354
355 WMSCapabilities wmsCapabilities = wms.getCapabilities();
356
357
358
359 geoService
360 .getServiceCapabilities()
361 .capabilities(new TMServiceCapsCapabilities()
362 .version(wmsCapabilities.getVersion())
363 .updateSequence(wmsCapabilities.getUpdateSequence())
364 .abstractText(wmsCapabilities.getService().get_abstract())
365 .request(new TMServiceCapabilitiesRequest()
366 .getMap(new TMServiceCapabilitiesRequestGetMap()
367 .formats(Set.copyOf(getMap.getFormats())))
368 .getFeatureInfo(
369 getFeatureInfo == null
370 ? null
371 : new TMServiceCapabilitiesRequestGetFeatureInfo()
372 .formats(Set.copyOf(getFeatureInfo.getFormats())))
373 .describeLayer(
374 wms.getCapabilities().getRequest().getDescribeLayer() != null)));
375
376 if (logger.isDebugEnabled()) {
377 logger.debug("Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
378 } else {
379 logger.info(
380 "Loaded capabilities from \"{}\", title: \"{}\"",
381 geoService.getUrl(),
382 geoService.getServiceCapabilities() != null
383 && geoService.getServiceCapabilities().getServiceInfo() != null
384 ? geoService
385 .getServiceCapabilities()
386 .getServiceInfo()
387 .getTitle()
388 : "(none)");
389 }
390 geoService.setLayers(new ArrayList<>());
391 addLayerRecursive(
392 geoService,
393 wms.getCapabilities().getLayerList(),
394 wms.getCapabilities().getLayer(),
395 Collections.emptySet());
396 }
397
398 void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
399 WebMapTileServer wmts = new WebMapTileServer(new URI(geoService.getUrl()).toURL(), client);
400 setServiceInfo(geoService, client, wmts);
401
402
403
404 List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
405 geoService.setLayers(
406 layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
407 }
408
409 public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
410
411
412 if (CollectionUtils.isEmpty(geoService.getLayers())) {
413 return Collections.emptyMap();
414 }
415
416
417
418
419
420
421
422 List<String> layers = geoService.getLayers().stream()
423 .filter(l -> !l.getVirtual())
424 .map(GeoServiceLayer::getName)
425 .filter(n -> {
426
427 boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
428 if (!noWhitespace) {
429 logger.warn(
430 "Not doing WFS DescribeLayer request for layer name with space: \"{}\" of WMS {}",
431 n,
432 geoService.getUrl());
433 }
434 return noWhitespace;
435 })
436 .collect(Collectors.toList());
437
438
439 Map<String, SimpleWFSLayerDescription> descriptions =
440 SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
441
442 for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
443 String layerName = entry.getKey();
444 SimpleWFSLayerDescription description = entry.getValue();
445 if (description.typeNames().size() == 1 && layerName.equals(description.getFirstTypeName())) {
446 logger.info(
447 "layer \"{}\" linked to feature type with same name of WFS {}",
448 layerName,
449 description.wfsUrl());
450 } else {
451 logger.info(
452 "layer \"{}\" -> feature type(s) {} of WFS {}",
453 layerName,
454 description.typeNames(),
455 description.wfsUrl());
456 }
457 }
458 return descriptions;
459 }
460
461 public void findAndSaveRelatedWFS(GeoService geoService) {
462 if (geoService.getProtocol() != WMS) {
463 throw new IllegalArgumentException();
464 }
465
466
467
468 Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
469
470 wfsByLayer.values().stream()
471 .map(SimpleWFSLayerDescription::wfsUrl)
472 .distinct()
473 .forEach(url -> {
474 TMFeatureSource fs = featureSourceRepository.findByUrl(url);
475 if (fs == null) {
476 fs = new TMFeatureSource()
477 .setProtocol(WFS)
478 .setUrl(url)
479 .setTitle("WFS for " + geoService.getTitle())
480 .setLinkedService(geoService);
481 try {
482 new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
483 } catch (IOException e) {
484 String msg = "Error loading WFS from URL %s: %s: %s"
485 .formatted(url, e.getClass(), e.getMessage());
486 if (logger.isTraceEnabled()) {
487 logger.error(msg, e);
488 } else {
489 logger.error(msg);
490 }
491 }
492 featureSourceRepository.save(fs);
493 }
494 });
495 }
496
497
498
499
500
501
502
503
504
505
506
507 public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
508 if (serviceLayer.getRoot()) {
509
510
511 return serviceLayer.getStyles().stream()
512 .findFirst()
513 .map(WMSStyle::getLegendURL)
514 .orElse(null);
515 }
516
517 final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
518 if (allOurLayersStyles.size() == 1) {
519 return allOurLayersStyles.getFirst().getLegendURL();
520 }
521
522 service.getLayers().stream()
523 .filter(layer -> !layer.equals(serviceLayer))
524 .forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
525
526 return allOurLayersStyles.stream()
527 .findFirst()
528 .map(WMSStyle::getLegendURL)
529 .orElse(null);
530 }
531 }