1
2
3
4
5
6 package org.tailormap.api.persistence.helper;
7
8 import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
9 import static org.tailormap.api.persistence.json.GeoServiceProtocol.QUANTIZEDMESH;
10 import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILES3D;
11 import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
12 import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
13
14 import java.io.IOException;
15 import java.lang.invoke.MethodHandles;
16 import java.net.URI;
17 import java.net.URISyntaxException;
18 import java.nio.charset.StandardCharsets;
19 import java.time.Instant;
20 import java.util.ArrayList;
21 import java.util.List;
22 import java.util.Map;
23 import java.util.Objects;
24 import java.util.Optional;
25 import java.util.Set;
26 import java.util.function.Predicate;
27 import java.util.stream.Collectors;
28 import org.apache.commons.lang3.StringUtils;
29 import org.geotools.api.data.ServiceInfo;
30 import org.geotools.data.ows.AbstractOpenWebService;
31 import org.geotools.data.ows.Capabilities;
32 import org.geotools.data.ows.OperationType;
33 import org.geotools.http.HTTPClientFinder;
34 import org.geotools.ows.wms.Layer;
35 import org.geotools.ows.wms.WMSCapabilities;
36 import org.geotools.ows.wms.WebMapServer;
37 import org.geotools.ows.wmts.WebMapTileServer;
38 import org.geotools.ows.wmts.model.WMTSLayer;
39 import org.slf4j.Logger;
40 import org.slf4j.LoggerFactory;
41 import org.springframework.beans.factory.annotation.Autowired;
42 import org.springframework.http.MediaType;
43 import org.springframework.stereotype.Service;
44 import org.springframework.util.CollectionUtils;
45 import org.springframework.web.util.UriComponentsBuilder;
46 import org.tailormap.api.configuration.TailormapConfig;
47 import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
48 import org.tailormap.api.geotools.WMSServiceExceptionUtil;
49 import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
50 import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
51 import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
52 import org.tailormap.api.persistence.GeoService;
53 import org.tailormap.api.persistence.TMFeatureSource;
54 import org.tailormap.api.persistence.json.GeoServiceLayer;
55 import org.tailormap.api.persistence.json.ServiceAuthentication;
56 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
57 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
58 import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
59 import org.tailormap.api.persistence.json.TMServiceCaps;
60 import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
61 import org.tailormap.api.persistence.json.TMServiceInfo;
62 import org.tailormap.api.persistence.json.WMSStyle;
63 import org.tailormap.api.repository.FeatureSourceRepository;
64
65 @Service
66 public class GeoServiceHelper {
67
68 private static final Logger logger =
69 LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
70 private final TailormapConfig tailormapConfig;
71 private final FeatureSourceRepository featureSourceRepository;
72
73 @Autowired
74 public GeoServiceHelper(TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
75 this.tailormapConfig = tailormapConfig;
76 this.featureSourceRepository = featureSourceRepository;
77 }
78
79 public static org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
80
81 if (StringUtils.isBlank(url)) {
82 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
83 }
84 if (url.contains("/arcgis/")) {
85 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
86 }
87 if (url.contains("/geoserver/")) {
88 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
89 }
90 if (url.contains("/mapserv")) {
91 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
92 }
93 return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
94 }
95
96 public static String getWmsRequest(String uri) {
97 return getWmsRequest(uri == null ? null : URI.create(uri));
98 }
99
100
101
102
103
104
105
106 public static String getWmsRequest(URI uri) {
107 if (uri == null || uri.getQuery() == null) {
108 return null;
109 }
110 return UriComponentsBuilder.fromUri(uri).build().getQueryParams().entrySet().stream()
111 .filter(entry -> "request".equalsIgnoreCase(entry.getKey()))
112 .map(entry -> entry.getValue().getFirst())
113 .findFirst()
114 .orElse(null);
115 }
116
117 public void loadServiceCapabilities(GeoService geoService) throws Exception {
118
119 if (geoService.getProtocol() == XYZ) {
120 setXyzCapabilities(geoService);
121 return;
122 }
123
124 if (geoService.getProtocol() == TILES3D) {
125 set3DTilesCapabilities(geoService);
126 return;
127 }
128
129 if (geoService.getProtocol() == QUANTIZEDMESH) {
130 setQuantizedMeshCapabilities(geoService);
131 return;
132 }
133
134 ResponseTeeingHTTPClient client = new ResponseTeeingHTTPClient(
135 HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
136
137 ServiceAuthentication auth = geoService.getAuthentication();
138 if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
139 client.setUser(auth.getUsername());
140 client.setPassword(auth.getPassword());
141 }
142
143 client.setReadTimeout(this.tailormapConfig.getTimeout());
144 client.setConnectTimeout(this.tailormapConfig.getTimeout());
145 client.setTryGzip(true);
146
147 logger.info(
148 "Get capabilities for {} {} from URL {}",
149 geoService.getProtocol(),
150 geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
151 geoService.getUrl());
152
153
154
155 switch (geoService.getProtocol()) {
156 case WMS -> loadWMSCapabilities(geoService, client);
157 case WMTS -> loadWMTSCapabilities(geoService, client);
158 default ->
159 throw new UnsupportedOperationException(
160 "Unsupported geo service protocol: " + geoService.getProtocol());
161 }
162
163 if (geoService.getTitle() == null) {
164 geoService.setTitle(Optional.ofNullable(geoService.getServiceCapabilities())
165 .map(TMServiceCaps::getServiceInfo)
166 .map(TMServiceInfo::getTitle)
167 .orElse(null));
168 }
169
170 if (logger.isDebugEnabled()) {
171 logger.debug("Loaded service layers: {}", geoService.getLayers());
172 } else {
173 logger.info(
174 "Loaded service layers: {}",
175 geoService.getLayers().stream()
176 .filter(Predicate.not(GeoServiceLayer::getVirtual))
177 .map(GeoServiceLayer::getName)
178 .collect(Collectors.toList()));
179 }
180 }
181
182 private static void setXyzCapabilities(GeoService geoService) {
183 geoService.setLayers(List.of(new GeoServiceLayer()
184 .id("0")
185 .root(true)
186 .name("xyz")
187 .title(geoService.getTitle())
188 .crs(Set.of(geoService.getSettings().getXyzCrs()))
189 .virtual(false)
190 .queryable(false)));
191 }
192
193 private static void set3DTilesCapabilities(GeoService geoService) {
194 geoService.setLayers(List.of(new GeoServiceLayer()
195 .id("0")
196 .root(true)
197 .name("tiles3d")
198 .title(geoService.getTitle())
199 .virtual(false)
200 .queryable(false)));
201 }
202
203 private static void setQuantizedMeshCapabilities(GeoService geoService) {
204 geoService.setLayers(List.of(new GeoServiceLayer()
205 .id("0")
206 .root(true)
207 .name("quantizedmesh")
208 .title(geoService.getTitle())
209 .virtual(false)
210 .queryable(false)));
211 }
212
213 private void setServiceInfo(
214 GeoService geoService,
215 ResponseTeeingHTTPClient client,
216 AbstractOpenWebService<? extends Capabilities, Layer> ows) {
217 geoService.setCapabilities(client.getLatestResponseCopy());
218 geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
219
220 geoService.setCapabilitiesFetched(Instant.now());
221
222 ServiceInfo info = ows.getInfo();
223
224 TMServiceCaps caps = new TMServiceCaps();
225 geoService.setServiceCapabilities(caps);
226
227 caps.setCorsAllowOrigin(client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
228
229 if (info != null) {
230 if (StringUtils.isBlank(geoService.getTitle())) {
231 geoService.setTitle(info.getTitle());
232 }
233
234 caps.serviceInfo(new TMServiceInfo()
235 .keywords(info.getKeywords())
236 .description(info.getDescription())
237 .title(info.getTitle())
238 .publisher(info.getPublisher())
239 .schema(info.getSchema())
240 .source(info.getSource()));
241
242 geoService.setAdvertisedUrl(info.getSource().toString());
243 } else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
244 org.geotools.data.ows.Service service = ows.getCapabilities().getService();
245
246 if (StringUtils.isBlank(geoService.getTitle())) {
247 geoService.setTitle(service.getTitle());
248 }
249 caps.setServiceInfo(new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
250 }
251 }
252
253 private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
254 return new GeoServiceLayer()
255 .id(String.valueOf(layers.indexOf(l)))
256 .name(l.getName())
257 .root(l.getParent() == null)
258 .title(l.getTitle())
259 .maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
260 .minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
261 .virtual(l.getName() == null)
262 .crs(l.getSrs())
263 .latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
264 .styles(l.getStyles().stream()
265 .map(gtStyle -> {
266 WMSStyle style = new WMSStyle()
267 .name(gtStyle.getName())
268 .title(Optional.ofNullable(gtStyle.getTitle())
269 .map(Objects::toString)
270 .orElse(null))
271 .abstractText(Optional.ofNullable(gtStyle.getAbstract())
272 .map(Objects::toString)
273 .orElse(null));
274 try {
275 List<?> legendUrls = gtStyle.getLegendURLs();
276
277 if (legendUrls != null && !legendUrls.isEmpty() && legendUrls.getFirst() != null) {
278 style.legendUrl(new URI((String) legendUrls.getFirst()));
279 }
280 } catch (URISyntaxException ignored) {
281
282
283 }
284 return style;
285 })
286 .collect(Collectors.toList()))
287 .queryable(l.isQueryable())
288 .abstractText(l.get_abstract())
289 .children(l.getLayerChildren().stream()
290 .map(layers::indexOf)
291 .map(String::valueOf)
292 .collect(Collectors.toList()));
293 }
294
295 private void addLayerRecursive(
296 GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
297 GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
298
299
300
301 geoServiceLayer.getCrs().removeAll(parentCrs);
302 geoService.getLayers().add(geoServiceLayer);
303 for (Layer l : layer.getLayerChildren()) {
304 addLayerRecursive(geoService, layers, l, layer.getSrs());
305 }
306 }
307
308 void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
309 WebMapServer wms;
310 try {
311 wms = new WebMapServer(new URI(geoService.getUrl()).toURL(), client);
312 } catch (ClassCastException | IllegalStateException e) {
313
314
315
316
317
318
319
320
321
322 String contentType = client.getLatestResponse().getContentType();
323 if (contentType != null && contentType.contains("text/xml")) {
324 String wmsException =
325 WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
326 throw new Exception("Error loading WMS capabilities: "
327 + (wmsException != null
328 ? wmsException
329 : new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
330 } else {
331 throw e;
332 }
333 } catch (IOException e) {
334
335
336
337 if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
338 throw new Exception(
339 "Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
340 } else {
341 throw e;
342 }
343 }
344
345 OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
346 OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
347
348 if (getMap == null) {
349 throw new Exception("Service does not support GetMap");
350 }
351
352 setServiceInfo(geoService, client, wms);
353
354 WMSCapabilities wmsCapabilities = wms.getCapabilities();
355
356
357
358 geoService
359 .getServiceCapabilities()
360 .capabilities(new TMServiceCapsCapabilities()
361 .version(wmsCapabilities.getVersion())
362 .updateSequence(wmsCapabilities.getUpdateSequence())
363 .abstractText(wmsCapabilities.getService().get_abstract())
364 .request(new TMServiceCapabilitiesRequest()
365 .getMap(new TMServiceCapabilitiesRequestGetMap()
366 .formats(Set.copyOf(getMap.getFormats())))
367 .getFeatureInfo(
368 getFeatureInfo == null
369 ? null
370 : new TMServiceCapabilitiesRequestGetFeatureInfo()
371 .formats(Set.copyOf(getFeatureInfo.getFormats())))
372 .describeLayer(
373 wms.getCapabilities().getRequest().getDescribeLayer() != null)));
374
375 if (logger.isDebugEnabled()) {
376 logger.debug("Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
377 } else {
378 logger.info(
379 "Loaded capabilities from \"{}\", title: \"{}\"",
380 geoService.getUrl(),
381 geoService.getServiceCapabilities() != null
382 && geoService.getServiceCapabilities().getServiceInfo() != null
383 ? geoService
384 .getServiceCapabilities()
385 .getServiceInfo()
386 .getTitle()
387 : "(none)");
388 }
389 geoService.setLayers(new ArrayList<>());
390 addLayerRecursive(
391 geoService,
392 wms.getCapabilities().getLayerList(),
393 wms.getCapabilities().getLayer(),
394 Set.of());
395 }
396
397 void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
398 WebMapTileServer wmts = new WebMapTileServer(new URI(geoService.getUrl()).toURL(), client);
399 setServiceInfo(geoService, client, wmts);
400
401
402
403 List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
404 geoService.setLayers(
405 layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
406 }
407
408 public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
409
410
411 if (CollectionUtils.isEmpty(geoService.getLayers())) {
412 return Map.of();
413 }
414
415
416
417
418
419
420
421 List<String> layers = geoService.getLayers().stream()
422 .filter(l -> !l.getVirtual())
423 .map(GeoServiceLayer::getName)
424 .filter(n -> {
425
426 boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
427 if (!noWhitespace) {
428 logger.warn(
429 "Not doing WFS DescribeLayer request for layer name with space: \"{}\" of WMS {}",
430 n,
431 geoService.getUrl());
432 }
433 return noWhitespace;
434 })
435 .collect(Collectors.toList());
436
437
438 Map<String, SimpleWFSLayerDescription> descriptions =
439 SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
440
441 for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
442 String layerName = entry.getKey();
443 SimpleWFSLayerDescription description = entry.getValue();
444 if (description.typeNames().size() == 1 && layerName.equals(description.getFirstTypeName())) {
445 logger.info(
446 "layer \"{}\" linked to feature type with same name of WFS {}",
447 layerName,
448 description.wfsUrl());
449 } else {
450 logger.info(
451 "layer \"{}\" -> feature type(s) {} of WFS {}",
452 layerName,
453 description.typeNames(),
454 description.wfsUrl());
455 }
456 }
457 return descriptions;
458 }
459
460 public void findAndSaveRelatedWFS(GeoService geoService) {
461 if (geoService.getProtocol() != WMS) {
462 throw new IllegalArgumentException();
463 }
464
465
466
467 Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
468
469 wfsByLayer.values().stream()
470 .map(SimpleWFSLayerDescription::wfsUrl)
471 .distinct()
472 .forEach(url -> {
473 TMFeatureSource fs = featureSourceRepository.findByUrl(url);
474 if (fs == null) {
475 fs = new TMFeatureSource()
476 .setProtocol(WFS)
477 .setUrl(url)
478 .setTitle("WFS for " + geoService.getTitle())
479 .setLinkedService(geoService);
480 try {
481 new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
482 } catch (IOException e) {
483 String msg = "Error loading WFS from URL %s: %s: %s"
484 .formatted(url, e.getClass(), e.getMessage());
485 if (logger.isTraceEnabled()) {
486 logger.error(msg, e);
487 } else {
488 logger.error(msg);
489 }
490 }
491 featureSourceRepository.save(fs);
492 }
493 });
494 }
495
496
497
498
499
500
501
502
503
504
505
506 public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
507 if (serviceLayer.getRoot()) {
508
509
510 return serviceLayer.getStyles().stream()
511 .findFirst()
512 .map(WMSStyle::getLegendUrl)
513 .orElse(null);
514 }
515
516 final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
517 if (allOurLayersStyles.size() == 1) {
518 return allOurLayersStyles.getFirst().getLegendUrl();
519 }
520
521 service.getLayers().stream()
522 .filter(layer -> !layer.equals(serviceLayer))
523 .forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
524
525 return allOurLayersStyles.stream()
526 .findFirst()
527 .map(WMSStyle::getLegendUrl)
528 .orElse(null);
529 }
530 }