View Javadoc
1   /*
2    * Copyright (C) 2023 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.persistence.helper;
7   
8   import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
9   import static org.tailormap.api.persistence.json.GeoServiceProtocol.QUANTIZEDMESH;
10  import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILES3D;
11  import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
12  import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
13  
14  import java.io.IOException;
15  import java.lang.invoke.MethodHandles;
16  import java.net.URI;
17  import java.net.URISyntaxException;
18  import java.nio.charset.StandardCharsets;
19  import java.time.Instant;
20  import java.util.ArrayList;
21  import java.util.Collections;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Objects;
25  import java.util.Optional;
26  import java.util.Set;
27  import java.util.function.Predicate;
28  import java.util.stream.Collectors;
29  import org.apache.commons.lang3.StringUtils;
30  import org.geotools.api.data.ServiceInfo;
31  import org.geotools.data.ows.AbstractOpenWebService;
32  import org.geotools.data.ows.Capabilities;
33  import org.geotools.data.ows.OperationType;
34  import org.geotools.http.HTTPClientFinder;
35  import org.geotools.ows.wms.Layer;
36  import org.geotools.ows.wms.WMSCapabilities;
37  import org.geotools.ows.wms.WebMapServer;
38  import org.geotools.ows.wmts.WebMapTileServer;
39  import org.geotools.ows.wmts.model.WMTSLayer;
40  import org.slf4j.Logger;
41  import org.slf4j.LoggerFactory;
42  import org.springframework.beans.factory.annotation.Autowired;
43  import org.springframework.http.MediaType;
44  import org.springframework.stereotype.Service;
45  import org.springframework.util.CollectionUtils;
46  import org.springframework.web.util.UriComponentsBuilder;
47  import org.tailormap.api.configuration.TailormapConfig;
48  import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
49  import org.tailormap.api.geotools.WMSServiceExceptionUtil;
50  import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
51  import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
52  import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
53  import org.tailormap.api.persistence.GeoService;
54  import org.tailormap.api.persistence.TMFeatureSource;
55  import org.tailormap.api.persistence.json.GeoServiceLayer;
56  import org.tailormap.api.persistence.json.ServiceAuthentication;
57  import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
58  import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
59  import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
60  import org.tailormap.api.persistence.json.TMServiceCaps;
61  import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
62  import org.tailormap.api.persistence.json.TMServiceInfo;
63  import org.tailormap.api.persistence.json.WMSStyle;
64  import org.tailormap.api.repository.FeatureSourceRepository;
65  
66  @Service
67  public class GeoServiceHelper {
68  
69    private static final Logger logger =
70        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
71    private final TailormapConfig tailormapConfig;
72    private final FeatureSourceRepository featureSourceRepository;
73  
74    @Autowired
75    public GeoServiceHelper(TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
76      this.tailormapConfig = tailormapConfig;
77      this.featureSourceRepository = featureSourceRepository;
78    }
79  
80    public static org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
81  
82      if (StringUtils.isBlank(url)) {
83        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
84      }
85      if (url.contains("/arcgis/")) {
86        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
87      }
88      if (url.contains("/geoserver/")) {
89        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
90      }
91      if (url.contains("/mapserv")) { // /cgi-bin/mapserv, /cgi-bin/mapserv.cgi, /cgi-bin/mapserv.fcgi
92        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
93      }
94      return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
95    }
96  
97    public static String getWmsRequest(String uri) {
98      return getWmsRequest(uri == null ? null : URI.create(uri));
99    }
100 
101   /**
102    * Extracts the value of the WMS "REQUEST" parameter (case-insensitive) from the given URI.
103    *
104    * @param uri the URI to extract the request parameter from
105    * @return the value of the request parameter, or null if not found
106    */
107   public static String getWmsRequest(URI uri) {
108     if (uri == null || uri.getQuery() == null) {
109       return null;
110     }
111     return UriComponentsBuilder.fromUri(uri).build().getQueryParams().entrySet().stream()
112         .filter(entry -> "request".equalsIgnoreCase(entry.getKey()))
113         .map(entry -> entry.getValue().getFirst())
114         .findFirst()
115         .orElse(null);
116   }
117 
118   public void loadServiceCapabilities(GeoService geoService) throws Exception {
119 
120     if (geoService.getProtocol() == XYZ) {
121       setXyzCapabilities(geoService);
122       return;
123     }
124 
125     if (geoService.getProtocol() == TILES3D) {
126       set3DTilesCapabilities(geoService);
127       return;
128     }
129 
130     if (geoService.getProtocol() == QUANTIZEDMESH) {
131       setQuantizedMeshCapabilities(geoService);
132       return;
133     }
134 
135     ResponseTeeingHTTPClient client = new ResponseTeeingHTTPClient(
136         HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
137 
138     ServiceAuthentication auth = geoService.getAuthentication();
139     if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
140       client.setUser(auth.getUsername());
141       client.setPassword(auth.getPassword());
142     }
143 
144     client.setReadTimeout(this.tailormapConfig.getTimeout());
145     client.setConnectTimeout(this.tailormapConfig.getTimeout());
146     client.setTryGzip(true);
147 
148     logger.info(
149         "Get capabilities for {} {} from URL {}",
150         geoService.getProtocol(),
151         geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
152         geoService.getUrl());
153 
154     // TODO: micrometer met tags voor URL/id van service
155 
156     switch (geoService.getProtocol()) {
157       case WMS -> loadWMSCapabilities(geoService, client);
158       case WMTS -> loadWMTSCapabilities(geoService, client);
159       default ->
160         throw new UnsupportedOperationException(
161             "Unsupported geo service protocol: " + geoService.getProtocol());
162     }
163 
164     if (geoService.getTitle() == null) {
165       geoService.setTitle(Optional.ofNullable(geoService.getServiceCapabilities())
166           .map(TMServiceCaps::getServiceInfo)
167           .map(TMServiceInfo::getTitle)
168           .orElse(null));
169     }
170 
171     if (logger.isDebugEnabled()) {
172       logger.debug("Loaded service layers: {}", geoService.getLayers());
173     } else {
174       logger.info(
175           "Loaded service layers: {}",
176           geoService.getLayers().stream()
177               .filter(Predicate.not(GeoServiceLayer::getVirtual))
178               .map(GeoServiceLayer::getName)
179               .collect(Collectors.toList()));
180     }
181   }
182 
183   private static void setXyzCapabilities(GeoService geoService) {
184     geoService.setLayers(List.of(new GeoServiceLayer()
185         .id("0")
186         .root(true)
187         .name("xyz")
188         .title(geoService.getTitle())
189         .crs(Set.of(geoService.getSettings().getXyzCrs()))
190         .virtual(false)
191         .queryable(false)));
192   }
193 
194   private static void set3DTilesCapabilities(GeoService geoService) {
195     geoService.setLayers(List.of(new GeoServiceLayer()
196         .id("0")
197         .root(true)
198         .name("tiles3d")
199         .title(geoService.getTitle())
200         .virtual(false)
201         .queryable(false)));
202   }
203 
204   private static void setQuantizedMeshCapabilities(GeoService geoService) {
205     geoService.setLayers(List.of(new GeoServiceLayer()
206         .id("0")
207         .root(true)
208         .name("quantizedmesh")
209         .title(geoService.getTitle())
210         .virtual(false)
211         .queryable(false)));
212   }
213 
214   private void setServiceInfo(
215       GeoService geoService,
216       ResponseTeeingHTTPClient client,
217       AbstractOpenWebService<? extends Capabilities, Layer> ows) {
218     geoService.setCapabilities(client.getLatestResponseCopy());
219     geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
220     // geoService.setCapabilitiesContentType(client.getResponse().getContentType());
221     geoService.setCapabilitiesFetched(Instant.now());
222 
223     ServiceInfo info = ows.getInfo();
224 
225     TMServiceCaps caps = new TMServiceCaps();
226     geoService.setServiceCapabilities(caps);
227 
228     caps.setCorsAllowOrigin(client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
229 
230     if (info != null) {
231       if (StringUtils.isBlank(geoService.getTitle())) {
232         geoService.setTitle(info.getTitle());
233       }
234 
235       caps.serviceInfo(new TMServiceInfo()
236           .keywords(info.getKeywords())
237           .description(info.getDescription())
238           .title(info.getTitle())
239           .publisher(info.getPublisher())
240           .schema(info.getSchema())
241           .source(info.getSource()));
242 
243       geoService.setAdvertisedUrl(info.getSource().toString());
244     } else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
245       org.geotools.data.ows.Service service = ows.getCapabilities().getService();
246 
247       if (StringUtils.isBlank(geoService.getTitle())) {
248         geoService.setTitle(service.getTitle());
249       }
250       caps.setServiceInfo(new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
251     }
252   }
253 
254   private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
255     return new GeoServiceLayer()
256         .id(String.valueOf(layers.indexOf(l)))
257         .name(l.getName())
258         .root(l.getParent() == null)
259         .title(l.getTitle())
260         .maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
261         .minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
262         .virtual(l.getName() == null)
263         .crs(l.getSrs())
264         .latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
265         .styles(l.getStyles().stream()
266             .map(gtStyle -> {
267               WMSStyle style = new WMSStyle()
268                   .name(gtStyle.getName())
269                   .title(Optional.ofNullable(gtStyle.getTitle())
270                       .map(Objects::toString)
271                       .orElse(null))
272                   .abstractText(Optional.ofNullable(gtStyle.getAbstract())
273                       .map(Objects::toString)
274                       .orElse(null));
275               try {
276                 List<?> legendURLs = gtStyle.getLegendURLs();
277                 // GeoTools will replace invalid URLs with null in legendURLs
278                 if (legendURLs != null && !legendURLs.isEmpty() && legendURLs.getFirst() != null) {
279                   style.legendURL(new URI((String) legendURLs.getFirst()));
280                 }
281               } catch (URISyntaxException ignored) {
282                 // Won't occur because GeoTools would have already returned null on
283                 // invalid URL
284               }
285               return style;
286             })
287             .collect(Collectors.toList()))
288         .queryable(l.isQueryable())
289         .abstractText(l.get_abstract())
290         .children(l.getLayerChildren().stream()
291             .map(layers::indexOf)
292             .map(String::valueOf)
293             .collect(Collectors.toList()));
294   }
295 
296   private void addLayerRecursive(
297       GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
298     GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
299     // Crses are inherited from the parent and this is applied by GeoTools so Layer.getSrs() has all
300     // supported crses, but to save space we reverse that by only saving new crses for child layers
301     // -- just like the original WMS capabilities.
302     geoServiceLayer.getCrs().removeAll(parentCrs);
303     geoService.getLayers().add(geoServiceLayer);
304     for (Layer l : layer.getLayerChildren()) {
305       addLayerRecursive(geoService, layers, l, layer.getSrs());
306     }
307   }
308 
309   void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
310     WebMapServer wms;
311     try {
312       wms = new WebMapServer(new URI(geoService.getUrl()).toURL(), client);
313     } catch (ClassCastException | IllegalStateException e) {
314       // The gt-wms module tries to cast the XML unmarshalling result expecting capabilities, but a
315       // WMS 1.0.0/1.1.0 ServiceException may have been unmarshalled which leads to a
316       // ClassCastException.
317 
318       // A WMS 1.3.0 ServiceExceptionReport leads to an IllegalStateException because of a call to
319       // Throwable.initCause() on a SAXException that already has a cause.
320 
321       // In these cases, try to extract a message from the HTTP response
322 
323       String contentType = client.getLatestResponse().getContentType();
324       if (contentType != null && contentType.contains("text/xml")) {
325         String wmsException =
326             WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
327         throw new Exception("Error loading WMS capabilities: "
328             + (wmsException != null
329                 ? wmsException
330                 : new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
331       } else {
332         throw e;
333       }
334     } catch (IOException e) {
335       // This tries to match a HttpURLConnection (which the default GeoTools SimpleHTTPClient uses)
336       // exception message. In a container environment the JVM is always in English so never
337       // localized.
338       if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
339         throw new Exception(
340             "Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
341       } else {
342         throw e;
343       }
344     }
345 
346     OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
347     OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
348 
349     if (getMap == null) {
350       throw new Exception("Service does not support GetMap");
351     }
352 
353     setServiceInfo(geoService, client, wms);
354 
355     WMSCapabilities wmsCapabilities = wms.getCapabilities();
356 
357     // TODO Jackson annotations op GeoTools classes of iets anders slims?
358 
359     geoService
360         .getServiceCapabilities()
361         .capabilities(new TMServiceCapsCapabilities()
362             .version(wmsCapabilities.getVersion())
363             .updateSequence(wmsCapabilities.getUpdateSequence())
364             .abstractText(wmsCapabilities.getService().get_abstract())
365             .request(new TMServiceCapabilitiesRequest()
366                 .getMap(new TMServiceCapabilitiesRequestGetMap()
367                     .formats(Set.copyOf(getMap.getFormats())))
368                 .getFeatureInfo(
369                     getFeatureInfo == null
370                         ? null
371                         : new TMServiceCapabilitiesRequestGetFeatureInfo()
372                             .formats(Set.copyOf(getFeatureInfo.getFormats())))
373                 .describeLayer(
374                     wms.getCapabilities().getRequest().getDescribeLayer() != null)));
375 
376     if (logger.isDebugEnabled()) {
377       logger.debug("Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
378     } else {
379       logger.info(
380           "Loaded capabilities from \"{}\", title: \"{}\"",
381           geoService.getUrl(),
382           geoService.getServiceCapabilities() != null
383                   && geoService.getServiceCapabilities().getServiceInfo() != null
384               ? geoService
385                   .getServiceCapabilities()
386                   .getServiceInfo()
387                   .getTitle()
388               : "(none)");
389     }
390     geoService.setLayers(new ArrayList<>());
391     addLayerRecursive(
392         geoService,
393         wms.getCapabilities().getLayerList(),
394         wms.getCapabilities().getLayer(),
395         Collections.emptySet());
396   }
397 
398   void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
399     WebMapTileServer wmts = new WebMapTileServer(new URI(geoService.getUrl()).toURL(), client);
400     setServiceInfo(geoService, client, wmts);
401 
402     // TODO set capabilities if we need something from it
403 
404     List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
405     geoService.setLayers(
406         layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
407   }
408 
409   public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
410     // TODO: report back progress
411 
412     if (CollectionUtils.isEmpty(geoService.getLayers())) {
413       return Collections.emptyMap();
414     }
415 
416     // Do one DescribeLayer request for all layers in a WMS. This is faster than one request per
417     // layer, but when one layer has an error this prevents describing valid layers. But that's a
418     // problem with the WMS / GeoServer.
419     // For now at least ignore layers with space in the name because GeoServer chokes out invalid
420     // XML for those.
421 
422     List<String> layers = geoService.getLayers().stream()
423         .filter(l -> !l.getVirtual())
424         .map(GeoServiceLayer::getName)
425         .filter(n -> {
426           // filter out white-space (non-greedy regex)
427           boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
428           if (!noWhitespace) {
429             logger.warn(
430                 "Not doing WFS DescribeLayer request for layer name with space: \"{}\" of WMS {}",
431                 n,
432                 geoService.getUrl());
433           }
434           return noWhitespace;
435         })
436         .collect(Collectors.toList());
437 
438     // TODO: add authentication
439     Map<String, SimpleWFSLayerDescription> descriptions =
440         SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
441 
442     for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
443       String layerName = entry.getKey();
444       SimpleWFSLayerDescription description = entry.getValue();
445       if (description.typeNames().size() == 1 && layerName.equals(description.getFirstTypeName())) {
446         logger.info(
447             "layer \"{}\" linked to feature type with same name of WFS {}",
448             layerName,
449             description.wfsUrl());
450       } else {
451         logger.info(
452             "layer \"{}\" -> feature type(s) {} of WFS {}",
453             layerName,
454             description.typeNames(),
455             description.wfsUrl());
456       }
457     }
458     return descriptions;
459   }
460 
461   public void findAndSaveRelatedWFS(GeoService geoService) {
462     if (geoService.getProtocol() != WMS) {
463       throw new IllegalArgumentException();
464     }
465 
466     // TODO: report back progress
467 
468     Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
469 
470     wfsByLayer.values().stream()
471         .map(SimpleWFSLayerDescription::wfsUrl)
472         .distinct()
473         .forEach(url -> {
474           TMFeatureSource fs = featureSourceRepository.findByUrl(url);
475           if (fs == null) {
476             fs = new TMFeatureSource()
477                 .setProtocol(WFS)
478                 .setUrl(url)
479                 .setTitle("WFS for " + geoService.getTitle())
480                 .setLinkedService(geoService);
481             try {
482               new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
483             } catch (IOException e) {
484               String msg = "Error loading WFS from URL %s: %s: %s"
485                   .formatted(url, e.getClass(), e.getMessage());
486               if (logger.isTraceEnabled()) {
487                 logger.error(msg, e);
488               } else {
489                 logger.error(msg);
490               }
491             }
492             featureSourceRepository.save(fs);
493           }
494         });
495   }
496 
497   /**
498    * Try to extract the legend url from the styles of the layer. This works by getting all styles for the layer and
499    * then removing any styles attached to other (parent) layers from the list of all styles. What remains is/are the
500    * legend url(s) for the layer. <i>NOTE: when a layer has more than one -not an inherited style- style the first
501    * style is used.</i>
502    *
503    * @param service the service that has the layer
504    * @param serviceLayer the layer to get the legend url for
505    * @return a URI to the legend image or null if not found
506    */
507   public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
508     if (serviceLayer.getRoot()) {
509       // if this is a root layer, there are no parent layers, return the first style we find for
510       // this layer, if any
511       return serviceLayer.getStyles().stream()
512           .findFirst()
513           .map(WMSStyle::getLegendURL)
514           .orElse(null);
515     }
516 
517     final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
518     if (allOurLayersStyles.size() == 1) {
519       return allOurLayersStyles.getFirst().getLegendURL();
520     }
521     // remove the styles from all the other layer(s) from the list of all our layers styles
522     service.getLayers().stream()
523         .filter(layer -> !layer.equals(serviceLayer))
524         .forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
525 
526     return allOurLayersStyles.stream()
527         .findFirst()
528         .map(WMSStyle::getLegendURL)
529         .orElse(null);
530   }
531 }