View Javadoc
1   /*
2    * Copyright (C) 2023 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.persistence.helper;
7   
8   import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
9   import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILESET3D;
10  import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
11  import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
12  
13  import java.io.IOException;
14  import java.lang.invoke.MethodHandles;
15  import java.net.URI;
16  import java.net.URISyntaxException;
17  import java.net.URL;
18  import java.nio.charset.StandardCharsets;
19  import java.time.Instant;
20  import java.util.ArrayList;
21  import java.util.Collections;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Objects;
25  import java.util.Optional;
26  import java.util.Set;
27  import java.util.function.Predicate;
28  import java.util.stream.Collectors;
29  import org.apache.commons.lang3.StringUtils;
30  import org.geotools.api.data.ServiceInfo;
31  import org.geotools.data.ows.AbstractOpenWebService;
32  import org.geotools.data.ows.Capabilities;
33  import org.geotools.data.ows.OperationType;
34  import org.geotools.http.HTTPClientFinder;
35  import org.geotools.ows.wms.Layer;
36  import org.geotools.ows.wms.WMSCapabilities;
37  import org.geotools.ows.wms.WebMapServer;
38  import org.geotools.ows.wmts.WebMapTileServer;
39  import org.geotools.ows.wmts.model.WMTSLayer;
40  import org.slf4j.Logger;
41  import org.slf4j.LoggerFactory;
42  import org.springframework.beans.factory.annotation.Autowired;
43  import org.springframework.http.MediaType;
44  import org.springframework.stereotype.Service;
45  import org.springframework.util.CollectionUtils;
46  import org.tailormap.api.configuration.TailormapConfig;
47  import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
48  import org.tailormap.api.geotools.WMSServiceExceptionUtil;
49  import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
50  import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
51  import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
52  import org.tailormap.api.persistence.GeoService;
53  import org.tailormap.api.persistence.TMFeatureSource;
54  import org.tailormap.api.persistence.json.GeoServiceLayer;
55  import org.tailormap.api.persistence.json.ServiceAuthentication;
56  import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
57  import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
58  import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
59  import org.tailormap.api.persistence.json.TMServiceCaps;
60  import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
61  import org.tailormap.api.persistence.json.TMServiceInfo;
62  import org.tailormap.api.persistence.json.WMSStyle;
63  import org.tailormap.api.repository.FeatureSourceRepository;
64  
65  @Service
66  public class GeoServiceHelper {
67  
68    private static final Logger logger =
69        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
70    private final TailormapConfig tailormapConfig;
71    private final FeatureSourceRepository featureSourceRepository;
72  
73    @Autowired
74    public GeoServiceHelper(TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
75      this.tailormapConfig = tailormapConfig;
76      this.featureSourceRepository = featureSourceRepository;
77    }
78  
79    public org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
80  
81      if (StringUtils.isBlank(url)) {
82        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
83      }
84      if (url.contains("/arcgis/")) {
85        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
86      }
87      if (url.contains("/geoserver/")) {
88        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
89      }
90      if (url.contains("/mapserv")) { // /cgi-bin/mapserv, /cgi-bin/mapserv.cgi, /cgi-bin/mapserv.fcgi
91        return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
92      }
93      return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
94    }
95  
96    public void loadServiceCapabilities(GeoService geoService) throws Exception {
97  
98      if (geoService.getProtocol() == XYZ) {
99        setXyzCapabilities(geoService);
100       return;
101     }
102 
103     if (geoService.getProtocol() == TILESET3D) {
104       setTileset3DCapabilities(geoService);
105       return;
106     }
107 
108     ResponseTeeingHTTPClient client = new ResponseTeeingHTTPClient(
109         HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
110 
111     ServiceAuthentication auth = geoService.getAuthentication();
112     if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
113       client.setUser(auth.getUsername());
114       client.setPassword(auth.getPassword());
115     }
116 
117     client.setReadTimeout(this.tailormapConfig.getTimeout());
118     client.setConnectTimeout(this.tailormapConfig.getTimeout());
119     client.setTryGzip(true);
120 
121     logger.info(
122         "Get capabilities for {} {} from URL {}",
123         geoService.getProtocol(),
124         geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
125         geoService.getUrl());
126 
127     // TODO: micrometer met tags voor URL/id van service
128 
129     switch (geoService.getProtocol()) {
130       case WMS:
131         loadWMSCapabilities(geoService, client);
132         break;
133       case WMTS:
134         loadWMTSCapabilities(geoService, client);
135         break;
136       default:
137         throw new UnsupportedOperationException(
138             "Unsupported geo service protocol: " + geoService.getProtocol());
139     }
140 
141     if (geoService.getTitle() == null) {
142       geoService.setTitle(Optional.ofNullable(geoService.getServiceCapabilities())
143           .map(TMServiceCaps::getServiceInfo)
144           .map(TMServiceInfo::getTitle)
145           .orElse(null));
146     }
147 
148     if (logger.isDebugEnabled()) {
149       logger.debug("Loaded service layers: {}", geoService.getLayers());
150     } else {
151       logger.info(
152           "Loaded service layers: {}",
153           geoService.getLayers().stream()
154               .filter(Predicate.not(GeoServiceLayer::getVirtual))
155               .map(GeoServiceLayer::getName)
156               .collect(Collectors.toList()));
157     }
158   }
159 
160   private static void setXyzCapabilities(GeoService geoService) {
161     geoService.setLayers(List.of(new GeoServiceLayer()
162         .id("0")
163         .root(true)
164         .name("xyz")
165         .title(geoService.getTitle())
166         .crs(Set.of(geoService.getSettings().getXyzCrs()))
167         .virtual(false)
168         .queryable(false)));
169   }
170 
171   private static void setTileset3DCapabilities(GeoService geoService) {
172     geoService.setLayers(List.of(new GeoServiceLayer()
173         .id("0")
174         .root(true)
175         .name("Tileset3D")
176         .title(geoService.getTitle())
177         .virtual(false)
178         .queryable(false)));
179   }
180 
181   private void setServiceInfo(
182       GeoService geoService,
183       ResponseTeeingHTTPClient client,
184       AbstractOpenWebService<? extends Capabilities, Layer> ows) {
185     geoService.setCapabilities(client.getLatestResponseCopy());
186     geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
187     // geoService.setCapabilitiesContentType(client.getResponse().getContentType());
188     geoService.setCapabilitiesFetched(Instant.now());
189 
190     ServiceInfo info = ows.getInfo();
191 
192     TMServiceCaps caps = new TMServiceCaps();
193     geoService.setServiceCapabilities(caps);
194 
195     caps.setCorsAllowOrigin(client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
196 
197     if (info != null) {
198       if (StringUtils.isBlank(geoService.getTitle())) {
199         geoService.setTitle(info.getTitle());
200       }
201 
202       caps.serviceInfo(new TMServiceInfo()
203           .keywords(info.getKeywords())
204           .description(info.getDescription())
205           .title(info.getTitle())
206           .publisher(info.getPublisher())
207           .schema(info.getSchema())
208           .source(info.getSource()));
209 
210       geoService.setAdvertisedUrl(info.getSource().toString());
211     } else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
212       org.geotools.data.ows.Service service = ows.getCapabilities().getService();
213 
214       if (StringUtils.isBlank(geoService.getTitle())) {
215         geoService.setTitle(service.getTitle());
216       }
217       caps.setServiceInfo(new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
218     }
219   }
220 
221   private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
222     return new GeoServiceLayer()
223         .id(String.valueOf(layers.indexOf(l)))
224         .name(l.getName())
225         .root(l.getParent() == null)
226         .title(l.getTitle())
227         .maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
228         .minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
229         .virtual(l.getName() == null)
230         .crs(l.getSrs())
231         .latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
232         .styles(l.getStyles().stream()
233             .map(gtStyle -> {
234               WMSStyle style = new WMSStyle()
235                   .name(gtStyle.getName())
236                   .title(Optional.ofNullable(gtStyle.getTitle())
237                       .map(Objects::toString)
238                       .orElse(null))
239                   .abstractText(Optional.ofNullable(gtStyle.getAbstract())
240                       .map(Objects::toString)
241                       .orElse(null));
242               try {
243                 List<?> legendURLs = gtStyle.getLegendURLs();
244                 // GeoTools will replace invalid URLs with null in legendURLs
245                 if (legendURLs != null && !legendURLs.isEmpty() && legendURLs.get(0) != null) {
246                   style.legendURL(new URI((String) legendURLs.get(0)));
247                 }
248               } catch (URISyntaxException ignored) {
249                 // Won't occur because GeoTools would have already returned null on
250                 // invalid URL
251               }
252               return style;
253             })
254             .collect(Collectors.toList()))
255         .queryable(l.isQueryable())
256         .abstractText(l.get_abstract())
257         .children(l.getLayerChildren().stream()
258             .map(layers::indexOf)
259             .map(String::valueOf)
260             .collect(Collectors.toList()));
261   }
262 
263   private void addLayerRecursive(
264       GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
265     GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
266     // Crses are inherited from the parent and this is applied by GeoTools so Layer.getSrs() has all
267     // supported crses, but to save space we reverse that by only saving new crses for child layers
268     // -- just like the original WMS capabilities.
269     geoServiceLayer.getCrs().removeAll(parentCrs);
270     geoService.getLayers().add(geoServiceLayer);
271     for (Layer l : layer.getLayerChildren()) {
272       addLayerRecursive(geoService, layers, l, layer.getSrs());
273     }
274   }
275 
276   void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
277     WebMapServer wms;
278     try {
279       wms = new WebMapServer(new URL(geoService.getUrl()), client);
280     } catch (ClassCastException | IllegalStateException e) {
281       // The gt-wms module tries to cast the XML unmarshalling result expecting capabilities, but a
282       // WMS 1.0.0/1.1.0 ServiceException may have been unmarshalled which leads to a
283       // ClassCastException.
284 
285       // A WMS 1.3.0 ServiceExceptionReport leads to an IllegalStateException because of a call to
286       // Throwable.initCause() on a SAXException that already has a cause.
287 
288       // In these cases, try to extract a message from the HTTP response
289 
290       String contentType = client.getLatestResponse().getContentType();
291       if (contentType != null && contentType.contains("text/xml")) {
292         String wmsException =
293             WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
294         throw new Exception("Error loading WMS capabilities: "
295             + (wmsException != null
296                 ? wmsException
297                 : new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
298       } else {
299         throw e;
300       }
301     } catch (IOException e) {
302       // This tries to match a HttpURLConnection (which the default GeoTools SimpleHTTPClient uses)
303       // exception message. In a container environment the JVM is always in English so never
304       // localized.
305       if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
306         throw new Exception(
307             "Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
308       } else {
309         throw e;
310       }
311     }
312 
313     OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
314     OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
315 
316     if (getMap == null) {
317       throw new Exception("Service does not support GetMap");
318     }
319 
320     setServiceInfo(geoService, client, wms);
321 
322     WMSCapabilities wmsCapabilities = wms.getCapabilities();
323 
324     // TODO Jackson annotations op GeoTools classes of iets anders slims?
325 
326     geoService
327         .getServiceCapabilities()
328         .capabilities(new TMServiceCapsCapabilities()
329             .version(wmsCapabilities.getVersion())
330             .updateSequence(wmsCapabilities.getUpdateSequence())
331             .abstractText(wmsCapabilities.getService().get_abstract())
332             .request(new TMServiceCapabilitiesRequest()
333                 .getMap(new TMServiceCapabilitiesRequestGetMap()
334                     .formats(Set.copyOf(getMap.getFormats())))
335                 .getFeatureInfo(
336                     getFeatureInfo == null
337                         ? null
338                         : new TMServiceCapabilitiesRequestGetFeatureInfo()
339                             .formats(Set.copyOf(getFeatureInfo.getFormats())))
340                 .describeLayer(
341                     wms.getCapabilities().getRequest().getDescribeLayer() != null)));
342 
343     if (logger.isDebugEnabled()) {
344       logger.debug("Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
345     } else {
346       logger.info(
347           "Loaded capabilities from \"{}\", title: \"{}\"",
348           geoService.getUrl(),
349           geoService.getServiceCapabilities() != null
350                   && geoService.getServiceCapabilities().getServiceInfo() != null
351               ? geoService
352                   .getServiceCapabilities()
353                   .getServiceInfo()
354                   .getTitle()
355               : "(none)");
356     }
357     geoService.setLayers(new ArrayList<>());
358     addLayerRecursive(
359         geoService,
360         wms.getCapabilities().getLayerList(),
361         wms.getCapabilities().getLayer(),
362         Collections.emptySet());
363   }
364 
365   void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
366     WebMapTileServer wmts = new WebMapTileServer(new URL(geoService.getUrl()), client);
367     setServiceInfo(geoService, client, wmts);
368 
369     // TODO set capabilities if we need something from it
370 
371     List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
372     geoService.setLayers(
373         layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
374   }
375 
376   public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
377     // TODO: report back progress
378 
379     if (CollectionUtils.isEmpty(geoService.getLayers())) {
380       return Collections.emptyMap();
381     }
382 
383     // Do one DescribeLayer request for all layers in a WMS. This is faster than one request per
384     // layer, but when one layer has an error this prevents describing valid layers. But that's a
385     // problem with the WMS / GeoServer.
386     // For now at least ignore layers with space in the name because GeoServer chokes out invalid
387     // XML for those.
388 
389     List<String> layers = geoService.getLayers().stream()
390         .filter(l -> !l.getVirtual())
391         .map(GeoServiceLayer::getName)
392         .filter(n -> {
393           // filter out white-space (non-greedy regex)
394           boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
395           if (!noWhitespace) {
396             logger.warn(
397                 "Not doing WFS DescribeLayer request for layer name with space: \"{}\" of WMS {}",
398                 n,
399                 geoService.getUrl());
400           }
401           return noWhitespace;
402         })
403         .collect(Collectors.toList());
404 
405     // TODO: add authentication
406     Map<String, SimpleWFSLayerDescription> descriptions =
407         SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
408 
409     for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
410       String layerName = entry.getKey();
411       SimpleWFSLayerDescription description = entry.getValue();
412       if (description.typeNames().size() == 1 && layerName.equals(description.getFirstTypeName())) {
413         logger.info(
414             "layer \"{}\" linked to feature type with same name of WFS {}",
415             layerName,
416             description.wfsUrl());
417       } else {
418         logger.info(
419             "layer \"{}\" -> feature type(s) {} of WFS {}",
420             layerName,
421             description.typeNames(),
422             description.wfsUrl());
423       }
424     }
425     return descriptions;
426   }
427 
428   public void findAndSaveRelatedWFS(GeoService geoService) {
429     if (geoService.getProtocol() != WMS) {
430       throw new IllegalArgumentException();
431     }
432 
433     // TODO: report back progress
434 
435     Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
436 
437     wfsByLayer.values().stream()
438         .map(SimpleWFSLayerDescription::wfsUrl)
439         .distinct()
440         .forEach(url -> {
441           TMFeatureSource fs = featureSourceRepository.findByUrl(url);
442           if (fs == null) {
443             fs = new TMFeatureSource()
444                 .setProtocol(WFS)
445                 .setUrl(url)
446                 .setTitle("WFS for " + geoService.getTitle())
447                 .setLinkedService(geoService);
448             try {
449               new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
450             } catch (IOException e) {
451               String msg = String.format(
452                   "Error loading WFS from URL %s: %s: %s", url, e.getClass(), e.getMessage());
453               if (logger.isTraceEnabled()) {
454                 logger.error(msg, e);
455               } else {
456                 logger.error(msg);
457               }
458             }
459             featureSourceRepository.save(fs);
460           }
461         });
462   }
463 
464   /**
465    * Try to extract the legend url from the styles of the layer. This works by getting all styles for the layer and
466    * then removing any styles attached to other (parent) layers from the list of all styles. What remains is/are the
467    * legend url(s) for the layer. <i>NOTE: when a layer has more than one -not an inherited style- style the first
468    * style is used.</i>
469    *
470    * @param service the service that has the layer
471    * @param serviceLayer the layer to get the legend url for
472    * @return a URI to the legend image or null if not found
473    */
474   public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
475     if (serviceLayer.getRoot()) {
476       // if this is a root layer, there are no parent layers, return the first style we find for
477       // this layer, if any
478       return serviceLayer.getStyles().stream()
479           .findFirst()
480           .map(WMSStyle::getLegendURL)
481           .orElse(null);
482     }
483 
484     final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
485     if (allOurLayersStyles.size() == 1) {
486       return allOurLayersStyles.get(0).getLegendURL();
487     }
488     // remove the styles from all the other layer(s) from the list of all our layers styles
489     service.getLayers().stream()
490         .filter(layer -> !layer.equals(serviceLayer))
491         .forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
492 
493     return allOurLayersStyles.stream()
494         .findFirst()
495         .map(WMSStyle::getLegendURL)
496         .orElse(null);
497   }
498 }