GeoServiceHelper.java
/*
* Copyright (C) 2023 B3Partners B.V.
*
* SPDX-License-Identifier: MIT
*/
package org.tailormap.api.persistence.helper;
import static org.tailormap.api.persistence.TMFeatureSource.Protocol.WFS;
import static org.tailormap.api.persistence.json.GeoServiceProtocol.QUANTIZEDMESH;
import static org.tailormap.api.persistence.json.GeoServiceProtocol.TILES3D;
import static org.tailormap.api.persistence.json.GeoServiceProtocol.WMS;
import static org.tailormap.api.persistence.json.GeoServiceProtocol.XYZ;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.geotools.api.data.ServiceInfo;
import org.geotools.data.ows.AbstractOpenWebService;
import org.geotools.data.ows.Capabilities;
import org.geotools.data.ows.OperationType;
import org.geotools.http.HTTPClientFinder;
import org.geotools.ows.wms.Layer;
import org.geotools.ows.wms.WMSCapabilities;
import org.geotools.ows.wms.WebMapServer;
import org.geotools.ows.wmts.WebMapTileServer;
import org.geotools.ows.wmts.model.WMTSLayer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.web.util.UriComponentsBuilder;
import org.tailormap.api.configuration.TailormapConfig;
import org.tailormap.api.geotools.ResponseTeeingHTTPClient;
import org.tailormap.api.geotools.WMSServiceExceptionUtil;
import org.tailormap.api.geotools.featuresources.WFSFeatureSourceHelper;
import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
import org.tailormap.api.persistence.GeoService;
import org.tailormap.api.persistence.TMFeatureSource;
import org.tailormap.api.persistence.json.GeoServiceLayer;
import org.tailormap.api.persistence.json.ServiceAuthentication;
import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequest;
import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetFeatureInfo;
import org.tailormap.api.persistence.json.TMServiceCapabilitiesRequestGetMap;
import org.tailormap.api.persistence.json.TMServiceCaps;
import org.tailormap.api.persistence.json.TMServiceCapsCapabilities;
import org.tailormap.api.persistence.json.TMServiceInfo;
import org.tailormap.api.persistence.json.WMSStyle;
import org.tailormap.api.repository.FeatureSourceRepository;
@Service
public class GeoServiceHelper {
private static final Logger logger =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final TailormapConfig tailormapConfig;
private final FeatureSourceRepository featureSourceRepository;
@Autowired
public GeoServiceHelper(TailormapConfig tailormapConfig, FeatureSourceRepository featureSourceRepository) {
this.tailormapConfig = tailormapConfig;
this.featureSourceRepository = featureSourceRepository;
}
public static org.tailormap.api.viewer.model.Service.ServerTypeEnum guessServerTypeFromUrl(String url) {
if (StringUtils.isBlank(url)) {
return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
}
if (url.contains("/arcgis/")) {
return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
}
if (url.contains("/geoserver/")) {
return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GEOSERVER;
}
if (url.contains("/mapserv")) { // /cgi-bin/mapserv, /cgi-bin/mapserv.cgi, /cgi-bin/mapserv.fcgi
return org.tailormap.api.viewer.model.Service.ServerTypeEnum.MAPSERVER;
}
return org.tailormap.api.viewer.model.Service.ServerTypeEnum.GENERIC;
}
public static String getWmsRequest(String uri) {
return getWmsRequest(uri == null ? null : URI.create(uri));
}
/**
* Extracts the value of the WMS "REQUEST" parameter (case-insensitive) from the given URI.
*
* @param uri the URI to extract the request parameter from
* @return the value of the request parameter, or null if not found
*/
public static String getWmsRequest(URI uri) {
if (uri == null || uri.getQuery() == null) {
return null;
}
return UriComponentsBuilder.fromUri(uri).build().getQueryParams().entrySet().stream()
.filter(entry -> "request".equalsIgnoreCase(entry.getKey()))
.map(entry -> entry.getValue().getFirst())
.findFirst()
.orElse(null);
}
public void loadServiceCapabilities(GeoService geoService) throws Exception {
if (geoService.getProtocol() == XYZ) {
setXyzCapabilities(geoService);
return;
}
if (geoService.getProtocol() == TILES3D) {
set3DTilesCapabilities(geoService);
return;
}
if (geoService.getProtocol() == QUANTIZEDMESH) {
setQuantizedMeshCapabilities(geoService);
return;
}
ResponseTeeingHTTPClient client = new ResponseTeeingHTTPClient(
HTTPClientFinder.createClient(), null, Set.of("Access-Control-Allow-Origin"));
ServiceAuthentication auth = geoService.getAuthentication();
if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
client.setUser(auth.getUsername());
client.setPassword(auth.getPassword());
}
client.setReadTimeout(this.tailormapConfig.getTimeout());
client.setConnectTimeout(this.tailormapConfig.getTimeout());
client.setTryGzip(true);
logger.info(
"Get capabilities for {} {} from URL {}",
geoService.getProtocol(),
geoService.getId() == null ? "(new)" : "id " + geoService.getId(),
geoService.getUrl());
// TODO: micrometer met tags voor URL/id van service
switch (geoService.getProtocol()) {
case WMS -> loadWMSCapabilities(geoService, client);
case WMTS -> loadWMTSCapabilities(geoService, client);
default ->
throw new UnsupportedOperationException(
"Unsupported geo service protocol: " + geoService.getProtocol());
}
if (geoService.getTitle() == null) {
geoService.setTitle(Optional.ofNullable(geoService.getServiceCapabilities())
.map(TMServiceCaps::getServiceInfo)
.map(TMServiceInfo::getTitle)
.orElse(null));
}
if (logger.isDebugEnabled()) {
logger.debug("Loaded service layers: {}", geoService.getLayers());
} else {
logger.info(
"Loaded service layers: {}",
geoService.getLayers().stream()
.filter(Predicate.not(GeoServiceLayer::getVirtual))
.map(GeoServiceLayer::getName)
.collect(Collectors.toList()));
}
}
private static void setXyzCapabilities(GeoService geoService) {
geoService.setLayers(List.of(new GeoServiceLayer()
.id("0")
.root(true)
.name("xyz")
.title(geoService.getTitle())
.crs(Set.of(geoService.getSettings().getXyzCrs()))
.virtual(false)
.queryable(false)));
}
private static void set3DTilesCapabilities(GeoService geoService) {
geoService.setLayers(List.of(new GeoServiceLayer()
.id("0")
.root(true)
.name("tiles3d")
.title(geoService.getTitle())
.virtual(false)
.queryable(false)));
}
private static void setQuantizedMeshCapabilities(GeoService geoService) {
geoService.setLayers(List.of(new GeoServiceLayer()
.id("0")
.root(true)
.name("quantizedmesh")
.title(geoService.getTitle())
.virtual(false)
.queryable(false)));
}
private void setServiceInfo(
GeoService geoService,
ResponseTeeingHTTPClient client,
AbstractOpenWebService<? extends Capabilities, Layer> ows) {
geoService.setCapabilities(client.getLatestResponseCopy());
geoService.setCapabilitiesContentType(MediaType.APPLICATION_XML_VALUE);
// geoService.setCapabilitiesContentType(client.getResponse().getContentType());
geoService.setCapabilitiesFetched(Instant.now());
ServiceInfo info = ows.getInfo();
TMServiceCaps caps = new TMServiceCaps();
geoService.setServiceCapabilities(caps);
caps.setCorsAllowOrigin(client.getLatestResponse().getResponseHeader("Access-Control-Allow-Origin"));
if (info != null) {
if (StringUtils.isBlank(geoService.getTitle())) {
geoService.setTitle(info.getTitle());
}
caps.serviceInfo(new TMServiceInfo()
.keywords(info.getKeywords())
.description(info.getDescription())
.title(info.getTitle())
.publisher(info.getPublisher())
.schema(info.getSchema())
.source(info.getSource()));
geoService.setAdvertisedUrl(info.getSource().toString());
} else if (ows.getCapabilities() != null && ows.getCapabilities().getService() != null) {
org.geotools.data.ows.Service service = ows.getCapabilities().getService();
if (StringUtils.isBlank(geoService.getTitle())) {
geoService.setTitle(service.getTitle());
}
caps.setServiceInfo(new TMServiceInfo().keywords(Set.copyOf(List.of(service.getKeywordList()))));
}
}
private GeoServiceLayer toGeoServiceLayer(Layer l, List<? extends Layer> layers) {
return new GeoServiceLayer()
.id(String.valueOf(layers.indexOf(l)))
.name(l.getName())
.root(l.getParent() == null)
.title(l.getTitle())
.maxScale(Double.isNaN(l.getScaleDenominatorMax()) ? null : l.getScaleDenominatorMax())
.minScale(Double.isNaN(l.getScaleDenominatorMin()) ? null : l.getScaleDenominatorMin())
.virtual(l.getName() == null)
.crs(l.getSrs())
.latLonBoundingBox(GeoToolsHelper.boundsFromCRSEnvelope(l.getLatLonBoundingBox()))
.styles(l.getStyles().stream()
.map(gtStyle -> {
WMSStyle style = new WMSStyle()
.name(gtStyle.getName())
.title(Optional.ofNullable(gtStyle.getTitle())
.map(Objects::toString)
.orElse(null))
.abstractText(Optional.ofNullable(gtStyle.getAbstract())
.map(Objects::toString)
.orElse(null));
try {
List<?> legendURLs = gtStyle.getLegendURLs();
// GeoTools will replace invalid URLs with null in legendURLs
if (legendURLs != null && !legendURLs.isEmpty() && legendURLs.getFirst() != null) {
style.legendURL(new URI((String) legendURLs.getFirst()));
}
} catch (URISyntaxException ignored) {
// Won't occur because GeoTools would have already returned null on
// invalid URL
}
return style;
})
.collect(Collectors.toList()))
.queryable(l.isQueryable())
.abstractText(l.get_abstract())
.children(l.getLayerChildren().stream()
.map(layers::indexOf)
.map(String::valueOf)
.collect(Collectors.toList()));
}
private void addLayerRecursive(
GeoService geoService, List<? extends Layer> layers, Layer layer, Set<String> parentCrs) {
GeoServiceLayer geoServiceLayer = toGeoServiceLayer(layer, layers);
// Crses are inherited from the parent and this is applied by GeoTools so Layer.getSrs() has all
// supported crses, but to save space we reverse that by only saving new crses for child layers
// -- just like the original WMS capabilities.
geoServiceLayer.getCrs().removeAll(parentCrs);
geoService.getLayers().add(geoServiceLayer);
for (Layer l : layer.getLayerChildren()) {
addLayerRecursive(geoService, layers, l, layer.getSrs());
}
}
void loadWMSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
WebMapServer wms;
try {
wms = new WebMapServer(new URI(geoService.getUrl()).toURL(), client);
} catch (ClassCastException | IllegalStateException e) {
// The gt-wms module tries to cast the XML unmarshalling result expecting capabilities, but a
// WMS 1.0.0/1.1.0 ServiceException may have been unmarshalled which leads to a
// ClassCastException.
// A WMS 1.3.0 ServiceExceptionReport leads to an IllegalStateException because of a call to
// Throwable.initCause() on a SAXException that already has a cause.
// In these cases, try to extract a message from the HTTP response
String contentType = client.getLatestResponse().getContentType();
if (contentType != null && contentType.contains("text/xml")) {
String wmsException =
WMSServiceExceptionUtil.tryGetServiceExceptionMessage(client.getLatestResponseCopy());
throw new Exception("Error loading WMS capabilities: "
+ (wmsException != null
? wmsException
: new String(client.getLatestResponseCopy(), StandardCharsets.UTF_8)));
} else {
throw e;
}
} catch (IOException e) {
// This tries to match a HttpURLConnection (which the default GeoTools SimpleHTTPClient uses)
// exception message. In a container environment the JVM is always in English so never
// localized.
if (e.getMessage().contains("Server returned HTTP response code: 401 for URL:")) {
throw new Exception(
"Error loading WMS, got 401 unauthorized response (credentials may be required or invalid)");
} else {
throw e;
}
}
OperationType getMap = wms.getCapabilities().getRequest().getGetMap();
OperationType getFeatureInfo = wms.getCapabilities().getRequest().getGetFeatureInfo();
if (getMap == null) {
throw new Exception("Service does not support GetMap");
}
setServiceInfo(geoService, client, wms);
WMSCapabilities wmsCapabilities = wms.getCapabilities();
// TODO Jackson annotations op GeoTools classes of iets anders slims?
geoService
.getServiceCapabilities()
.capabilities(new TMServiceCapsCapabilities()
.version(wmsCapabilities.getVersion())
.updateSequence(wmsCapabilities.getUpdateSequence())
.abstractText(wmsCapabilities.getService().get_abstract())
.request(new TMServiceCapabilitiesRequest()
.getMap(new TMServiceCapabilitiesRequestGetMap()
.formats(Set.copyOf(getMap.getFormats())))
.getFeatureInfo(
getFeatureInfo == null
? null
: new TMServiceCapabilitiesRequestGetFeatureInfo()
.formats(Set.copyOf(getFeatureInfo.getFormats())))
.describeLayer(
wms.getCapabilities().getRequest().getDescribeLayer() != null)));
if (logger.isDebugEnabled()) {
logger.debug("Loaded capabilities, service capabilities: {}", geoService.getServiceCapabilities());
} else {
logger.info(
"Loaded capabilities from \"{}\", title: \"{}\"",
geoService.getUrl(),
geoService.getServiceCapabilities() != null
&& geoService.getServiceCapabilities().getServiceInfo() != null
? geoService
.getServiceCapabilities()
.getServiceInfo()
.getTitle()
: "(none)");
}
geoService.setLayers(new ArrayList<>());
addLayerRecursive(
geoService,
wms.getCapabilities().getLayerList(),
wms.getCapabilities().getLayer(),
Collections.emptySet());
}
void loadWMTSCapabilities(GeoService geoService, ResponseTeeingHTTPClient client) throws Exception {
WebMapTileServer wmts = new WebMapTileServer(new URI(geoService.getUrl()).toURL(), client);
setServiceInfo(geoService, client, wmts);
// TODO set capabilities if we need something from it
List<WMTSLayer> layers = wmts.getCapabilities().getLayerList();
geoService.setLayers(
layers.stream().map(l -> toGeoServiceLayer(l, layers)).collect(Collectors.toList()));
}
public Map<String, SimpleWFSLayerDescription> findRelatedWFS(GeoService geoService) {
// TODO: report back progress
if (CollectionUtils.isEmpty(geoService.getLayers())) {
return Collections.emptyMap();
}
// Do one DescribeLayer request for all layers in a WMS. This is faster than one request per
// layer, but when one layer has an error this prevents describing valid layers. But that's a
// problem with the WMS / GeoServer.
// For now at least ignore layers with space in the name because GeoServer chokes out invalid
// XML for those.
List<String> layers = geoService.getLayers().stream()
.filter(l -> !l.getVirtual())
.map(GeoServiceLayer::getName)
.filter(n -> {
// filter out white-space (non-greedy regex)
boolean noWhitespace = !n.contains("(.*?)\\s(.*?)");
if (!noWhitespace) {
logger.warn(
"Not doing WFS DescribeLayer request for layer name with space: \"{}\" of WMS {}",
n,
geoService.getUrl());
}
return noWhitespace;
})
.collect(Collectors.toList());
// TODO: add authentication
Map<String, SimpleWFSLayerDescription> descriptions =
SimpleWFSHelper.describeWMSLayers(geoService.getUrl(), null, null, layers);
for (Map.Entry<String, SimpleWFSLayerDescription> entry : descriptions.entrySet()) {
String layerName = entry.getKey();
SimpleWFSLayerDescription description = entry.getValue();
if (description.typeNames().size() == 1 && layerName.equals(description.getFirstTypeName())) {
logger.info(
"layer \"{}\" linked to feature type with same name of WFS {}",
layerName,
description.wfsUrl());
} else {
logger.info(
"layer \"{}\" -> feature type(s) {} of WFS {}",
layerName,
description.typeNames(),
description.wfsUrl());
}
}
return descriptions;
}
public void findAndSaveRelatedWFS(GeoService geoService) {
if (geoService.getProtocol() != WMS) {
throw new IllegalArgumentException();
}
// TODO: report back progress
Map<String, SimpleWFSLayerDescription> wfsByLayer = this.findRelatedWFS(geoService);
wfsByLayer.values().stream()
.map(SimpleWFSLayerDescription::wfsUrl)
.distinct()
.forEach(url -> {
TMFeatureSource fs = featureSourceRepository.findByUrl(url);
if (fs == null) {
fs = new TMFeatureSource()
.setProtocol(WFS)
.setUrl(url)
.setTitle("WFS for " + geoService.getTitle())
.setLinkedService(geoService);
try {
new WFSFeatureSourceHelper().loadCapabilities(fs, tailormapConfig.getTimeout());
} catch (IOException e) {
String msg = "Error loading WFS from URL %s: %s: %s"
.formatted(url, e.getClass(), e.getMessage());
if (logger.isTraceEnabled()) {
logger.error(msg, e);
} else {
logger.error(msg);
}
}
featureSourceRepository.save(fs);
}
});
}
/**
* Try to extract the legend url from the styles of the layer. This works by getting all styles for the layer and
* then removing any styles attached to other (parent) layers from the list of all styles. What remains is/are the
* legend url(s) for the layer. <i>NOTE: when a layer has more than one -not an inherited style- style the first
* style is used.</i>
*
* @param service the service that has the layer
* @param serviceLayer the layer to get the legend url for
* @return a URI to the legend image or null if not found
*/
public static URI getLayerLegendUrlFromStyles(GeoService service, GeoServiceLayer serviceLayer) {
if (serviceLayer.getRoot()) {
// if this is a root layer, there are no parent layers, return the first style we find for
// this layer, if any
return serviceLayer.getStyles().stream()
.findFirst()
.map(WMSStyle::getLegendURL)
.orElse(null);
}
final List<WMSStyle> allOurLayersStyles = serviceLayer.getStyles();
if (allOurLayersStyles.size() == 1) {
return allOurLayersStyles.getFirst().getLegendURL();
}
// remove the styles from all the other layer(s) from the list of all our layers styles
service.getLayers().stream()
.filter(layer -> !layer.equals(serviceLayer))
.forEach(layer -> allOurLayersStyles.removeAll(layer.getStyles()));
return allOurLayersStyles.stream()
.findFirst()
.map(WMSStyle::getLegendURL)
.orElse(null);
}
}