View Javadoc
1   /*
2    * Copyright (C) 2023 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller;
7   
8   import static org.tailormap.api.persistence.helper.TMFeatureTypeHelper.getConfiguredAttributes;
9   import static org.tailormap.api.util.HttpProxyUtil.passthroughResponseHeaders;
10  
11  import io.micrometer.core.annotation.Counted;
12  import io.micrometer.core.annotation.Timed;
13  import jakarta.servlet.http.HttpServletRequest;
14  import java.io.IOException;
15  import java.io.InputStream;
16  import java.io.Serializable;
17  import java.lang.invoke.MethodHandles;
18  import java.net.URI;
19  import java.net.http.HttpResponse;
20  import java.util.ArrayList;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Set;
25  import org.geotools.api.feature.type.AttributeDescriptor;
26  import org.geotools.data.wfs.WFSDataStore;
27  import org.geotools.data.wfs.WFSDataStoreFactory;
28  import org.slf4j.Logger;
29  import org.slf4j.LoggerFactory;
30  import org.springframework.beans.factory.annotation.Value;
31  import org.springframework.core.io.InputStreamResource;
32  import org.springframework.http.HttpStatus;
33  import org.springframework.http.ResponseEntity;
34  import org.springframework.transaction.annotation.Transactional;
35  import org.springframework.util.CollectionUtils;
36  import org.springframework.util.LinkedMultiValueMap;
37  import org.springframework.util.MultiValueMap;
38  import org.springframework.web.bind.annotation.GetMapping;
39  import org.springframework.web.bind.annotation.ModelAttribute;
40  import org.springframework.web.bind.annotation.RequestMapping;
41  import org.springframework.web.bind.annotation.RequestMethod;
42  import org.springframework.web.bind.annotation.RequestParam;
43  import org.springframework.web.server.ResponseStatusException;
44  import org.tailormap.api.annotation.AppRestController;
45  import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
46  import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
47  import org.tailormap.api.geotools.wfs.WFSProxy;
48  import org.tailormap.api.persistence.Application;
49  import org.tailormap.api.persistence.GeoService;
50  import org.tailormap.api.persistence.TMFeatureSource;
51  import org.tailormap.api.persistence.TMFeatureType;
52  import org.tailormap.api.persistence.json.AppLayerSettings;
53  import org.tailormap.api.persistence.json.AppTreeLayerNode;
54  import org.tailormap.api.persistence.json.GeoServiceLayer;
55  import org.tailormap.api.persistence.json.GeoServiceProtocol;
56  import org.tailormap.api.persistence.json.ServiceAuthentication;
57  import org.tailormap.api.repository.FeatureSourceRepository;
58  import org.tailormap.api.viewer.model.LayerExportCapabilities;
59  
60  @AppRestController
61  @RequestMapping(
62      path = "${tailormap-api.base-path}/{viewerKind}/{viewerName}/layer/{appLayerId}/export/")
63  public class LayerExportController {
64    private static final Logger logger =
65        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
66  
67    @Value("#{'${tailormap-api.export.allowed-outputformats}'.split(',')}")
68    private List<String> allowedOutputFormats;
69  
70    private final FeatureSourceRepository featureSourceRepository;
71  
72    public LayerExportController(FeatureSourceRepository featureSourceRepository) {
73      this.featureSourceRepository = featureSourceRepository;
74    }
75  
76    @Transactional
77    @GetMapping(path = "capabilities")
78    @Timed(value = "export_get_capabilities", description = "Get layer export capabilities")
79    public ResponseEntity<Serializable> capabilities(
80        @ModelAttribute GeoService service, @ModelAttribute GeoServiceLayer layer) {
81  
82      final LayerExportCapabilities capabilities = new LayerExportCapabilities().exportable(false);
83  
84      TMFeatureType tmft = service.findFeatureTypeForLayer(layer, featureSourceRepository);
85  
86      if (tmft != null) {
87        WFSTypeNameDescriptor wfsTypeNameDescriptor = findWFSFeatureType(service, layer, tmft);
88  
89        if (wfsTypeNameDescriptor != null) {
90          try {
91            List<String> outputFormats =
92                SimpleWFSHelper.getOutputFormats(
93                    wfsTypeNameDescriptor.wfsUrl(),
94                    wfsTypeNameDescriptor.typeName(),
95                    wfsTypeNameDescriptor.username(),
96                    wfsTypeNameDescriptor.password());
97            capabilities.setOutputFormats(outputFormats);
98          } catch (Exception e) {
99            String msg =
100               String.format(
101                   "Error getting capabilities for WFS \"%s\"", wfsTypeNameDescriptor.wfsUrl());
102           if (logger.isTraceEnabled()) {
103             logger.trace(msg, e);
104           } else {
105             logger.warn("{}: {}: {}", msg, e.getClass(), e.getMessage());
106           }
107           capabilities.setOutputFormats(null);
108         }
109       }
110       capabilities.setExportable(
111           capabilities.getOutputFormats() != null && !capabilities.getOutputFormats().isEmpty());
112     }
113 
114     return ResponseEntity.status(HttpStatus.OK).body(capabilities);
115   }
116 
117   @Transactional
118   @RequestMapping(
119       path = "download",
120       method = {RequestMethod.GET, RequestMethod.POST})
121   @Counted(value = "export_download", description = "Count of layer downloads")
122   public ResponseEntity<?> download(
123       @ModelAttribute GeoService service,
124       @ModelAttribute GeoServiceLayer layer,
125       @ModelAttribute Application application,
126       @ModelAttribute AppTreeLayerNode appTreeLayerNode,
127       @RequestParam String outputFormat,
128       @RequestParam(required = false) List<String> attributes,
129       @RequestParam(required = false) String filter,
130       @RequestParam(required = false) String sortBy,
131       @RequestParam(required = false) String sortOrder,
132       @RequestParam(required = false) String crs,
133       HttpServletRequest request) {
134 
135     // Validate outputFormat
136     if (!allowedOutputFormats.contains(outputFormat)) {
137       logger.warn("Invalid output format requested: {}", outputFormat);
138       return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Invalid output format");
139     }
140 
141     TMFeatureType tmft = service.findFeatureTypeForLayer(layer, featureSourceRepository);
142     AppLayerSettings appLayerSettings = application.getAppLayerSettings(appTreeLayerNode);
143 
144     if (tmft == null) {
145       logger.debug("Layer export requested for layer without feature type");
146       throw new ResponseStatusException(HttpStatus.NOT_FOUND);
147     }
148 
149     // Find a WFS feature type either because it is configured in Tailormap or by a SLD
150     // DescribeLayer request
151     WFSTypeNameDescriptor wfsTypeNameDescriptor = findWFSFeatureType(service, layer, tmft);
152 
153     if (wfsTypeNameDescriptor == null) {
154       throw new ResponseStatusException(
155           HttpStatus.SERVICE_UNAVAILABLE, "No suitable WFS available for layer export");
156     }
157 
158     if (attributes == null) {
159       attributes = new ArrayList<>();
160     }
161 
162     // Get attributes in configured or original order
163     Set<String> nonHiddenAttributes = getConfiguredAttributes(tmft, appLayerSettings).keySet();
164 
165     if (!attributes.isEmpty()) {
166       // Only export non-hidden property names
167       if (!nonHiddenAttributes.containsAll(attributes)) {
168         throw new ResponseStatusException(
169             HttpStatus.BAD_REQUEST,
170             "One or more requested attributes are not available on the feature type");
171       }
172     } else if (!tmft.getSettings().getHideAttributes().isEmpty()) {
173       // Only specify specific propNames if there are hidden attributes. Having no propNames
174       // request parameter to request all propNames is less error-prone than specifying the ones
175       // we have saved in the feature type
176       attributes = new ArrayList<>(nonHiddenAttributes);
177     }
178 
179     // Empty attributes means we won't specify propNames in GetFeature request, but if we do select
180     // only some property names we need the geometry attribute which is not in the 'attributes'
181     // request param so spatial export formats don't have the geometry missing.
182     if (!attributes.isEmpty() && tmft.getDefaultGeometryAttribute() != null) {
183       attributes.add(tmft.getDefaultGeometryAttribute());
184     }
185 
186     // Remove attributes which the WFS does not expose. This can be the case when using the
187     // 'customize attributes' feature in GeoServer but when TM has been configured with a JDBC
188     // feature type with all the attributes. Requesting a non-existing attribute will return an
189     // error.
190     try {
191       List<String> wfsAttributeNames = getWFSAttributeNames(wfsTypeNameDescriptor);
192       attributes.retainAll(wfsAttributeNames);
193     } catch (IOException e) {
194       logger.error("Error getting WFS feature type", e);
195       return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
196           .body("Error getting WFS feature type");
197     }
198 
199     return downloadFromWFS(
200         wfsTypeNameDescriptor, outputFormat, attributes, filter, sortBy, sortOrder, crs, request);
201   }
202 
203   private ResponseEntity<?> downloadFromWFS(
204       WFSTypeNameDescriptor wfsTypeName,
205       String outputFormat,
206       List<String> attributes,
207       String filter,
208       String sortBy,
209       String sortOrder,
210       String crs,
211       HttpServletRequest request) {
212 
213     MultiValueMap<String, String> getFeatureParameters = new LinkedMultiValueMap<>();
214     // A layer could have more than one featureType as source, currently we assume it's just one
215     getFeatureParameters.add("typeNames", wfsTypeName.typeName());
216     getFeatureParameters.add("outputFormat", outputFormat);
217     if (filter != null) {
218       // GeoServer vendor-specific
219       // https://docs.geoserver.org/latest/en/user/services/wfs/vendor.html#cql-filters
220       getFeatureParameters.add("cql_filter", filter);
221     }
222     if (crs != null) {
223       getFeatureParameters.add("srsName", crs);
224     }
225     if (!CollectionUtils.isEmpty(attributes)) {
226       getFeatureParameters.add("propertyName", String.join(",", attributes));
227     }
228     if (sortBy != null) {
229       getFeatureParameters.add("sortBy", sortBy + ("asc".equals(sortOrder) ? " A" : " D"));
230     }
231     URI wfsGetFeature =
232         SimpleWFSHelper.getWFSRequestURL(wfsTypeName.wfsUrl(), "GetFeature", getFeatureParameters);
233 
234     logger.info("Layer download, proxying WFS GetFeature request {}", wfsGetFeature);
235     try {
236       // TODO: close JPA connection before proxying
237       HttpResponse<InputStream> response =
238           WFSProxy.proxyWfsRequest(
239               wfsGetFeature, wfsTypeName.username(), wfsTypeName.password(), request);
240 
241       logger.info(
242           "Layer download response code: {}, content type: {}, disposition: {}",
243           response.statusCode(),
244           response.headers().firstValue("Content-Type").map(Object::toString).orElse("<none>"),
245           response
246               .headers()
247               .firstValue("Content-Disposition")
248               .map(Object::toString)
249               .orElse("<none>"));
250 
251       InputStreamResource body = new InputStreamResource(response.body());
252 
253       org.springframework.http.HttpHeaders headers =
254           passthroughResponseHeaders(
255               response.headers(), Set.of("Content-Type", "Content-Disposition"));
256 
257       // TODO: record response size and time with micrometer
258       return ResponseEntity.status(response.statusCode()).headers(headers).body(body);
259     } catch (Exception e) {
260       return ResponseEntity.status(HttpStatus.BAD_GATEWAY).body("Bad Gateway");
261     }
262   }
263 
264   private record WFSTypeNameDescriptor(
265       String wfsUrl, String typeName, String username, String password) {}
266 
267   private WFSTypeNameDescriptor findWFSFeatureType(
268       GeoService service, GeoServiceLayer layer, TMFeatureType tmft) {
269 
270     String wfsUrl = null;
271     String typeName = null;
272     String username = null;
273     String password = null;
274     ServiceAuthentication auth = null;
275 
276     if (tmft != null) {
277       TMFeatureSource featureSource = tmft.getFeatureSource();
278 
279       if (featureSource.getProtocol() == TMFeatureSource.Protocol.WFS) {
280         wfsUrl = featureSource.getUrl();
281         typeName = tmft.getName();
282         auth = featureSource.getAuthentication();
283       }
284     }
285 
286     if ((wfsUrl == null || typeName == null) && service.getProtocol() == GeoServiceProtocol.WMS) {
287       // Try to find out the WFS by doing a DescribeLayer request (from OGC SLD spec)
288       auth = service.getAuthentication();
289 
290       SimpleWFSLayerDescription wfsLayerDescription =
291           getWFSLayerDescriptionForWMS(service, layer.getName());
292       if (wfsLayerDescription != null
293           && wfsLayerDescription.wfsUrl() != null
294           && wfsLayerDescription.getFirstTypeName() != null) {
295         wfsUrl = wfsLayerDescription.wfsUrl();
296         // Ignores possibly multiple feature types associated with the layer (a group layer for
297         // instance)
298         typeName = wfsLayerDescription.getFirstTypeName();
299         auth = service.getAuthentication();
300       }
301     }
302 
303     if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
304       username = auth.getUsername();
305       password = auth.getPassword();
306     }
307 
308     if (wfsUrl != null && typeName != null) {
309       return new WFSTypeNameDescriptor(wfsUrl, typeName, username, password);
310     } else {
311       return null;
312     }
313   }
314 
315   private SimpleWFSLayerDescription getWFSLayerDescriptionForWMS(
316       GeoService wmsService, String layerName) {
317     String username = null;
318     String password = null;
319     if (wmsService.getAuthentication() != null
320         && wmsService.getAuthentication().getMethod()
321             == ServiceAuthentication.MethodEnum.PASSWORD) {
322       username = wmsService.getAuthentication().getUsername();
323       password = wmsService.getAuthentication().getPassword();
324     }
325     SimpleWFSLayerDescription wfsLayerDescription =
326         SimpleWFSHelper.describeWMSLayer(wmsService.getUrl(), username, password, layerName);
327     if (wfsLayerDescription != null && !wfsLayerDescription.typeNames().isEmpty()) {
328       logger.info(
329           "WMS described layer \"{}\" with typeNames \"{}\" of WFS \"{}\" for WMS \"{}\"",
330           layerName,
331           wfsLayerDescription.typeNames(),
332           wfsLayerDescription.wfsUrl(),
333           wmsService.getUrl());
334 
335       return wfsLayerDescription;
336     }
337     return null;
338   }
339 
340   /**
341    * Get the (exposed) attribute names of the WFS feature type.
342    *
343    * @param wfsTypeNameDescriptor provides the WFS feature type to get the attribute names for
344    * @return a list of attribute names for the WFS feature type
345    * @throws IOException if there were any problems setting up (creating or connecting) the
346    *     datasource.
347    */
348   private static List<String> getWFSAttributeNames(WFSTypeNameDescriptor wfsTypeNameDescriptor)
349       throws IOException {
350     Map<String, Object> connectionParameters = new HashMap<>();
351     connectionParameters.put(
352         WFSDataStoreFactory.URL.key,
353         SimpleWFSHelper.getWFSRequestURL(wfsTypeNameDescriptor.wfsUrl(), "GetCapabilities")
354             .toURL());
355     connectionParameters.put(WFSDataStoreFactory.PROTOCOL.key, Boolean.FALSE);
356     connectionParameters.put(WFSDataStoreFactory.WFS_STRATEGY.key, "geoserver");
357     connectionParameters.put(WFSDataStoreFactory.LENIENT.key, Boolean.TRUE);
358     connectionParameters.put(WFSDataStoreFactory.TIMEOUT.key, SimpleWFSHelper.TIMEOUT);
359     if (wfsTypeNameDescriptor.username() != null) {
360       connectionParameters.put(WFSDataStoreFactory.USERNAME.key, wfsTypeNameDescriptor.username());
361       connectionParameters.put(WFSDataStoreFactory.PASSWORD.key, wfsTypeNameDescriptor.password());
362     }
363 
364     WFSDataStore wfs = new WFSDataStoreFactory().createDataStore(connectionParameters);
365     List<String> attributeNames =
366         wfs
367             .getFeatureSource(wfsTypeNameDescriptor.typeName())
368             .getSchema()
369             .getAttributeDescriptors()
370             .stream()
371             .map(AttributeDescriptor::getLocalName)
372             .toList();
373 
374     wfs.dispose();
375     return attributeNames;
376   }
377 }