View Javadoc
1   /*
2    * Copyright (C) 2023 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller;
7   
8   import static org.tailormap.api.persistence.helper.TMFeatureTypeHelper.getConfiguredAttributes;
9   import static org.tailormap.api.util.HttpProxyUtil.passthroughResponseHeaders;
10  
11  import io.micrometer.core.annotation.Counted;
12  import io.micrometer.core.annotation.Timed;
13  import jakarta.servlet.http.HttpServletRequest;
14  import java.io.IOException;
15  import java.io.InputStream;
16  import java.io.Serializable;
17  import java.lang.invoke.MethodHandles;
18  import java.net.URI;
19  import java.net.http.HttpResponse;
20  import java.util.ArrayList;
21  import java.util.Arrays;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Set;
26  import org.geotools.api.feature.type.AttributeDescriptor;
27  import org.geotools.data.wfs.WFSDataStore;
28  import org.geotools.data.wfs.WFSDataStoreFactory;
29  import org.slf4j.Logger;
30  import org.slf4j.LoggerFactory;
31  import org.springframework.core.io.InputStreamResource;
32  import org.springframework.http.HttpStatus;
33  import org.springframework.http.ResponseEntity;
34  import org.springframework.transaction.annotation.Transactional;
35  import org.springframework.util.CollectionUtils;
36  import org.springframework.util.LinkedMultiValueMap;
37  import org.springframework.util.MultiValueMap;
38  import org.springframework.web.bind.annotation.GetMapping;
39  import org.springframework.web.bind.annotation.ModelAttribute;
40  import org.springframework.web.bind.annotation.RequestMapping;
41  import org.springframework.web.bind.annotation.RequestMethod;
42  import org.springframework.web.bind.annotation.RequestParam;
43  import org.springframework.web.server.ResponseStatusException;
44  import org.tailormap.api.annotation.AppRestController;
45  import org.tailormap.api.geotools.wfs.SimpleWFSHelper;
46  import org.tailormap.api.geotools.wfs.SimpleWFSLayerDescription;
47  import org.tailormap.api.geotools.wfs.WFSProxy;
48  import org.tailormap.api.persistence.Application;
49  import org.tailormap.api.persistence.GeoService;
50  import org.tailormap.api.persistence.TMFeatureSource;
51  import org.tailormap.api.persistence.TMFeatureType;
52  import org.tailormap.api.persistence.json.AppLayerSettings;
53  import org.tailormap.api.persistence.json.AppTreeLayerNode;
54  import org.tailormap.api.persistence.json.GeoServiceLayer;
55  import org.tailormap.api.persistence.json.GeoServiceProtocol;
56  import org.tailormap.api.persistence.json.ServiceAuthentication;
57  import org.tailormap.api.repository.FeatureSourceRepository;
58  import org.tailormap.api.viewer.model.LayerExportCapabilities;
59  
60  @AppRestController
61  @RequestMapping(
62      path = "${tailormap-api.base-path}/{viewerKind}/{viewerName}/layer/{appLayerId}/export/")
63  public class LayerExportController {
64    private static final Logger logger =
65        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
66  
67    private final FeatureSourceRepository featureSourceRepository;
68  
69    public LayerExportController(FeatureSourceRepository featureSourceRepository) {
70      this.featureSourceRepository = featureSourceRepository;
71    }
72  
73    @Transactional
74    @GetMapping(path = "capabilities")
75    @Timed(value = "export_get_capabilities", description = "Get layer export capabilities")
76    public ResponseEntity<Serializable> capabilities(
77        @ModelAttribute GeoService service, @ModelAttribute GeoServiceLayer layer) throws Exception {
78  
79      final LayerExportCapabilities capabilities = new LayerExportCapabilities().exportable(false);
80  
81      TMFeatureType tmft = service.findFeatureTypeForLayer(layer, featureSourceRepository);
82  
83      if (tmft != null) {
84        WFSTypeNameDescriptor wfsTypeNameDescriptor = findWFSFeatureType(service, layer, tmft);
85  
86        if (wfsTypeNameDescriptor != null) {
87          try {
88            List<String> outputFormats =
89                SimpleWFSHelper.getOutputFormats(
90                    wfsTypeNameDescriptor.wfsUrl(),
91                    wfsTypeNameDescriptor.typeName(),
92                    wfsTypeNameDescriptor.username(),
93                    wfsTypeNameDescriptor.password());
94            capabilities.setOutputFormats(outputFormats);
95          } catch (Exception e) {
96            String msg =
97                String.format(
98                    "Error getting capabilities for WFS \"%s\"", wfsTypeNameDescriptor.wfsUrl());
99            if (logger.isTraceEnabled()) {
100             logger.trace(msg, e);
101           } else {
102             logger.warn("{}: {}: {}", msg, e.getClass(), e.getMessage());
103           }
104           capabilities.setOutputFormats(null);
105         }
106       }
107       capabilities.setExportable(
108           capabilities.getOutputFormats() != null && !capabilities.getOutputFormats().isEmpty());
109     }
110 
111     return ResponseEntity.status(HttpStatus.OK).body(capabilities);
112   }
113 
114   @Transactional
115   @RequestMapping(
116       path = "download",
117       method = {RequestMethod.GET, RequestMethod.POST})
118   @Counted(value = "export_download", description = "Count of layer downloads")
119   public ResponseEntity<?> download(
120       @ModelAttribute GeoService service,
121       @ModelAttribute GeoServiceLayer layer,
122       @ModelAttribute Application application,
123       @ModelAttribute AppTreeLayerNode appTreeLayerNode,
124       @RequestParam String outputFormat,
125       @RequestParam(required = false) List<String> attributes,
126       @RequestParam(required = false) String filter,
127       @RequestParam(required = false) String sortBy,
128       @RequestParam(required = false) String sortOrder,
129       @RequestParam(required = false) String crs,
130       HttpServletRequest request)
131       throws Exception {
132 
133     TMFeatureType tmft = service.findFeatureTypeForLayer(layer, featureSourceRepository);
134     AppLayerSettings appLayerSettings = application.getAppLayerSettings(appTreeLayerNode);
135 
136     if (tmft == null) {
137       logger.debug("Layer export requested for layer without feature type");
138       throw new ResponseStatusException(HttpStatus.NOT_FOUND);
139     }
140 
141     // Find a WFS feature type either because it is configured in Tailormap or by a SLD
142     // DescribeLayer request
143     WFSTypeNameDescriptor wfsTypeNameDescriptor = findWFSFeatureType(service, layer, tmft);
144 
145     if (wfsTypeNameDescriptor == null) {
146       throw new ResponseStatusException(
147           HttpStatus.SERVICE_UNAVAILABLE, "No suitable WFS available for layer export");
148     }
149 
150     if (attributes == null) {
151       attributes = new ArrayList<>();
152     }
153 
154     // Get attributes in configured or original order
155     Set<String> nonHiddenAttributes = getConfiguredAttributes(tmft, appLayerSettings).keySet();
156 
157     if (!attributes.isEmpty()) {
158       // Only export non-hidden property names
159       if (!nonHiddenAttributes.containsAll(attributes)) {
160         throw new ResponseStatusException(
161             HttpStatus.BAD_REQUEST,
162             "One or more requested attributes are not available on the feature type");
163       }
164     } else if (!tmft.getSettings().getHideAttributes().isEmpty()) {
165       // Only specify specific propNames if there are hidden attributes. Having no propNames
166       // request parameter to request all propNames is less error-prone than specifying the ones
167       // we have saved in the feature type
168       attributes = new ArrayList<>(nonHiddenAttributes);
169     }
170 
171     // Empty attributes means we won't specify propNames in GetFeature request, but if we do select
172     // only some property names we need the geometry attribute which is not in the 'attributes'
173     // request param so spatial export formats don't have the geometry missing.
174     if (!attributes.isEmpty() && tmft.getDefaultGeometryAttribute() != null) {
175       attributes.add(tmft.getDefaultGeometryAttribute());
176     }
177 
178     // Remove attributes which the WFS does not expose. This can be the case when using the
179     // 'customize attributes' feature in GeoServer but when TM has been configured with a JDBC
180     // feature type with all the attributes. Requesting a non-existing attribute will return an
181     // error.
182     try {
183       List<String> wfsAttributeNames = getWFSAttributeNames(wfsTypeNameDescriptor);
184       attributes.retainAll(wfsAttributeNames);
185     } catch (IOException e) {
186       logger.error("Error getting WFS feature type", e);
187       return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
188           .body("Error getting WFS feature type");
189     }
190 
191     return downloadFromWFS(
192         wfsTypeNameDescriptor, outputFormat, attributes, filter, sortBy, sortOrder, crs, request);
193   }
194 
195   private ResponseEntity<?> downloadFromWFS(
196       WFSTypeNameDescriptor wfsTypeName,
197       String outputFormat,
198       List<String> attributes,
199       String filter,
200       String sortBy,
201       String sortOrder,
202       String crs,
203       HttpServletRequest request) {
204 
205     MultiValueMap<String, String> getFeatureParameters = new LinkedMultiValueMap<>();
206     // A layer could have more than one featureType as source, currently we assume it's just one
207     getFeatureParameters.add("typeNames", wfsTypeName.typeName());
208     getFeatureParameters.add("outputFormat", outputFormat);
209     if (filter != null) {
210       // GeoServer vendor-specific
211       // https://docs.geoserver.org/latest/en/user/services/wfs/vendor.html#cql-filters
212       getFeatureParameters.add("cql_filter", filter);
213     }
214     if (crs != null) {
215       getFeatureParameters.add("srsName", crs);
216     }
217     if (!CollectionUtils.isEmpty(attributes)) {
218       getFeatureParameters.add("propertyName", String.join(",", attributes));
219     }
220     if (sortBy != null) {
221       getFeatureParameters.add("sortBy", sortBy + ("asc".equals(sortOrder) ? " A" : " D"));
222     }
223     URI wfsGetFeature =
224         SimpleWFSHelper.getWFSRequestURL(wfsTypeName.wfsUrl(), "GetFeature", getFeatureParameters);
225 
226     logger.info("Layer download, proxying WFS GetFeature request {}", wfsGetFeature);
227     try {
228       // TODO: close JPA connection before proxying
229       HttpResponse<InputStream> response =
230           WFSProxy.proxyWfsRequest(
231               wfsGetFeature, wfsTypeName.username(), wfsTypeName.password(), request);
232 
233       logger.info(
234           "Layer download response code: {}, content type: {}, disposition: {}",
235           response.statusCode(),
236           response.headers().firstValue("Content-Type").map(Object::toString).orElse("<none>"),
237           response
238               .headers()
239               .firstValue("Content-Disposition")
240               .map(Object::toString)
241               .orElse("<none>"));
242 
243       InputStreamResource body = new InputStreamResource(response.body());
244 
245       org.springframework.http.HttpHeaders headers =
246           passthroughResponseHeaders(
247               response.headers(), Set.of("Content-Type", "Content-Disposition"));
248 
249       // TODO: record response size and time with micrometer
250       return ResponseEntity.status(response.statusCode()).headers(headers).body(body);
251     } catch (Exception e) {
252       return ResponseEntity.status(HttpStatus.BAD_GATEWAY).body("Bad Gateway");
253     }
254   }
255 
256   private record WFSTypeNameDescriptor(
257       String wfsUrl, String typeName, String username, String password) {}
258 
259   private WFSTypeNameDescriptor findWFSFeatureType(
260       GeoService service, GeoServiceLayer layer, TMFeatureType tmft) {
261 
262     String wfsUrl = null;
263     String typeName = null;
264     String username = null;
265     String password = null;
266     ServiceAuthentication auth = null;
267 
268     if (tmft != null) {
269       TMFeatureSource featureSource = tmft.getFeatureSource();
270 
271       if (featureSource.getProtocol() == TMFeatureSource.Protocol.WFS) {
272         wfsUrl = featureSource.getUrl();
273         typeName = tmft.getName();
274         auth = featureSource.getAuthentication();
275       }
276     }
277 
278     if ((wfsUrl == null || typeName == null) && service.getProtocol() == GeoServiceProtocol.WMS) {
279       // Try to find out the WFS by doing a DescribeLayer request (from OGC SLD spec)
280       auth = service.getAuthentication();
281 
282       SimpleWFSLayerDescription wfsLayerDescription =
283           getWFSLayerDescriptionForWMS(service, layer.getName());
284       if (wfsLayerDescription != null
285           && wfsLayerDescription.getWfsUrl() != null
286           && wfsLayerDescription.getFirstTypeName() != null) {
287         wfsUrl = wfsLayerDescription.getWfsUrl();
288         // Ignores possibly multiple feature types associated with the layer (a group layer for
289         // instance)
290         typeName = wfsLayerDescription.getFirstTypeName();
291         auth = service.getAuthentication();
292       }
293     }
294 
295     if (auth != null && auth.getMethod() == ServiceAuthentication.MethodEnum.PASSWORD) {
296       username = auth.getUsername();
297       password = auth.getPassword();
298     }
299 
300     if (wfsUrl != null && typeName != null) {
301       return new WFSTypeNameDescriptor(wfsUrl, typeName, username, password);
302     } else {
303       return null;
304     }
305   }
306 
307   private SimpleWFSLayerDescription getWFSLayerDescriptionForWMS(
308       GeoService wmsService, String layerName) {
309     String username = null;
310     String password = null;
311     if (wmsService.getAuthentication() != null
312         && wmsService.getAuthentication().getMethod()
313             == ServiceAuthentication.MethodEnum.PASSWORD) {
314       username = wmsService.getAuthentication().getUsername();
315       password = wmsService.getAuthentication().getPassword();
316     }
317     SimpleWFSLayerDescription wfsLayerDescription =
318         SimpleWFSHelper.describeWMSLayer(wmsService.getUrl(), username, password, layerName);
319     if (wfsLayerDescription != null && wfsLayerDescription.getTypeNames().length > 0) {
320       logger.info(
321           "WMS described layer \"{}\" with typeNames \"{}\" of WFS \"{}\" for WMS \"{}\"",
322           layerName,
323           Arrays.toString(wfsLayerDescription.getTypeNames()),
324           wfsLayerDescription.getWfsUrl(),
325           wmsService.getUrl());
326 
327       return wfsLayerDescription;
328     }
329     return null;
330   }
331 
332   /**
333    * Get the (exposed) attribute names of the WFS feature type.
334    *
335    * @param wfsTypeNameDescriptor provides the WFS feature type to get the attribute names for
336    * @return a list of attribute names for the WFS feature type
337    * @throws IOException if there were any problems setting up (creating or connecting) the
338    *     datasource.
339    */
340   private static List<String> getWFSAttributeNames(WFSTypeNameDescriptor wfsTypeNameDescriptor)
341       throws IOException {
342     Map<String, Object> connectionParameters = new HashMap<>();
343     connectionParameters.put(
344         WFSDataStoreFactory.URL.key,
345         SimpleWFSHelper.getWFSRequestURL(wfsTypeNameDescriptor.wfsUrl(), "GetCapabilities")
346             .toURL());
347     connectionParameters.put(WFSDataStoreFactory.PROTOCOL.key, Boolean.FALSE);
348     connectionParameters.put(WFSDataStoreFactory.WFS_STRATEGY.key, "geoserver");
349     connectionParameters.put(WFSDataStoreFactory.LENIENT.key, Boolean.TRUE);
350     connectionParameters.put(WFSDataStoreFactory.TIMEOUT.key, SimpleWFSHelper.TIMEOUT);
351     if (wfsTypeNameDescriptor.username() != null) {
352       connectionParameters.put(WFSDataStoreFactory.USERNAME.key, wfsTypeNameDescriptor.username());
353       connectionParameters.put(WFSDataStoreFactory.PASSWORD.key, wfsTypeNameDescriptor.password());
354     }
355 
356     WFSDataStore wfs = new WFSDataStoreFactory().createDataStore(connectionParameters);
357     List<String> attributeNames =
358         wfs
359             .getFeatureSource(wfsTypeNameDescriptor.typeName())
360             .getSchema()
361             .getAttributeDescriptors()
362             .stream()
363             .map(AttributeDescriptor::getLocalName)
364             .toList();
365 
366     wfs.dispose();
367     return attributeNames;
368   }
369 }