View Javadoc
1   /*
2    * Copyright (C) 2024 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller.admin;
7   
8   import com.fasterxml.jackson.databind.ObjectMapper;
9   import io.micrometer.core.annotation.Timed;
10  import io.swagger.v3.oas.annotations.Operation;
11  import io.swagger.v3.oas.annotations.media.Content;
12  import io.swagger.v3.oas.annotations.media.Schema;
13  import io.swagger.v3.oas.annotations.responses.ApiResponse;
14  import java.io.IOException;
15  import java.lang.invoke.MethodHandles;
16  import java.util.Map;
17  import java.util.NoSuchElementException;
18  import java.util.UUID;
19  import org.apache.solr.client.solrj.SolrClient;
20  import org.apache.solr.client.solrj.SolrServerException;
21  import org.apache.solr.client.solrj.response.SolrPingResponse;
22  import org.apache.solr.common.SolrException;
23  import org.quartz.JobKey;
24  import org.quartz.Scheduler;
25  import org.quartz.SchedulerException;
26  import org.slf4j.Logger;
27  import org.slf4j.LoggerFactory;
28  import org.springframework.beans.factory.annotation.Autowired;
29  import org.springframework.beans.factory.annotation.Value;
30  import org.springframework.http.HttpStatus;
31  import org.springframework.http.MediaType;
32  import org.springframework.http.ResponseEntity;
33  import org.springframework.transaction.annotation.Transactional;
34  import org.springframework.web.bind.annotation.DeleteMapping;
35  import org.springframework.web.bind.annotation.ExceptionHandler;
36  import org.springframework.web.bind.annotation.GetMapping;
37  import org.springframework.web.bind.annotation.PathVariable;
38  import org.springframework.web.bind.annotation.PutMapping;
39  import org.springframework.web.bind.annotation.RestController;
40  import org.springframework.web.server.ResponseStatusException;
41  import org.tailormap.api.persistence.SearchIndex;
42  import org.tailormap.api.persistence.TMFeatureSource;
43  import org.tailormap.api.persistence.TMFeatureType;
44  import org.tailormap.api.repository.FeatureTypeRepository;
45  import org.tailormap.api.repository.SearchIndexRepository;
46  import org.tailormap.api.scheduling.IndexTask;
47  import org.tailormap.api.scheduling.TMJobDataMap;
48  import org.tailormap.api.scheduling.Task;
49  import org.tailormap.api.scheduling.TaskManagerService;
50  import org.tailormap.api.scheduling.TaskType;
51  import org.tailormap.api.solr.SolrHelper;
52  import org.tailormap.api.solr.SolrService;
53  import org.tailormap.api.viewer.model.ErrorResponse;
54  
55  /** Admin controller for Solr. */
56  @RestController
57  public class SolrAdminController {
58    private static final Logger logger =
59        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
60  
61    private final FeatureTypeRepository featureTypeRepository;
62    private final SearchIndexRepository searchIndexRepository;
63    private final SolrService solrService;
64    private final Scheduler scheduler;
65    private final TaskManagerService taskManagerService;
66  
67    @Value("${tailormap-api.solr-query-timeout-seconds:7}")
68    private int solrQueryTimeout;
69  
70    public SolrAdminController(
71        @Autowired FeatureTypeRepository featureTypeRepository,
72        @Autowired SearchIndexRepository searchIndexRepository,
73        @Autowired SolrService solrService,
74        @Autowired Scheduler scheduler,
75        @Autowired TaskManagerService taskManagerService) {
76      this.featureTypeRepository = featureTypeRepository;
77      this.searchIndexRepository = searchIndexRepository;
78      this.solrService = solrService;
79      this.scheduler = scheduler;
80      this.taskManagerService = taskManagerService;
81    }
82  
83    @ExceptionHandler({ResponseStatusException.class})
84    public ResponseEntity<?> handleException(ResponseStatusException ex) {
85      // wrap the exception in a proper json response
86      return ResponseEntity.status(ex.getStatusCode())
87          .contentType(MediaType.APPLICATION_JSON)
88          .body(
89              new ErrorResponse()
90                  .message(ex.getReason() != null ? ex.getReason() : ex.getBody().getTitle())
91                  .code(ex.getStatusCode().value()));
92    }
93  
94    /**
95     * Ping solr.
96     *
97     * @return the response entity (ok or an error response)
98     */
99    @Operation(summary = "Ping Solr", description = "Ping Solr to check if it is available")
100   @ApiResponse(
101       responseCode = "200",
102       description = "Solr is available",
103       content =
104           @Content(
105               mediaType = MediaType.APPLICATION_JSON_VALUE,
106               schema = @Schema(example = "{\"status\":\"OK\",\"timeElapsed\":1}")))
107   @ApiResponse(
108       responseCode = "500",
109       description = "Solr is not available",
110       content =
111           @Content(
112               mediaType = MediaType.APPLICATION_JSON_VALUE,
113               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
114   @GetMapping(
115       path = "${tailormap-api.admin.base-path}/index/ping",
116       produces = MediaType.APPLICATION_JSON_VALUE)
117   public ResponseEntity<?> pingSolr() {
118     try (SolrClient solrClient = solrService.getSolrClientForSearching()) {
119       final SolrPingResponse ping = solrClient.ping();
120       logger.info("Solr ping status {}", ping.getResponse().get("status"));
121       return ResponseEntity.ok(
122           new ObjectMapper()
123               .createObjectNode()
124               .put("status", ping.getResponse().get("status").toString())
125               .put("timeElapsed", ping.getElapsedTime()));
126     } catch (IOException | SolrServerException e) {
127       logger.error("Error pinging solr", e);
128       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
129     }
130   }
131 
132   /**
133    * (re-) Index a layer.
134    *
135    * @param searchIndexId the searchIndex id
136    * @return the response entity (accepted or an error response)
137    */
138   @Operation(
139       summary = "Create or update a feature type index",
140       description =
141           "Create or update a feature type index for a layer, will erase existing index if present")
142   @ApiResponse(
143       responseCode = "202",
144       description = "Index create or update request accepted",
145       content =
146           @Content(
147               mediaType = MediaType.APPLICATION_JSON_VALUE,
148               schema =
149                   @Schema(
150                       example =
151                           "{\"type\":\"index\", \"uuid\":\"6308d26e-fe1e-4268-bb28-20db2cd06914\",\"code\":202}")))
152   @ApiResponse(
153       responseCode = "404",
154       description = "Layer does not have feature type",
155       content =
156           @Content(
157               mediaType = MediaType.APPLICATION_JSON_VALUE,
158               schema =
159                   @Schema(
160                       example = "{\"message\":\"Layer does not have feature type\",\"code\":404}")))
161   @ApiResponse(
162       responseCode = "400",
163       description = "Indexing WFS feature types is not supported",
164       content =
165           @Content(
166               mediaType = MediaType.APPLICATION_JSON_VALUE,
167               schema =
168                   @Schema(
169                       example =
170                           "{\"message\":\"Layer does not have valid feature type for indexing\",\"code\":400}")))
171   @ApiResponse(
172       responseCode = "500",
173       description = "Error while indexing",
174       content =
175           @Content(
176               mediaType = MediaType.APPLICATION_JSON_VALUE,
177               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
178   @Transactional
179   @Timed(value = "index_feature_type", description = "time spent to index feature type")
180   @PutMapping(
181       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
182       produces = MediaType.APPLICATION_JSON_VALUE)
183   public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
184     SearchIndex searchIndex = validateInputAndFindIndex(searchIndexId);
185 
186     if (searchIndex.getStatus() == SearchIndex.Status.INDEXING) {
187       throw new ResponseStatusException(
188           HttpStatus.CONFLICT,
189           "Indexing already in progress, check tasks overview before retrying");
190     }
191 
192     boolean createNewIndex =
193         (null == searchIndex.getLastIndexed()
194             || searchIndex.getStatus() == SearchIndex.Status.INITIAL);
195 
196     boolean hasSchedule =
197         (null != searchIndex.getSchedule() && null != searchIndex.getSchedule().getUuid());
198 
199     UUID taskUuid;
200     try {
201       if (hasSchedule) {
202         taskUuid = searchIndex.getSchedule().getUuid();
203         startScheduledJobIndexing(searchIndex);
204       } else {
205         taskUuid = startOneTimeJobIndexing(searchIndex);
206       }
207       searchIndexRepository.save(searchIndex);
208     } catch (UnsupportedOperationException
209         | IOException
210         | SolrServerException
211         | SolrException
212         | SchedulerException e) {
213       logger.error("Error indexing", e);
214       searchIndex.setStatus(SearchIndex.Status.ERROR);
215       searchIndexRepository.save(searchIndex);
216       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
217     }
218 
219     logger.info(
220         "Scheduled {} index for search index {}",
221         (createNewIndex ? "creation of a new" : "update of"),
222         searchIndex.getName());
223     return ResponseEntity.accepted()
224         .body(
225             Map.of(
226                 "code",
227                 202,
228                 Task.TYPE_KEY,
229                 TaskType.INDEX.getValue(),
230                 Task.UUID_KEY,
231                 taskUuid,
232                 "message",
233                 "Indexing scheduled"));
234   }
235 
236   private UUID startOneTimeJobIndexing(SearchIndex searchIndex)
237       throws SolrServerException, IOException, SchedulerException {
238     UUID taskName =
239         taskManagerService.createTask(
240             IndexTask.class,
241             new TMJobDataMap(
242                 Map.of(
243                     Task.TYPE_KEY,
244                     TaskType.INDEX,
245                     Task.DESCRIPTION_KEY,
246                     "One-time indexing of " + searchIndex.getName(),
247                     IndexTask.INDEX_KEY,
248                     searchIndex.getId().toString(),
249                     Task.PRIORITY_KEY,
250                     0)));
251     logger.info("One-time indexing job with UUID {} started", taskName);
252     return taskName;
253   }
254 
255   private void startScheduledJobIndexing(SearchIndex searchIndex) throws SchedulerException {
256     JobKey jobKey =
257         taskManagerService.getJobKey(TaskType.INDEX, searchIndex.getSchedule().getUuid());
258     if (null == jobKey) {
259       throw new SchedulerException("Indexing job not found in scheduler");
260     }
261     scheduler.triggerJob(jobKey);
262     logger.info(
263         "Indexing of scheduled job with UUID {} started", searchIndex.getSchedule().getUuid());
264   }
265 
266   /**
267    * Validate input and find the search index.
268    *
269    * @param searchIndexId the search index id
270    * @return the search index
271    * @throws ResponseStatusException if the search index is not found or the feature type is not
272    *     found
273    */
274   private SearchIndex validateInputAndFindIndex(Long searchIndexId) {
275     // check if solr is available
276     this.pingSolr();
277 
278     // check if search index exists
279     SearchIndex searchIndex =
280         searchIndexRepository
281             .findById(searchIndexId)
282             .orElseThrow(
283                 () -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
284 
285     // check if feature type exists
286     TMFeatureType indexingFT =
287         featureTypeRepository
288             .findById(searchIndex.getFeatureTypeId())
289             .orElseThrow(
290                 () -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Feature type not found"));
291 
292     if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) {
293       // the search index should not exist for WFS feature types, but test just in case
294       searchIndex.setStatus(SearchIndex.Status.ERROR).setComment("WFS indexing not supported");
295       throw new ResponseStatusException(
296           HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing");
297     }
298     return searchIndex;
299   }
300 
301   /**
302    * Clear an index; does not remove the {@link SearchIndex} metadata.
303    *
304    * @param searchIndexId the searchindex id
305    * @return the response entity ({@code 204 NOCONTENT} or an error response)
306    */
307   @Operation(
308       summary = "Clear index for a feature type",
309       description = "Clear index for the feature type")
310   @ApiResponse(responseCode = "204", description = "Index cleared")
311   @ApiResponse(responseCode = "404", description = "Index not configured for feature type")
312   @ApiResponse(
313       responseCode = "500",
314       description = "Error while clearing index",
315       content =
316           @Content(
317               mediaType = MediaType.APPLICATION_JSON_VALUE,
318               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
319   @Timed(value = "index_delete", description = "time spent to delete an index of a feature type")
320   @DeleteMapping(
321       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
322       produces = MediaType.APPLICATION_JSON_VALUE)
323   @Transactional
324   public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
325     try (SolrClient solrClient = solrService.getSolrClientForSearching();
326         SolrHelper solrHelper = new SolrHelper(solrClient).withQueryTimeout(solrQueryTimeout)) {
327       solrHelper.clearIndexForLayer(searchIndexId);
328       // do not delete the SearchIndex metadata object
329       // searchIndexRepository.findById(searchIndexId).ifPresent(searchIndexRepository::delete);
330       SearchIndex searchIndex =
331           searchIndexRepository
332               .findById(searchIndexId)
333               .orElseThrow(
334                   () ->
335                       new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
336       searchIndex
337           .setLastIndexed(null)
338           .setStatus(SearchIndex.Status.INITIAL)
339           .setComment("Index cleared");
340       searchIndexRepository.save(searchIndex);
341     } catch (IOException | SolrServerException | NoSuchElementException e) {
342       logger.warn("Error clearing index", e);
343       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
344     }
345 
346     logger.info("Index cleared for index {}", searchIndexId);
347     return ResponseEntity.noContent().build();
348   }
349 }