View Javadoc
1   /*
2    * Copyright (C) 2024 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller.admin;
7   
8   import com.fasterxml.jackson.databind.ObjectMapper;
9   import io.micrometer.core.annotation.Timed;
10  import io.swagger.v3.oas.annotations.Operation;
11  import io.swagger.v3.oas.annotations.media.Content;
12  import io.swagger.v3.oas.annotations.media.Schema;
13  import io.swagger.v3.oas.annotations.responses.ApiResponse;
14  import java.io.IOException;
15  import java.lang.invoke.MethodHandles;
16  import java.util.Map;
17  import java.util.NoSuchElementException;
18  import java.util.UUID;
19  import org.apache.solr.client.solrj.SolrClient;
20  import org.apache.solr.client.solrj.SolrServerException;
21  import org.apache.solr.client.solrj.response.SolrPingResponse;
22  import org.apache.solr.common.SolrException;
23  import org.quartz.JobKey;
24  import org.quartz.Scheduler;
25  import org.quartz.SchedulerException;
26  import org.slf4j.Logger;
27  import org.slf4j.LoggerFactory;
28  import org.springframework.beans.factory.annotation.Autowired;
29  import org.springframework.beans.factory.annotation.Value;
30  import org.springframework.http.HttpStatus;
31  import org.springframework.http.MediaType;
32  import org.springframework.http.ResponseEntity;
33  import org.springframework.transaction.annotation.Transactional;
34  import org.springframework.web.bind.annotation.DeleteMapping;
35  import org.springframework.web.bind.annotation.ExceptionHandler;
36  import org.springframework.web.bind.annotation.GetMapping;
37  import org.springframework.web.bind.annotation.PathVariable;
38  import org.springframework.web.bind.annotation.PutMapping;
39  import org.springframework.web.bind.annotation.RestController;
40  import org.springframework.web.server.ResponseStatusException;
41  import org.tailormap.api.admin.model.SearchIndexSummary;
42  import org.tailormap.api.persistence.SearchIndex;
43  import org.tailormap.api.persistence.TMFeatureSource;
44  import org.tailormap.api.persistence.TMFeatureType;
45  import org.tailormap.api.repository.FeatureTypeRepository;
46  import org.tailormap.api.repository.SearchIndexRepository;
47  import org.tailormap.api.scheduling.IndexTask;
48  import org.tailormap.api.scheduling.TMJobDataMap;
49  import org.tailormap.api.scheduling.Task;
50  import org.tailormap.api.scheduling.TaskManagerService;
51  import org.tailormap.api.scheduling.TaskType;
52  import org.tailormap.api.solr.SolrHelper;
53  import org.tailormap.api.solr.SolrService;
54  import org.tailormap.api.viewer.model.ErrorResponse;
55  
56  /** Admin controller for Solr. */
57  @RestController
58  public class SolrAdminController {
59    private static final Logger logger =
60        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
61  
62    private final FeatureTypeRepository featureTypeRepository;
63    private final SearchIndexRepository searchIndexRepository;
64    private final SolrService solrService;
65    private final Scheduler scheduler;
66    private final TaskManagerService taskManagerService;
67  
68    @Value("${tailormap-api.solr-query-timeout-seconds:7}")
69    private int solrQueryTimeout;
70  
71    public SolrAdminController(
72        @Autowired FeatureTypeRepository featureTypeRepository,
73        @Autowired SearchIndexRepository searchIndexRepository,
74        @Autowired SolrService solrService,
75        @Autowired Scheduler scheduler,
76        @Autowired TaskManagerService taskManagerService) {
77      this.featureTypeRepository = featureTypeRepository;
78      this.searchIndexRepository = searchIndexRepository;
79      this.solrService = solrService;
80      this.scheduler = scheduler;
81      this.taskManagerService = taskManagerService;
82    }
83  
84    @ExceptionHandler({ResponseStatusException.class})
85    public ResponseEntity<?> handleException(ResponseStatusException ex) {
86      // wrap the exception in a proper json response
87      return ResponseEntity.status(ex.getStatusCode())
88          .contentType(MediaType.APPLICATION_JSON)
89          .body(new ErrorResponse()
90              .message(
91                  ex.getReason() != null
92                      ? ex.getReason()
93                      : ex.getBody().getTitle())
94              .code(ex.getStatusCode().value()));
95    }
96  
97    /**
98     * Ping solr.
99     *
100    * @return the response entity (ok or an error response)
101    */
102   @Operation(summary = "Ping Solr", description = "Ping Solr to check if it is available")
103   @ApiResponse(
104       responseCode = "200",
105       description = "Solr is available",
106       content =
107           @Content(
108               mediaType = MediaType.APPLICATION_JSON_VALUE,
109               schema = @Schema(example = "{\"status\":\"OK\",\"timeElapsed\":1}")))
110   @ApiResponse(
111       responseCode = "500",
112       description = "Solr is not available",
113       content =
114           @Content(
115               mediaType = MediaType.APPLICATION_JSON_VALUE,
116               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
117   @GetMapping(path = "${tailormap-api.admin.base-path}/index/ping", produces = MediaType.APPLICATION_JSON_VALUE)
118   public ResponseEntity<?> pingSolr() {
119     try (SolrClient solrClient = solrService.getSolrClientForSearching()) {
120       final SolrPingResponse ping = solrClient.ping();
121       logger.info("Solr ping status {}", ping.getResponse().get("status"));
122       return ResponseEntity.ok(new ObjectMapper()
123           .createObjectNode()
124           .put("status", ping.getResponse().get("status").toString())
125           .put("timeElapsed", ping.getElapsedTime()));
126     } catch (IOException | SolrServerException e) {
127       logger.error("Error pinging solr", e);
128       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
129     }
130   }
131 
132   /**
133    * (re-) Index a layer.
134    *
135    * @param searchIndexId the searchIndex id
136    * @return the response entity (accepted or an error response)
137    */
138   @Operation(
139       summary = "Create or update a feature type index",
140       description = "Create or update a feature type index for a layer, will erase existing index if present")
141   @ApiResponse(
142       responseCode = "202",
143       description = "Index create or update request accepted",
144       content =
145           @Content(
146               mediaType = MediaType.APPLICATION_JSON_VALUE,
147               schema =
148                   @Schema(
149                       example =
150                           "{\"type\":\"index\", \"uuid\":\"6308d26e-fe1e-4268-bb28-20db2cd06914\",\"code\":202}")))
151   @ApiResponse(
152       responseCode = "404",
153       description = "Layer does not have feature type",
154       content =
155           @Content(
156               mediaType = MediaType.APPLICATION_JSON_VALUE,
157               schema =
158                   @Schema(
159                       example =
160                           "{\"message\":\"Layer does not have feature type\",\"code\":404}")))
161   @ApiResponse(
162       responseCode = "400",
163       description = "Indexing WFS feature types is not supported",
164       content =
165           @Content(
166               mediaType = MediaType.APPLICATION_JSON_VALUE,
167               schema =
168                   @Schema(
169                       example =
170                           "{\"message\":\"Layer does not have valid feature type for indexing\",\"code\":400}")))
171   @ApiResponse(
172       responseCode = "500",
173       description = "Error while indexing",
174       content =
175           @Content(
176               mediaType = MediaType.APPLICATION_JSON_VALUE,
177               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
178   @Transactional
179   @Timed(value = "index_feature_type", description = "time spent to index feature type")
180   @PutMapping(
181       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
182       produces = MediaType.APPLICATION_JSON_VALUE)
183   public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
184     SearchIndex searchIndex = validateInputAndFindIndex(searchIndexId);
185 
186     if (searchIndex.getStatus() == SearchIndex.Status.INDEXING) {
187       throw new ResponseStatusException(
188           HttpStatus.CONFLICT, "Indexing already in progress, check tasks overview before retrying");
189     }
190 
191     boolean createNewIndex =
192         (null == searchIndex.getLastIndexed() || searchIndex.getStatus() == SearchIndex.Status.INITIAL);
193 
194     boolean hasSchedule = (null != searchIndex.getSchedule()
195         && null != searchIndex.getSchedule().getUuid());
196 
197     UUID taskUuid;
198     try {
199       if (hasSchedule) {
200         taskUuid = searchIndex.getSchedule().getUuid();
201         startScheduledJobIndexing(searchIndex);
202       } else {
203         taskUuid = startOneTimeJobIndexing(searchIndex);
204       }
205       searchIndexRepository.save(searchIndex);
206     } catch (UnsupportedOperationException
207         | IOException
208         | SolrServerException
209         | SolrException
210         | SchedulerException e) {
211       logger.error("Error indexing", e);
212       searchIndex.setStatus(SearchIndex.Status.ERROR);
213       searchIndexRepository.save(searchIndex);
214       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
215     }
216 
217     logger.info(
218         "Scheduled {} index for search index {}",
219         (createNewIndex ? "creation of a new" : "update of"),
220         searchIndex.getName());
221     return ResponseEntity.accepted()
222         .body(Map.of(
223             "code",
224             202,
225             Task.TYPE_KEY,
226             TaskType.INDEX.getValue(),
227             Task.UUID_KEY,
228             taskUuid,
229             "message",
230             "Indexing scheduled"));
231   }
232 
233   private UUID startOneTimeJobIndexing(SearchIndex searchIndex)
234       throws SolrServerException, IOException, SchedulerException {
235     UUID taskName = taskManagerService.createTask(
236         IndexTask.class,
237         new TMJobDataMap(Map.of(
238             Task.TYPE_KEY,
239             TaskType.INDEX,
240             Task.DESCRIPTION_KEY,
241             "One-time indexing of " + searchIndex.getName(),
242             IndexTask.INDEX_KEY,
243             searchIndex.getId().toString(),
244             Task.PRIORITY_KEY,
245             0)));
246     logger.info("One-time indexing job with UUID {} started", taskName);
247     return taskName;
248   }
249 
250   private void startScheduledJobIndexing(SearchIndex searchIndex) throws SchedulerException {
251     JobKey jobKey = taskManagerService.getJobKey(
252         TaskType.INDEX, searchIndex.getSchedule().getUuid());
253     if (null == jobKey) {
254       throw new SchedulerException("Indexing job not found in scheduler");
255     }
256     scheduler.triggerJob(jobKey);
257     logger.info(
258         "Indexing of scheduled job with UUID {} started",
259         searchIndex.getSchedule().getUuid());
260   }
261 
262   /**
263    * Validate input and find the search index.
264    *
265    * @param searchIndexId the search index id
266    * @return the search index
267    * @throws ResponseStatusException if the search index is not found or the feature type is not found
268    */
269   private SearchIndex validateInputAndFindIndex(Long searchIndexId) {
270     // check if solr is available
271     this.pingSolr();
272 
273     // check if search index exists
274     SearchIndex searchIndex = searchIndexRepository
275         .findById(searchIndexId)
276         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
277 
278     // check if feature type exists
279     TMFeatureType indexingFT = featureTypeRepository
280         .findById(searchIndex.getFeatureTypeId())
281         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Feature type not found"));
282 
283     if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) {
284       // the search index should not exist for WFS feature types, but test just in case
285       searchIndex
286           .setStatus(SearchIndex.Status.ERROR)
287           .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported"));
288       throw new ResponseStatusException(
289           HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing");
290     }
291     return searchIndex;
292   }
293 
294   /**
295    * Clear an index; does not remove the {@link SearchIndex} metadata.
296    *
297    * @param searchIndexId the searchindex id
298    * @return the response entity ({@code 204 NOCONTENT} or an error response)
299    */
300   @Operation(summary = "Clear index for a feature type", description = "Clear index for the feature type")
301   @ApiResponse(responseCode = "204", description = "Index cleared")
302   @ApiResponse(responseCode = "404", description = "Index not configured for feature type")
303   @ApiResponse(
304       responseCode = "500",
305       description = "Error while clearing index",
306       content =
307           @Content(
308               mediaType = MediaType.APPLICATION_JSON_VALUE,
309               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
310   @Timed(value = "index_delete", description = "time spent to delete an index of a feature type")
311   @DeleteMapping(
312       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
313       produces = MediaType.APPLICATION_JSON_VALUE)
314   @Transactional
315   public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
316     try (SolrClient solrClient = solrService.getSolrClientForSearching();
317         SolrHelper solrHelper = new SolrHelper(solrClient).withQueryTimeout(solrQueryTimeout)) {
318       solrHelper.clearIndexForLayer(searchIndexId);
319       // do not delete the SearchIndex metadata object
320       // searchIndexRepository.findById(searchIndexId).ifPresent(searchIndexRepository::delete);
321       SearchIndex searchIndex = searchIndexRepository
322           .findById(searchIndexId)
323           .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
324       searchIndex
325           .setLastIndexed(null)
326           .setStatus(SearchIndex.Status.INITIAL)
327           .setSummary(new SearchIndexSummary().total(0));
328       searchIndexRepository.save(searchIndex);
329     } catch (IOException | SolrServerException | NoSuchElementException e) {
330       logger.warn("Error clearing index", e);
331       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
332     }
333 
334     logger.info("Index cleared for index {}", searchIndexId);
335     return ResponseEntity.noContent().build();
336   }
337 }