View Javadoc
1   /*
2    * Copyright (C) 2024 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller.admin;
7   
8   import com.fasterxml.jackson.databind.ObjectMapper;
9   import io.micrometer.core.annotation.Timed;
10  import io.micrometer.core.instrument.Metrics;
11  import io.swagger.v3.oas.annotations.Operation;
12  import io.swagger.v3.oas.annotations.media.Content;
13  import io.swagger.v3.oas.annotations.media.Schema;
14  import io.swagger.v3.oas.annotations.responses.ApiResponse;
15  import java.io.IOException;
16  import java.lang.invoke.MethodHandles;
17  import java.util.Map;
18  import java.util.NoSuchElementException;
19  import java.util.UUID;
20  import java.util.concurrent.TimeUnit;
21  import org.apache.solr.client.solrj.SolrClient;
22  import org.apache.solr.client.solrj.SolrServerException;
23  import org.apache.solr.client.solrj.response.SolrPingResponse;
24  import org.apache.solr.common.SolrException;
25  import org.quartz.JobKey;
26  import org.quartz.Scheduler;
27  import org.quartz.SchedulerException;
28  import org.slf4j.Logger;
29  import org.slf4j.LoggerFactory;
30  import org.springframework.beans.factory.InitializingBean;
31  import org.springframework.beans.factory.annotation.Autowired;
32  import org.springframework.beans.factory.annotation.Value;
33  import org.springframework.http.HttpStatus;
34  import org.springframework.http.MediaType;
35  import org.springframework.http.ResponseEntity;
36  import org.springframework.transaction.annotation.Transactional;
37  import org.springframework.web.bind.annotation.DeleteMapping;
38  import org.springframework.web.bind.annotation.ExceptionHandler;
39  import org.springframework.web.bind.annotation.GetMapping;
40  import org.springframework.web.bind.annotation.PathVariable;
41  import org.springframework.web.bind.annotation.PutMapping;
42  import org.springframework.web.bind.annotation.RestController;
43  import org.springframework.web.server.ResponseStatusException;
44  import org.tailormap.api.admin.model.SearchIndexSummary;
45  import org.tailormap.api.persistence.SearchIndex;
46  import org.tailormap.api.persistence.TMFeatureSource;
47  import org.tailormap.api.persistence.TMFeatureType;
48  import org.tailormap.api.repository.FeatureTypeRepository;
49  import org.tailormap.api.repository.SearchIndexRepository;
50  import org.tailormap.api.scheduling.IndexTask;
51  import org.tailormap.api.scheduling.SolrPingTask;
52  import org.tailormap.api.scheduling.TMJobDataMap;
53  import org.tailormap.api.scheduling.Task;
54  import org.tailormap.api.scheduling.TaskManagerService;
55  import org.tailormap.api.scheduling.TaskType;
56  import org.tailormap.api.solr.SolrHelper;
57  import org.tailormap.api.solr.SolrService;
58  import org.tailormap.api.viewer.model.ErrorResponse;
59  
60  /** Admin controller for Solr. */
61  @RestController
62  public class SolrAdminController implements InitializingBean {
63    private static final Logger logger =
64        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
65  
66    private final FeatureTypeRepository featureTypeRepository;
67    private final SearchIndexRepository searchIndexRepository;
68    private final SolrService solrService;
69    private final Scheduler scheduler;
70    private final TaskManagerService taskManagerService;
71  
72    @Value("${tailormap-api.solr-query-timeout-seconds:7}")
73    private int solrQueryTimeout;
74  
75    @Value("${tailormap-api.solr-api-ping-cron:0 0/5 * 1/1 * ? *}")
76    private String solrPingCron;
77  
78    public SolrAdminController(
79        @Autowired FeatureTypeRepository featureTypeRepository,
80        @Autowired SearchIndexRepository searchIndexRepository,
81        @Autowired SolrService solrService,
82        @Autowired Scheduler scheduler,
83        @Autowired TaskManagerService taskManagerService) {
84      this.featureTypeRepository = featureTypeRepository;
85      this.searchIndexRepository = searchIndexRepository;
86      this.solrService = solrService;
87      this.scheduler = scheduler;
88      this.taskManagerService = taskManagerService;
89    }
90  
91    @ExceptionHandler({ResponseStatusException.class})
92    public ResponseEntity<?> handleException(ResponseStatusException ex) {
93      // wrap the exception in a proper json response
94      return ResponseEntity.status(ex.getStatusCode())
95          .contentType(MediaType.APPLICATION_JSON)
96          .body(new ErrorResponse()
97              .message(
98                  ex.getReason() != null
99                      ? ex.getReason()
100                     : ex.getBody().getTitle())
101             .code(ex.getStatusCode().value()));
102   }
103 
104   /**
105    * Ping solr.
106    *
107    * @return the response entity (ok or an error response)
108    */
109   @Operation(summary = "Ping Solr", description = "Ping Solr to check if it is available")
110   @ApiResponse(
111       responseCode = "200",
112       description = "Solr is available",
113       content =
114           @Content(
115               mediaType = MediaType.APPLICATION_JSON_VALUE,
116               schema = @Schema(example = "{\"status\":\"OK\",\"timeElapsed\":1}")))
117   @ApiResponse(
118       responseCode = "500",
119       description = "Solr is not available",
120       content =
121           @Content(
122               mediaType = MediaType.APPLICATION_JSON_VALUE,
123               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
124   @GetMapping(path = "${tailormap-api.admin.base-path}/index/ping", produces = MediaType.APPLICATION_JSON_VALUE)
125   public ResponseEntity<?> pingSolr() {
126     try (SolrClient solrClient = solrService.getSolrClientForSearching()) {
127       final SolrPingResponse ping = solrClient.ping();
128       logger.info("Solr ping status {}", ping.getResponse().get("status"));
129       Metrics.timer("tailormap_solr_ping").record(ping.getElapsedTime(), TimeUnit.MILLISECONDS);
130       return ResponseEntity.ok(new ObjectMapper()
131           .createObjectNode()
132           .put("status", ping.getResponse().get("status").toString())
133           .put("timeElapsed", ping.getElapsedTime()));
134     } catch (IOException | SolrServerException e) {
135       logger.error("Error pinging solr", e);
136       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
137     }
138   }
139 
140   /**
141    * (re-) Index a layer.
142    *
143    * @param searchIndexId the searchIndex id
144    * @return the response entity (accepted or an error response)
145    */
146   @Operation(
147       summary = "Create or update a feature type index",
148       description = "Create or update a feature type index for a layer, will erase existing index if present")
149   @ApiResponse(
150       responseCode = "202",
151       description = "Index create or update request accepted",
152       content =
153           @Content(
154               mediaType = MediaType.APPLICATION_JSON_VALUE,
155               schema =
156                   @Schema(
157                       example =
158                           "{\"type\":\"index\", \"uuid\":\"6308d26e-fe1e-4268-bb28-20db2cd06914\",\"code\":202}")))
159   @ApiResponse(
160       responseCode = "404",
161       description = "Layer does not have feature type",
162       content =
163           @Content(
164               mediaType = MediaType.APPLICATION_JSON_VALUE,
165               schema =
166                   @Schema(
167                       example =
168                           "{\"message\":\"Layer does not have feature type\",\"code\":404}")))
169   @ApiResponse(
170       responseCode = "400",
171       description = "Indexing WFS feature types is not supported",
172       content =
173           @Content(
174               mediaType = MediaType.APPLICATION_JSON_VALUE,
175               schema =
176                   @Schema(
177                       example =
178                           "{\"message\":\"Layer does not have valid feature type for indexing\",\"code\":400}")))
179   @ApiResponse(
180       responseCode = "500",
181       description = "Error while indexing",
182       content =
183           @Content(
184               mediaType = MediaType.APPLICATION_JSON_VALUE,
185               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
186   @Transactional
187   @Timed(value = "index_feature_type", description = "time spent to index feature type")
188   @PutMapping(
189       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
190       produces = MediaType.APPLICATION_JSON_VALUE)
191   public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
192     SearchIndex searchIndex = validateInputAndFindIndex(searchIndexId);
193 
194     if (searchIndex.getStatus() == SearchIndex.Status.INDEXING) {
195       throw new ResponseStatusException(
196           HttpStatus.CONFLICT, "Indexing already in progress, check tasks overview before retrying");
197     }
198 
199     boolean createNewIndex =
200         (null == searchIndex.getLastIndexed() || searchIndex.getStatus() == SearchIndex.Status.INITIAL);
201 
202     boolean hasSchedule = (null != searchIndex.getSchedule()
203         && null != searchIndex.getSchedule().getUuid());
204 
205     UUID taskUuid;
206     try {
207       if (hasSchedule) {
208         taskUuid = searchIndex.getSchedule().getUuid();
209         startScheduledJobIndexing(searchIndex);
210       } else {
211         taskUuid = startOneTimeJobIndexing(searchIndex);
212       }
213       searchIndexRepository.save(searchIndex);
214     } catch (UnsupportedOperationException
215         | IOException
216         | SolrServerException
217         | SolrException
218         | SchedulerException e) {
219       logger.error("Error indexing", e);
220       searchIndex.setStatus(SearchIndex.Status.ERROR);
221       searchIndexRepository.save(searchIndex);
222       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
223     }
224 
225     logger.info(
226         "Scheduled {} index for search index {}",
227         (createNewIndex ? "creation of a new" : "update of"),
228         searchIndex.getName());
229     return ResponseEntity.accepted()
230         .body(Map.of(
231             "code",
232             202,
233             Task.TYPE_KEY,
234             TaskType.INDEX.getValue(),
235             Task.UUID_KEY,
236             taskUuid,
237             "message",
238             "Indexing scheduled"));
239   }
240 
241   private UUID startOneTimeJobIndexing(SearchIndex searchIndex)
242       throws SolrServerException, IOException, SchedulerException {
243     UUID taskName = taskManagerService.createTask(
244         IndexTask.class,
245         new TMJobDataMap(Map.of(
246             Task.TYPE_KEY,
247             TaskType.INDEX,
248             Task.DESCRIPTION_KEY,
249             "One-time indexing of " + searchIndex.getName(),
250             IndexTask.INDEX_KEY,
251             searchIndex.getId().toString(),
252             Task.PRIORITY_KEY,
253             0)));
254     logger.info("One-time indexing job with UUID {} started", taskName);
255     return taskName;
256   }
257 
258   private void startScheduledJobIndexing(SearchIndex searchIndex) throws SchedulerException {
259     JobKey jobKey = taskManagerService.getJobKey(
260         TaskType.INDEX, searchIndex.getSchedule().getUuid());
261     if (null == jobKey) {
262       throw new SchedulerException("Indexing job not found in scheduler");
263     }
264     scheduler.triggerJob(jobKey);
265     logger.info(
266         "Indexing of scheduled job with UUID {} started",
267         searchIndex.getSchedule().getUuid());
268   }
269 
270   /**
271    * Validate input and find the search index.
272    *
273    * @param searchIndexId the search index id
274    * @return the search index
275    * @throws ResponseStatusException if the search index is not found or the feature type is not found
276    */
277   private SearchIndex validateInputAndFindIndex(Long searchIndexId) {
278     // check if solr is available
279     this.pingSolr();
280 
281     // check if search index exists
282     SearchIndex searchIndex = searchIndexRepository
283         .findById(searchIndexId)
284         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
285 
286     // check if feature type exists
287     TMFeatureType indexingFT = featureTypeRepository
288         .findById(searchIndex.getFeatureTypeId())
289         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Feature type not found"));
290 
291     if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) {
292       // the search index should not exist for WFS feature types, but test just in case
293       searchIndex
294           .setStatus(SearchIndex.Status.ERROR)
295           .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported"));
296       throw new ResponseStatusException(
297           HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing");
298     }
299     return searchIndex;
300   }
301 
302   /**
303    * Clear an index; does not remove the {@link SearchIndex} metadata.
304    *
305    * @param searchIndexId the searchindex id
306    * @return the response entity ({@code 204 NOCONTENT} or an error response)
307    */
308   @Operation(summary = "Clear index for a feature type", description = "Clear index for the feature type")
309   @ApiResponse(responseCode = "204", description = "Index cleared")
310   @ApiResponse(responseCode = "404", description = "Index not configured for feature type")
311   @ApiResponse(
312       responseCode = "500",
313       description = "Error while clearing index",
314       content =
315           @Content(
316               mediaType = MediaType.APPLICATION_JSON_VALUE,
317               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
318   @Timed(value = "index_delete", description = "time spent to delete an index of a feature type")
319   @DeleteMapping(
320       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
321       produces = MediaType.APPLICATION_JSON_VALUE)
322   @Transactional
323   public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
324     try (SolrClient solrClient = solrService.getSolrClientForSearching();
325         SolrHelper solrHelper = new SolrHelper(solrClient).withQueryTimeout(solrQueryTimeout)) {
326       solrHelper.clearIndexForLayer(searchIndexId);
327       // do not delete the SearchIndex metadata object
328       // searchIndexRepository.findById(searchIndexId).ifPresent(searchIndexRepository::delete);
329       SearchIndex searchIndex = searchIndexRepository
330           .findById(searchIndexId)
331           .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
332       searchIndex
333           .setLastIndexed(null)
334           .setStatus(SearchIndex.Status.INITIAL)
335           .setSummary(new SearchIndexSummary().total(0));
336       searchIndexRepository.save(searchIndex);
337     } catch (IOException | SolrServerException | NoSuchElementException e) {
338       logger.warn("Error clearing index", e);
339       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
340     }
341 
342     logger.info("Index cleared for index {}", searchIndexId);
343     return ResponseEntity.noContent().build();
344   }
345 
346   @Override
347   public void afterPropertiesSet() throws Exception {
348     if (solrService.isSolrServiceAvailable()) {
349       logger.info("Solr is available, initializing SolrAdminController.");
350       try {
351         // there should be only one Solr ping task, so we delete any existing ones
352         taskManagerService.deleteTasksByGroupName(TaskType.SOLR_PING.getValue());
353         final UUID taskUuid = taskManagerService.createTask(
354             SolrPingTask.class,
355             new TMJobDataMap(Map.of(
356                 Task.TYPE_KEY,
357                 TaskType.SOLR_PING.getValue(),
358                 Task.DESCRIPTION_KEY,
359                 "Ping Solr service for availability.",
360                 Task.PRIORITY_KEY,
361                 58)),
362             solrPingCron);
363         logger.debug("Added Solr ping task with UUID: {}", taskUuid);
364       } catch (Exception e) {
365         logger.error("Error initializing Solr ping task", e);
366       }
367     } else {
368       logger.info("Solr is not available, /search endpoint will not be functional.");
369     }
370   }
371 }