View Javadoc
1   /*
2    * Copyright (C) 2024 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller.admin;
7   
8   import io.micrometer.core.annotation.Timed;
9   import io.micrometer.core.instrument.Metrics;
10  import io.swagger.v3.oas.annotations.Operation;
11  import io.swagger.v3.oas.annotations.media.Content;
12  import io.swagger.v3.oas.annotations.media.Schema;
13  import io.swagger.v3.oas.annotations.responses.ApiResponse;
14  import java.io.IOException;
15  import java.lang.invoke.MethodHandles;
16  import java.util.Map;
17  import java.util.NoSuchElementException;
18  import java.util.UUID;
19  import java.util.concurrent.TimeUnit;
20  import org.apache.solr.client.solrj.SolrClient;
21  import org.apache.solr.client.solrj.SolrServerException;
22  import org.apache.solr.client.solrj.request.HealthCheckRequest;
23  import org.apache.solr.client.solrj.response.HealthCheckResponse;
24  import org.apache.solr.common.SolrException;
25  import org.quartz.JobKey;
26  import org.quartz.Scheduler;
27  import org.quartz.SchedulerException;
28  import org.slf4j.Logger;
29  import org.slf4j.LoggerFactory;
30  import org.springframework.beans.factory.InitializingBean;
31  import org.springframework.beans.factory.annotation.Autowired;
32  import org.springframework.beans.factory.annotation.Value;
33  import org.springframework.http.HttpStatus;
34  import org.springframework.http.MediaType;
35  import org.springframework.http.ResponseEntity;
36  import org.springframework.transaction.annotation.Transactional;
37  import org.springframework.web.bind.annotation.DeleteMapping;
38  import org.springframework.web.bind.annotation.ExceptionHandler;
39  import org.springframework.web.bind.annotation.GetMapping;
40  import org.springframework.web.bind.annotation.PathVariable;
41  import org.springframework.web.bind.annotation.PutMapping;
42  import org.springframework.web.bind.annotation.RestController;
43  import org.springframework.web.server.ResponseStatusException;
44  import org.tailormap.api.admin.model.SearchIndexSummary;
45  import org.tailormap.api.persistence.SearchIndex;
46  import org.tailormap.api.persistence.TMFeatureSource;
47  import org.tailormap.api.persistence.TMFeatureType;
48  import org.tailormap.api.repository.FeatureTypeRepository;
49  import org.tailormap.api.repository.SearchIndexRepository;
50  import org.tailormap.api.scheduling.IndexTask;
51  import org.tailormap.api.scheduling.SolrPingTask;
52  import org.tailormap.api.scheduling.TMJobDataMap;
53  import org.tailormap.api.scheduling.Task;
54  import org.tailormap.api.scheduling.TaskManagerService;
55  import org.tailormap.api.scheduling.TaskType;
56  import org.tailormap.api.solr.SolrHelper;
57  import org.tailormap.api.solr.SolrService;
58  import org.tailormap.api.viewer.model.ErrorResponse;
59  import tools.jackson.databind.json.JsonMapper;
60  
61  /** Admin controller for Solr. */
62  @RestController
63  public class SolrAdminController implements InitializingBean {
64    private static final Logger logger =
65        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
66  
67    private final FeatureTypeRepository featureTypeRepository;
68    private final SearchIndexRepository searchIndexRepository;
69    private final SolrService solrService;
70    private final Scheduler scheduler;
71    private final TaskManagerService taskManagerService;
72    private final JsonMapper jsonMapper;
73  
74    @Value("${tailormap-api.solr-query-timeout-seconds:7}")
75    private int solrQueryTimeout;
76  
77    @Value("${tailormap-api.solr-api-ping-cron:0 0/5 * 1/1 * ? *}")
78    private String solrPingCron;
79  
80    public SolrAdminController(
81        @Autowired FeatureTypeRepository featureTypeRepository,
82        @Autowired SearchIndexRepository searchIndexRepository,
83        @Autowired SolrService solrService,
84        @Autowired Scheduler scheduler,
85        @Autowired TaskManagerService taskManagerService,
86        JsonMapper jsonMapper) {
87      this.featureTypeRepository = featureTypeRepository;
88      this.searchIndexRepository = searchIndexRepository;
89      this.solrService = solrService;
90      this.scheduler = scheduler;
91      this.taskManagerService = taskManagerService;
92      this.jsonMapper = jsonMapper;
93    }
94  
95    @ExceptionHandler({ResponseStatusException.class})
96    public ResponseEntity<?> handleException(ResponseStatusException ex) {
97      // wrap the exception in a proper json response
98      return ResponseEntity.status(ex.getStatusCode())
99          .contentType(MediaType.APPLICATION_JSON)
100         .body(new ErrorResponse()
101             .message(
102                 ex.getReason() != null
103                     ? ex.getReason()
104                     : ex.getBody().getTitle())
105             .code(ex.getStatusCode().value()));
106   }
107 
108   /**
109    * Ping solr.
110    *
111    * @return the response entity (ok or an error response)
112    */
113   @Operation(summary = "Ping Solr", description = "Ping Solr to check if it is available")
114   @ApiResponse(
115       responseCode = "200",
116       description = "Solr is available",
117       content =
118           @Content(
119               mediaType = MediaType.APPLICATION_JSON_VALUE,
120               schema = @Schema(example = "{\"status\":\"OK\",\"timeElapsed\":1}")))
121   @ApiResponse(
122       responseCode = "500",
123       description = "Solr is not available",
124       content =
125           @Content(
126               mediaType = MediaType.APPLICATION_JSON_VALUE,
127               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
128   @GetMapping(path = "${tailormap-api.admin.base-path}/index/ping", produces = MediaType.APPLICATION_JSON_VALUE)
129   public ResponseEntity<?> pingSolr() {
130     try (SolrClient solrClient = solrService.getBaseUrlSolrClient()) {
131       final HealthCheckRequest request = new HealthCheckRequest();
132       final HealthCheckResponse ping = request.process(solrClient);
133       logger.info("Solr ping status: {}", ping.getResponse().get("status"));
134       Metrics.timer("tailormap_solr_ping").record(ping.getElapsedTime(), TimeUnit.MILLISECONDS);
135       return ResponseEntity.ok(this.jsonMapper
136           .createObjectNode()
137           .put("status", ping.getResponse().get("status").toString())
138           .put("timeElapsed", ping.getElapsedTime()));
139     } catch (IOException | SolrServerException e) {
140       logger.error("Error pinging solr", e);
141       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
142     }
143   }
144 
145   /**
146    * (re-) Index a layer.
147    *
148    * @param searchIndexId the searchIndex id
149    * @return the response entity (accepted or an error response)
150    */
151   @Operation(
152       summary = "Create or update a feature type index",
153       description = "Create or update a feature type index for a layer, will erase existing index if present")
154   @ApiResponse(
155       responseCode = "202",
156       description = "Index create or update request accepted",
157       content =
158           @Content(
159               mediaType = MediaType.APPLICATION_JSON_VALUE,
160               schema =
161                   @Schema(
162                       example =
163                           "{\"type\":\"index\", \"uuid\":\"6308d26e-fe1e-4268-bb28-20db2cd06914\",\"code\":202}")))
164   @ApiResponse(
165       responseCode = "404",
166       description = "Layer does not have feature type",
167       content =
168           @Content(
169               mediaType = MediaType.APPLICATION_JSON_VALUE,
170               schema =
171                   @Schema(
172                       example =
173                           "{\"message\":\"Layer does not have feature type\",\"code\":404}")))
174   @ApiResponse(
175       responseCode = "400",
176       description = "Indexing WFS feature types is not supported",
177       content =
178           @Content(
179               mediaType = MediaType.APPLICATION_JSON_VALUE,
180               schema =
181                   @Schema(
182                       example =
183                           "{\"message\":\"Layer does not have valid feature type for indexing\",\"code\":400}")))
184   @ApiResponse(
185       responseCode = "500",
186       description = "Error while indexing",
187       content =
188           @Content(
189               mediaType = MediaType.APPLICATION_JSON_VALUE,
190               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
191   @Transactional
192   @Timed(value = "index_feature_type", description = "time spent to index feature type")
193   @PutMapping(
194       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
195       produces = MediaType.APPLICATION_JSON_VALUE)
196   public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
197     SearchIndex searchIndex = validateInputAndFindIndex(searchIndexId);
198 
199     if (searchIndex.getStatus() == SearchIndex.Status.INDEXING) {
200       throw new ResponseStatusException(
201           HttpStatus.CONFLICT, "Indexing already in progress, check tasks overview before retrying");
202     }
203 
204     boolean createNewIndex =
205         (null == searchIndex.getLastIndexed() || searchIndex.getStatus() == SearchIndex.Status.INITIAL);
206 
207     boolean hasSchedule = (null != searchIndex.getSchedule()
208         && null != searchIndex.getSchedule().getUuid());
209 
210     UUID taskUuid;
211     try {
212       if (hasSchedule) {
213         taskUuid = searchIndex.getSchedule().getUuid();
214         startScheduledJobIndexing(searchIndex);
215       } else {
216         taskUuid = startOneTimeJobIndexing(searchIndex);
217       }
218       searchIndexRepository.save(searchIndex);
219     } catch (UnsupportedOperationException
220         | IOException
221         | SolrServerException
222         | SolrException
223         | SchedulerException e) {
224       logger.error("Error indexing", e);
225       searchIndex.setStatus(SearchIndex.Status.ERROR);
226       searchIndexRepository.save(searchIndex);
227       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
228     }
229 
230     logger.info(
231         "Scheduled {} index for search index {}",
232         (createNewIndex ? "creation of a new" : "update of"),
233         searchIndex.getName());
234     return ResponseEntity.accepted()
235         .body(Map.of(
236             "code",
237             202,
238             Task.TYPE_KEY,
239             TaskType.INDEX.getValue(),
240             Task.UUID_KEY,
241             taskUuid,
242             "message",
243             "Indexing scheduled"));
244   }
245 
246   private UUID startOneTimeJobIndexing(SearchIndex searchIndex)
247       throws SolrServerException, IOException, SchedulerException {
248     UUID taskName = taskManagerService.createTask(
249         IndexTask.class,
250         new TMJobDataMap(Map.of(
251             Task.TYPE_KEY,
252             TaskType.INDEX,
253             Task.DESCRIPTION_KEY,
254             "One-time indexing of " + searchIndex.getName(),
255             IndexTask.INDEX_KEY,
256             searchIndex.getId().toString(),
257             Task.PRIORITY_KEY,
258             0)));
259     logger.info("One-time indexing job with UUID {} started", taskName);
260     return taskName;
261   }
262 
263   private void startScheduledJobIndexing(SearchIndex searchIndex) throws SchedulerException {
264     JobKey jobKey = taskManagerService.getJobKey(
265         TaskType.INDEX, searchIndex.getSchedule().getUuid());
266     if (null == jobKey) {
267       throw new SchedulerException("Indexing job not found in scheduler");
268     }
269     scheduler.triggerJob(jobKey);
270     logger.info(
271         "Indexing of scheduled job with UUID {} started",
272         searchIndex.getSchedule().getUuid());
273   }
274 
275   /**
276    * Validate input and find the search index.
277    *
278    * @param searchIndexId the search index id
279    * @return the search index
280    * @throws ResponseStatusException if the search index is not found or the feature type is not found
281    */
282   private SearchIndex validateInputAndFindIndex(Long searchIndexId) {
283     // check if solr is available
284     this.pingSolr();
285 
286     // check if search index exists
287     SearchIndex searchIndex = searchIndexRepository
288         .findById(searchIndexId)
289         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
290 
291     // check if feature type exists
292     TMFeatureType indexingFT = featureTypeRepository
293         .findById(searchIndex.getFeatureTypeId())
294         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Feature type not found"));
295 
296     if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) {
297       // the search index should not exist for WFS feature types, but test just in case
298       searchIndex
299           .setStatus(SearchIndex.Status.ERROR)
300           .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported"));
301       throw new ResponseStatusException(
302           HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing");
303     }
304     return searchIndex;
305   }
306 
307   /**
308    * Clear an index; does not remove the {@link SearchIndex} metadata.
309    *
310    * @param searchIndexId the searchindex id
311    * @return the response entity ({@code 204 NOCONTENT} or an error response)
312    */
313   @Operation(summary = "Clear index for a feature type", description = "Clear index for the feature type")
314   @ApiResponse(responseCode = "204", description = "Index cleared")
315   @ApiResponse(responseCode = "404", description = "Index not configured for feature type")
316   @ApiResponse(
317       responseCode = "500",
318       description = "Error while clearing index",
319       content =
320           @Content(
321               mediaType = MediaType.APPLICATION_JSON_VALUE,
322               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
323   @Timed(value = "index_delete", description = "time spent to delete an index of a feature type")
324   @DeleteMapping(
325       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
326       produces = MediaType.APPLICATION_JSON_VALUE)
327   @Transactional
328   public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
329     try (SolrClient solrClient = solrService.getSolrClientForSearching();
330         SolrHelper solrHelper = new SolrHelper(solrClient).withQueryTimeout(solrQueryTimeout)) {
331       solrHelper.clearIndexForLayer(searchIndexId);
332       // do not delete the SearchIndex metadata object
333       // searchIndexRepository.findById(searchIndexId).ifPresent(searchIndexRepository::delete);
334       SearchIndex searchIndex = searchIndexRepository
335           .findById(searchIndexId)
336           .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
337       searchIndex
338           .setLastIndexed(null)
339           .setStatus(SearchIndex.Status.INITIAL)
340           .setSummary(new SearchIndexSummary().total(0));
341       searchIndexRepository.save(searchIndex);
342     } catch (IOException | SolrServerException | NoSuchElementException e) {
343       logger.warn("Error clearing index", e);
344       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
345     }
346 
347     logger.info("Index cleared for index {}", searchIndexId);
348     return ResponseEntity.noContent().build();
349   }
350 
351   @Override
352   public void afterPropertiesSet() throws Exception {
353     if (solrService.isSolrServiceAvailable()) {
354       logger.info("Solr is available, initializing SolrAdminController.");
355       try {
356         // there should be only one Solr ping task, so we delete any existing ones
357         taskManagerService.deleteTasksByGroupName(TaskType.SOLR_PING.getValue());
358         final UUID taskUuid = taskManagerService.createTask(
359             SolrPingTask.class,
360             new TMJobDataMap(Map.of(
361                 Task.TYPE_KEY,
362                 TaskType.SOLR_PING.getValue(),
363                 Task.DESCRIPTION_KEY,
364                 "Ping Solr service for availability.",
365                 Task.PRIORITY_KEY,
366                 58)),
367             solrPingCron);
368         logger.debug("Added Solr ping task with UUID: {}", taskUuid);
369       } catch (Exception e) {
370         logger.error("Error initializing Solr ping task", e);
371       }
372     } else {
373       logger.info("Solr is not available, /search endpoint will not be functional.");
374     }
375   }
376 }