View Javadoc
1   /*
2    * Copyright (C) 2024 B3Partners B.V.
3    *
4    * SPDX-License-Identifier: MIT
5    */
6   package org.tailormap.api.controller.admin;
7   
8   import io.micrometer.core.annotation.Timed;
9   import io.micrometer.core.instrument.Metrics;
10  import io.swagger.v3.oas.annotations.Operation;
11  import io.swagger.v3.oas.annotations.media.Content;
12  import io.swagger.v3.oas.annotations.media.Schema;
13  import io.swagger.v3.oas.annotations.responses.ApiResponse;
14  import java.io.IOException;
15  import java.lang.invoke.MethodHandles;
16  import java.util.Map;
17  import java.util.NoSuchElementException;
18  import java.util.UUID;
19  import java.util.concurrent.TimeUnit;
20  import org.apache.solr.client.solrj.SolrClient;
21  import org.apache.solr.client.solrj.SolrServerException;
22  import org.apache.solr.client.solrj.response.SolrPingResponse;
23  import org.apache.solr.common.SolrException;
24  import org.quartz.JobKey;
25  import org.quartz.Scheduler;
26  import org.quartz.SchedulerException;
27  import org.slf4j.Logger;
28  import org.slf4j.LoggerFactory;
29  import org.springframework.beans.factory.InitializingBean;
30  import org.springframework.beans.factory.annotation.Autowired;
31  import org.springframework.beans.factory.annotation.Value;
32  import org.springframework.http.HttpStatus;
33  import org.springframework.http.MediaType;
34  import org.springframework.http.ResponseEntity;
35  import org.springframework.transaction.annotation.Transactional;
36  import org.springframework.web.bind.annotation.DeleteMapping;
37  import org.springframework.web.bind.annotation.ExceptionHandler;
38  import org.springframework.web.bind.annotation.GetMapping;
39  import org.springframework.web.bind.annotation.PathVariable;
40  import org.springframework.web.bind.annotation.PutMapping;
41  import org.springframework.web.bind.annotation.RestController;
42  import org.springframework.web.server.ResponseStatusException;
43  import org.tailormap.api.admin.model.SearchIndexSummary;
44  import org.tailormap.api.persistence.SearchIndex;
45  import org.tailormap.api.persistence.TMFeatureSource;
46  import org.tailormap.api.persistence.TMFeatureType;
47  import org.tailormap.api.repository.FeatureTypeRepository;
48  import org.tailormap.api.repository.SearchIndexRepository;
49  import org.tailormap.api.scheduling.IndexTask;
50  import org.tailormap.api.scheduling.SolrPingTask;
51  import org.tailormap.api.scheduling.TMJobDataMap;
52  import org.tailormap.api.scheduling.Task;
53  import org.tailormap.api.scheduling.TaskManagerService;
54  import org.tailormap.api.scheduling.TaskType;
55  import org.tailormap.api.solr.SolrHelper;
56  import org.tailormap.api.solr.SolrService;
57  import org.tailormap.api.viewer.model.ErrorResponse;
58  import tools.jackson.databind.json.JsonMapper;
59  
60  /** Admin controller for Solr. */
61  @RestController
62  public class SolrAdminController implements InitializingBean {
63    private static final Logger logger =
64        LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
65  
66    private final FeatureTypeRepository featureTypeRepository;
67    private final SearchIndexRepository searchIndexRepository;
68    private final SolrService solrService;
69    private final Scheduler scheduler;
70    private final TaskManagerService taskManagerService;
71    private final JsonMapper jsonMapper;
72  
73    @Value("${tailormap-api.solr-query-timeout-seconds:7}")
74    private int solrQueryTimeout;
75  
76    @Value("${tailormap-api.solr-api-ping-cron:0 0/5 * 1/1 * ? *}")
77    private String solrPingCron;
78  
79    public SolrAdminController(
80        @Autowired FeatureTypeRepository featureTypeRepository,
81        @Autowired SearchIndexRepository searchIndexRepository,
82        @Autowired SolrService solrService,
83        @Autowired Scheduler scheduler,
84        @Autowired TaskManagerService taskManagerService,
85        JsonMapper jsonMapper) {
86      this.featureTypeRepository = featureTypeRepository;
87      this.searchIndexRepository = searchIndexRepository;
88      this.solrService = solrService;
89      this.scheduler = scheduler;
90      this.taskManagerService = taskManagerService;
91      this.jsonMapper = jsonMapper;
92    }
93  
94    @ExceptionHandler({ResponseStatusException.class})
95    public ResponseEntity<?> handleException(ResponseStatusException ex) {
96      // wrap the exception in a proper json response
97      return ResponseEntity.status(ex.getStatusCode())
98          .contentType(MediaType.APPLICATION_JSON)
99          .body(new ErrorResponse()
100             .message(
101                 ex.getReason() != null
102                     ? ex.getReason()
103                     : ex.getBody().getTitle())
104             .code(ex.getStatusCode().value()));
105   }
106 
107   /**
108    * Ping solr.
109    *
110    * @return the response entity (ok or an error response)
111    */
112   @Operation(summary = "Ping Solr", description = "Ping Solr to check if it is available")
113   @ApiResponse(
114       responseCode = "200",
115       description = "Solr is available",
116       content =
117           @Content(
118               mediaType = MediaType.APPLICATION_JSON_VALUE,
119               schema = @Schema(example = "{\"status\":\"OK\",\"timeElapsed\":1}")))
120   @ApiResponse(
121       responseCode = "500",
122       description = "Solr is not available",
123       content =
124           @Content(
125               mediaType = MediaType.APPLICATION_JSON_VALUE,
126               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
127   @GetMapping(path = "${tailormap-api.admin.base-path}/index/ping", produces = MediaType.APPLICATION_JSON_VALUE)
128   public ResponseEntity<?> pingSolr() {
129     try (SolrClient solrClient = solrService.getSolrClientForSearching()) {
130       final SolrPingResponse ping = solrClient.ping();
131       logger.info("Solr ping status {}", ping.getResponse().get("status"));
132       Metrics.timer("tailormap_solr_ping").record(ping.getElapsedTime(), TimeUnit.MILLISECONDS);
133       return ResponseEntity.ok(this.jsonMapper
134           .createObjectNode()
135           .put("status", ping.getResponse().get("status").toString())
136           .put("timeElapsed", ping.getElapsedTime()));
137     } catch (IOException | SolrServerException e) {
138       logger.error("Error pinging solr", e);
139       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
140     }
141   }
142 
143   /**
144    * (re-) Index a layer.
145    *
146    * @param searchIndexId the searchIndex id
147    * @return the response entity (accepted or an error response)
148    */
149   @Operation(
150       summary = "Create or update a feature type index",
151       description = "Create or update a feature type index for a layer, will erase existing index if present")
152   @ApiResponse(
153       responseCode = "202",
154       description = "Index create or update request accepted",
155       content =
156           @Content(
157               mediaType = MediaType.APPLICATION_JSON_VALUE,
158               schema =
159                   @Schema(
160                       example =
161                           "{\"type\":\"index\", \"uuid\":\"6308d26e-fe1e-4268-bb28-20db2cd06914\",\"code\":202}")))
162   @ApiResponse(
163       responseCode = "404",
164       description = "Layer does not have feature type",
165       content =
166           @Content(
167               mediaType = MediaType.APPLICATION_JSON_VALUE,
168               schema =
169                   @Schema(
170                       example =
171                           "{\"message\":\"Layer does not have feature type\",\"code\":404}")))
172   @ApiResponse(
173       responseCode = "400",
174       description = "Indexing WFS feature types is not supported",
175       content =
176           @Content(
177               mediaType = MediaType.APPLICATION_JSON_VALUE,
178               schema =
179                   @Schema(
180                       example =
181                           "{\"message\":\"Layer does not have valid feature type for indexing\",\"code\":400}")))
182   @ApiResponse(
183       responseCode = "500",
184       description = "Error while indexing",
185       content =
186           @Content(
187               mediaType = MediaType.APPLICATION_JSON_VALUE,
188               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
189   @Transactional
190   @Timed(value = "index_feature_type", description = "time spent to index feature type")
191   @PutMapping(
192       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
193       produces = MediaType.APPLICATION_JSON_VALUE)
194   public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
195     SearchIndex searchIndex = validateInputAndFindIndex(searchIndexId);
196 
197     if (searchIndex.getStatus() == SearchIndex.Status.INDEXING) {
198       throw new ResponseStatusException(
199           HttpStatus.CONFLICT, "Indexing already in progress, check tasks overview before retrying");
200     }
201 
202     boolean createNewIndex =
203         (null == searchIndex.getLastIndexed() || searchIndex.getStatus() == SearchIndex.Status.INITIAL);
204 
205     boolean hasSchedule = (null != searchIndex.getSchedule()
206         && null != searchIndex.getSchedule().getUuid());
207 
208     UUID taskUuid;
209     try {
210       if (hasSchedule) {
211         taskUuid = searchIndex.getSchedule().getUuid();
212         startScheduledJobIndexing(searchIndex);
213       } else {
214         taskUuid = startOneTimeJobIndexing(searchIndex);
215       }
216       searchIndexRepository.save(searchIndex);
217     } catch (UnsupportedOperationException
218         | IOException
219         | SolrServerException
220         | SolrException
221         | SchedulerException e) {
222       logger.error("Error indexing", e);
223       searchIndex.setStatus(SearchIndex.Status.ERROR);
224       searchIndexRepository.save(searchIndex);
225       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
226     }
227 
228     logger.info(
229         "Scheduled {} index for search index {}",
230         (createNewIndex ? "creation of a new" : "update of"),
231         searchIndex.getName());
232     return ResponseEntity.accepted()
233         .body(Map.of(
234             "code",
235             202,
236             Task.TYPE_KEY,
237             TaskType.INDEX.getValue(),
238             Task.UUID_KEY,
239             taskUuid,
240             "message",
241             "Indexing scheduled"));
242   }
243 
244   private UUID startOneTimeJobIndexing(SearchIndex searchIndex)
245       throws SolrServerException, IOException, SchedulerException {
246     UUID taskName = taskManagerService.createTask(
247         IndexTask.class,
248         new TMJobDataMap(Map.of(
249             Task.TYPE_KEY,
250             TaskType.INDEX,
251             Task.DESCRIPTION_KEY,
252             "One-time indexing of " + searchIndex.getName(),
253             IndexTask.INDEX_KEY,
254             searchIndex.getId().toString(),
255             Task.PRIORITY_KEY,
256             0)));
257     logger.info("One-time indexing job with UUID {} started", taskName);
258     return taskName;
259   }
260 
261   private void startScheduledJobIndexing(SearchIndex searchIndex) throws SchedulerException {
262     JobKey jobKey = taskManagerService.getJobKey(
263         TaskType.INDEX, searchIndex.getSchedule().getUuid());
264     if (null == jobKey) {
265       throw new SchedulerException("Indexing job not found in scheduler");
266     }
267     scheduler.triggerJob(jobKey);
268     logger.info(
269         "Indexing of scheduled job with UUID {} started",
270         searchIndex.getSchedule().getUuid());
271   }
272 
273   /**
274    * Validate input and find the search index.
275    *
276    * @param searchIndexId the search index id
277    * @return the search index
278    * @throws ResponseStatusException if the search index is not found or the feature type is not found
279    */
280   private SearchIndex validateInputAndFindIndex(Long searchIndexId) {
281     // check if solr is available
282     this.pingSolr();
283 
284     // check if search index exists
285     SearchIndex searchIndex = searchIndexRepository
286         .findById(searchIndexId)
287         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
288 
289     // check if feature type exists
290     TMFeatureType indexingFT = featureTypeRepository
291         .findById(searchIndex.getFeatureTypeId())
292         .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Feature type not found"));
293 
294     if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) {
295       // the search index should not exist for WFS feature types, but test just in case
296       searchIndex
297           .setStatus(SearchIndex.Status.ERROR)
298           .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported"));
299       throw new ResponseStatusException(
300           HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing");
301     }
302     return searchIndex;
303   }
304 
305   /**
306    * Clear an index; does not remove the {@link SearchIndex} metadata.
307    *
308    * @param searchIndexId the searchindex id
309    * @return the response entity ({@code 204 NOCONTENT} or an error response)
310    */
311   @Operation(summary = "Clear index for a feature type", description = "Clear index for the feature type")
312   @ApiResponse(responseCode = "204", description = "Index cleared")
313   @ApiResponse(responseCode = "404", description = "Index not configured for feature type")
314   @ApiResponse(
315       responseCode = "500",
316       description = "Error while clearing index",
317       content =
318           @Content(
319               mediaType = MediaType.APPLICATION_JSON_VALUE,
320               schema = @Schema(example = "{\"message\":\"Some error message..\",\"code\":500}")))
321   @Timed(value = "index_delete", description = "time spent to delete an index of a feature type")
322   @DeleteMapping(
323       path = "${tailormap-api.admin.base-path}/index/{searchIndexId}",
324       produces = MediaType.APPLICATION_JSON_VALUE)
325   @Transactional
326   public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
327     try (SolrClient solrClient = solrService.getSolrClientForSearching();
328         SolrHelper solrHelper = new SolrHelper(solrClient).withQueryTimeout(solrQueryTimeout)) {
329       solrHelper.clearIndexForLayer(searchIndexId);
330       // do not delete the SearchIndex metadata object
331       // searchIndexRepository.findById(searchIndexId).ifPresent(searchIndexRepository::delete);
332       SearchIndex searchIndex = searchIndexRepository
333           .findById(searchIndexId)
334           .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Search index not found"));
335       searchIndex
336           .setLastIndexed(null)
337           .setStatus(SearchIndex.Status.INITIAL)
338           .setSummary(new SearchIndexSummary().total(0));
339       searchIndexRepository.save(searchIndex);
340     } catch (IOException | SolrServerException | NoSuchElementException e) {
341       logger.warn("Error clearing index", e);
342       throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
343     }
344 
345     logger.info("Index cleared for index {}", searchIndexId);
346     return ResponseEntity.noContent().build();
347   }
348 
349   @Override
350   public void afterPropertiesSet() throws Exception {
351     if (solrService.isSolrServiceAvailable()) {
352       logger.info("Solr is available, initializing SolrAdminController.");
353       try {
354         // there should be only one Solr ping task, so we delete any existing ones
355         taskManagerService.deleteTasksByGroupName(TaskType.SOLR_PING.getValue());
356         final UUID taskUuid = taskManagerService.createTask(
357             SolrPingTask.class,
358             new TMJobDataMap(Map.of(
359                 Task.TYPE_KEY,
360                 TaskType.SOLR_PING.getValue(),
361                 Task.DESCRIPTION_KEY,
362                 "Ping Solr service for availability.",
363                 Task.PRIORITY_KEY,
364                 58)),
365             solrPingCron);
366         logger.debug("Added Solr ping task with UUID: {}", taskUuid);
367       } catch (Exception e) {
368         logger.error("Error initializing Solr ping task", e);
369       }
370     } else {
371       logger.info("Solr is not available, /search endpoint will not be functional.");
372     }
373   }
374 }