1
2
3
4
5
6 package org.tailormap.api.solr;
7
8 import static org.tailormap.api.scheduling.IndexTask.INDEX_KEY;
9
10 import jakarta.validation.constraints.NotNull;
11 import jakarta.validation.constraints.Positive;
12 import java.io.IOException;
13 import java.lang.invoke.MethodHandles;
14 import java.math.BigDecimal;
15 import java.time.Duration;
16 import java.time.Instant;
17 import java.time.OffsetDateTime;
18 import java.time.ZoneId;
19 import java.util.ArrayList;
20 import java.util.HashMap;
21 import java.util.HashSet;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Set;
25 import java.util.UUID;
26 import java.util.function.Consumer;
27 import org.apache.solr.client.solrj.SolrClient;
28 import org.apache.solr.client.solrj.SolrQuery;
29 import org.apache.solr.client.solrj.SolrResponse;
30 import org.apache.solr.client.solrj.SolrServerException;
31 import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
32 import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
33 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
34 import org.apache.solr.client.solrj.response.QueryResponse;
35 import org.apache.solr.client.solrj.response.UpdateResponse;
36 import org.apache.solr.client.solrj.response.schema.SchemaResponse;
37 import org.apache.solr.common.SolrDocumentList;
38 import org.apache.solr.common.SolrException;
39 import org.geotools.api.data.Query;
40 import org.geotools.api.data.SimpleFeatureSource;
41 import org.geotools.api.feature.simple.SimpleFeature;
42 import org.geotools.data.simple.SimpleFeatureCollection;
43 import org.geotools.data.simple.SimpleFeatureIterator;
44 import org.locationtech.jts.geom.Geometry;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47 import org.springframework.lang.NonNull;
48 import org.springframework.lang.Nullable;
49 import org.tailormap.api.admin.model.SearchIndexSummary;
50 import org.tailormap.api.admin.model.TaskProgressEvent;
51 import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper;
52 import org.tailormap.api.geotools.processing.GeometryProcessor;
53 import org.tailormap.api.persistence.SearchIndex;
54 import org.tailormap.api.persistence.TMFeatureType;
55 import org.tailormap.api.repository.SearchIndexRepository;
56 import org.tailormap.api.scheduling.TaskType;
57 import org.tailormap.api.util.Constants;
58 import org.tailormap.api.viewer.model.SearchDocument;
59 import org.tailormap.api.viewer.model.SearchResponse;
60
61
62
63
64
65
66 public class SolrHelper implements AutoCloseable, Constants {
67 private static final Logger logger =
68 LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
69
70
71 private static final String SOLR_SPATIAL_FIELDNAME = "tm_geometry_rpt";
72
73 private final SolrClient solrClient;
74
75
76 private final Map<String, SchemaRequest.AddField> solrSearchFields = Map.of(
77 SEARCH_LAYER,
78 new SchemaRequest.AddField(Map.of(
79 "name", SEARCH_LAYER,
80 "type", "string",
81 "indexed", true,
82 "stored", true,
83 "multiValued", false,
84 "required", true,
85 "uninvertible", false)),
86 INDEX_GEOM_FIELD,
87 new SchemaRequest.AddField(
88 Map.of("name", INDEX_GEOM_FIELD, "type", SOLR_SPATIAL_FIELDNAME, "stored", true)),
89 INDEX_SEARCH_FIELD,
90 new SchemaRequest.AddField(Map.of(
91 "name", INDEX_SEARCH_FIELD,
92 "type", "text_general",
93 "indexed", true,
94 "stored", true,
95 "multiValued", true,
96 "required", true,
97 "uninvertible", false)),
98 INDEX_DISPLAY_FIELD,
99 new SchemaRequest.AddField(Map.of(
100 "name", INDEX_DISPLAY_FIELD,
101 "type", "text_general",
102 "indexed", false,
103 "stored", true,
104 "multiValued", true,
105 "required", true,
106 "uninvertible", false)));
107
108 private int solrQueryTimeout = 7000;
109 private int solrBatchSize = 1000;
110 private String solrGeometryValidationRule = "repairBuffer0";
111
112
113
114
115
116
117 public SolrHelper(@NotNull SolrClient solrClient) {
118 this.solrClient = solrClient;
119 }
120
121
122
123
124
125
126 public SolrHelper withQueryTimeout(
127 @Positive(message = "Must use a positive integer for query timeout") int solrQueryTimeout) {
128 this.solrQueryTimeout = solrQueryTimeout * 1000;
129 return this;
130 }
131
132
133
134
135
136
137 public SolrHelper withBatchSize(@Positive(message = "Must use a positive integer for batching") int solrBatchSize) {
138 this.solrBatchSize = solrBatchSize;
139 return this;
140 }
141
142
143
144
145
146
147
148
149 public SolrHelper withGeometryValidationRule(@NonNull String solrGeometryValidationRule) {
150 if (List.of("error", "none", "repairBuffer0", "repairConvexHull").contains(solrGeometryValidationRule)) {
151 logger.trace("Setting geometry validation rule for Solr geometry field to {}", solrGeometryValidationRule);
152 this.solrGeometryValidationRule = solrGeometryValidationRule;
153 }
154 return this;
155 }
156
157
158
159
160
161
162
163
164
165
166
167
168 @SuppressWarnings("FromTemporalAccessor")
169 public SearchIndex addFeatureTypeIndex(
170 @NotNull SearchIndex searchIndex,
171 @NotNull TMFeatureType tmFeatureType,
172 @NotNull FeatureSourceFactoryHelper featureSourceFactoryHelper,
173 @NotNull SearchIndexRepository searchIndexRepository)
174 throws IOException, SolrServerException {
175
176 Consumer<TaskProgressEvent> progressListener = (event) -> logger.debug("Progress event: {}", event);
177
178 return this.addFeatureTypeIndex(
179 searchIndex, tmFeatureType, featureSourceFactoryHelper, searchIndexRepository, progressListener, null);
180 }
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196 @SuppressWarnings("FromTemporalAccessor")
197 public SearchIndex addFeatureTypeIndex(
198 @NotNull SearchIndex searchIndex,
199 @NotNull TMFeatureType tmFeatureType,
200 @NotNull FeatureSourceFactoryHelper featureSourceFactoryHelper,
201 @NotNull SearchIndexRepository searchIndexRepository,
202 @NotNull Consumer<TaskProgressEvent> progressListener,
203 @Nullable UUID taskUuid)
204 throws IOException, SolrServerException {
205
206 createSchemaIfNotExists();
207
208 final Instant startedAt = Instant.now();
209 final OffsetDateTime startedAtOffset =
210 startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt));
211
212 if (null == taskUuid && null != searchIndex.getSchedule()) {
213
214
215
216 taskUuid = searchIndex.getSchedule().getUuid();
217 }
218
219 SearchIndexSummary summary =
220 new SearchIndexSummary().startedAt(startedAtOffset).total(0).duration(0.0);
221
222 if (null == searchIndex.getSearchFieldsUsed()) {
223 logger.warn(
224 "No search fields configured for search index: {}, skipping index {}.",
225 tmFeatureType.getName(),
226 searchIndex.getName());
227 return searchIndexRepository.save(searchIndex
228 .setStatus(SearchIndex.Status.ERROR)
229 .setSummary(summary.errorMessage("No search fields configured")));
230 }
231
232 TaskProgressEvent taskProgressEvent = new TaskProgressEvent()
233 .type(TaskType.INDEX.getValue())
234 .uuid(taskUuid)
235 .startedAt(startedAtOffset)
236 .progress(0)
237 .taskData(Map.of(INDEX_KEY, searchIndex.getId()));
238 progressListener.accept(taskProgressEvent);
239
240
241 List<String> searchFields = searchIndex.getSearchFieldsUsed().stream()
242 .filter(s -> !tmFeatureType.getSettings().getHideAttributes().contains(s))
243 .toList();
244 List<String> displayFields = searchIndex.getSearchDisplayFieldsUsed().stream()
245 .filter(s -> !tmFeatureType.getSettings().getHideAttributes().contains(s))
246 .toList();
247
248 if (searchFields.isEmpty()) {
249 logger.warn(
250 "No valid search fields configured for feature type: {}, skipping index {}.",
251 tmFeatureType.getName(),
252 searchIndex.getName());
253 return searchIndexRepository.save(searchIndex
254 .setStatus(SearchIndex.Status.ERROR)
255 .setSummary(summary.errorMessage("No search fields configured")));
256 }
257
258
259 Set<String> propertyNames = new HashSet<>();
260
261
262 if (null == tmFeatureType.getPrimaryKeyAttribute()) {
263 logger.error(
264 "No primary key attribute configured for feature type: {}, skipping index {}.",
265 tmFeatureType.getName(),
266 searchIndex.getName());
267 return searchIndexRepository.save(searchIndex
268 .setStatus(SearchIndex.Status.ERROR)
269 .setSummary(summary.errorMessage("No primary key attribute configured")));
270 }
271 propertyNames.add(tmFeatureType.getPrimaryKeyAttribute());
272 if (null == tmFeatureType.getDefaultGeometryAttribute()) {
273 logger.error(
274 "No default geometry attribute configured for feature type: {}, skipping index {}.",
275 tmFeatureType.getName(),
276 searchIndex.getName());
277 return searchIndexRepository.save(searchIndex
278 .setStatus(SearchIndex.Status.ERROR)
279 .setSummary(summary.errorMessage("No default geometry attribute configured")));
280 }
281 propertyNames.add(tmFeatureType.getDefaultGeometryAttribute());
282 propertyNames.addAll(searchFields);
283
284 if (!displayFields.isEmpty()) {
285 propertyNames.addAll(displayFields);
286 }
287
288 clearIndexForLayer(searchIndex.getId());
289
290 logger.info("Indexing started for index: {}, feature type: {}", searchIndex.getName(), tmFeatureType.getName());
291 searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXING));
292
293
294 SimpleFeatureSource fs = featureSourceFactoryHelper.openGeoToolsFeatureSource(tmFeatureType);
295 Query q = new Query(fs.getName().toString());
296
297 tmFeatureType.getSettings().getHideAttributes().forEach(propertyNames::remove);
298 if (propertyNames.isEmpty()) {
299 logger.warn(
300 "No valid properties to index for feature type: {}, skipping index {}.",
301 tmFeatureType.getName(),
302 searchIndex.getName());
303 return searchIndexRepository.save(searchIndex
304 .setStatus(SearchIndex.Status.ERROR)
305 .setSummary(
306 summary.errorMessage("No valid properties to index. Check if any properties are hidden.")));
307 }
308 q.setPropertyNames(List.copyOf(propertyNames));
309 q.setStartIndex(0);
310
311
312 logger.trace("Indexing query: {}", q);
313 SimpleFeatureCollection simpleFeatureCollection = fs.getFeatures(q);
314 final int total = simpleFeatureCollection.size();
315 List<FeatureIndexingDocument> docsBatch = new ArrayList<>(solrBatchSize);
316
317
318 UpdateResponse updateResponse;
319 int indexCounter = 0;
320 int indexSkippedCounter = 0;
321 try (SimpleFeatureIterator iterator = simpleFeatureCollection.features()) {
322 while (iterator.hasNext()) {
323 indexCounter++;
324 SimpleFeature feature = iterator.next();
325
326 FeatureIndexingDocument doc = new FeatureIndexingDocument(feature.getID(), searchIndex.getId());
327 List<String> searchValues = new ArrayList<>();
328 List<String> displayValues = new ArrayList<>();
329 propertyNames.forEach(propertyName -> {
330 Object value = feature.getAttribute(propertyName);
331 if (value != null) {
332 if (value instanceof Geometry
333 && propertyName.equals(tmFeatureType.getDefaultGeometryAttribute())) {
334
335
336 doc.setGeometry(GeometryProcessor.processGeometry(value, true, true, null));
337 } else {
338 if (searchFields.contains(propertyName)) {
339 searchValues.add(value.toString());
340 }
341 if (displayFields.contains(propertyName)) {
342 displayValues.add(value.toString());
343 }
344 }
345 }
346 });
347 if (searchValues.isEmpty() || displayValues.isEmpty()) {
348
349 logger.trace(
350 "No search or display values found for feature: {} in feature type: {}, skipped for indexing",
351 feature.getID(),
352 tmFeatureType.getName());
353 indexSkippedCounter++;
354 } else {
355 doc.setSearchFields(searchValues.toArray(new String[0]));
356 doc.setDisplayFields(displayValues.toArray(new String[0]));
357 docsBatch.add(doc);
358 }
359
360 if (indexCounter % solrBatchSize == 0) {
361 updateResponse = solrClient.addBeans(docsBatch, solrQueryTimeout);
362 logger.info(
363 "Added {} documents of {} to index, result status: {}",
364 indexCounter - indexSkippedCounter,
365 total,
366 updateResponse.getStatus());
367 progressListener.accept(
368 taskProgressEvent.total(total).progress((indexCounter - indexSkippedCounter)));
369 docsBatch.clear();
370 }
371 }
372 } finally {
373 if (fs.getDataStore() != null) fs.getDataStore().dispose();
374 }
375
376 if (!docsBatch.isEmpty()) {
377 solrClient.addBeans(docsBatch, solrQueryTimeout);
378 logger.info("Added last {} documents of {} to index", docsBatch.size(), total);
379 progressListener.accept(taskProgressEvent
380 .progress((indexCounter - indexSkippedCounter))
381 .total(total));
382 }
383 final Instant finishedAt = Instant.now();
384 final OffsetDateTime finishedAtOffset =
385 finishedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(finishedAt));
386 Duration processTime = Duration.between(startedAt, finishedAt).abs();
387 logger.info(
388 "Indexing finished for index: {}, feature type: {} at {} in {}",
389 searchIndex.getName(),
390 tmFeatureType.getName(),
391 finishedAtOffset,
392 processTime);
393 updateResponse = this.solrClient.commit();
394 logger.trace("Update response commit status: {}", updateResponse.getStatus());
395
396 if (indexSkippedCounter > 0) {
397 logger.warn(
398 "{} features were skipped because no search or display values were found.", indexSkippedCounter);
399 }
400
401 return searchIndexRepository.save(searchIndex
402 .setLastIndexed(finishedAtOffset)
403 .setStatus(SearchIndex.Status.INDEXED)
404 .setSummary(summary.total(total)
405 .skippedCounter(indexSkippedCounter)
406 .duration(BigDecimal.valueOf(processTime.getSeconds())
407 .add(BigDecimal.valueOf(processTime.getNano(), 9))
408 .doubleValue())
409 .errorMessage(null)));
410 }
411
412
413
414
415
416
417
418
419 public void clearIndexForLayer(@NotNull Long searchLayerId) throws IOException, SolrServerException {
420
421 QueryResponse response =
422 solrClient.query(new SolrQuery("exists(query(" + SEARCH_LAYER + ":" + searchLayerId + "))"));
423 if (response.getResults().getNumFound() > 0) {
424 logger.info("Clearing index for searchLayer {}", searchLayerId);
425 UpdateResponse updateResponse = solrClient.deleteByQuery(SEARCH_LAYER + ":" + searchLayerId);
426 logger.trace("Delete response status: {}", updateResponse.getStatus());
427 updateResponse = solrClient.commit();
428 logger.trace("Commit response status: {}", updateResponse.getStatus());
429 } else {
430 logger.info("No index to clear for layer {}", searchLayerId);
431 }
432 }
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447 public SearchResponse findInIndex(
448 @NotNull SearchIndex searchIndex,
449 String solrQuery,
450 String solrFilterQuery,
451 String solrPoint,
452 Double solrDistance,
453 int start,
454 int numResultsToReturn)
455 throws IOException, SolrServerException, SolrException {
456
457 if (null == solrQuery || solrQuery.isBlank()) {
458 solrQuery = "*";
459 }
460
461 logger.info("Query index for '{}' in {} (id {})", solrQuery, searchIndex.getName(), searchIndex.getId());
462
463
464
465
466
467 final SolrQuery query = new SolrQuery(INDEX_SEARCH_FIELD + ":" + solrQuery)
468 .setShowDebugInfo(logger.isDebugEnabled())
469 .setTimeAllowed(solrQueryTimeout)
470 .setIncludeScore(true)
471 .setFields(SEARCH_ID_FIELD, INDEX_DISPLAY_FIELD, INDEX_GEOM_FIELD)
472 .addFilterQuery(SEARCH_LAYER + ":" + searchIndex.getId())
473 .setSort("score", SolrQuery.ORDER.desc)
474 .addSort(SEARCH_ID_FIELD, SolrQuery.ORDER.asc)
475 .setRows(numResultsToReturn)
476 .setStart(start);
477
478 if (null != solrFilterQuery && !solrFilterQuery.isBlank()) {
479 query.addFilterQuery(solrFilterQuery);
480 }
481 if (null != solrPoint && null != solrDistance) {
482 if (null == solrFilterQuery
483 || !(solrFilterQuery.startsWith("{!geofilt") || solrFilterQuery.startsWith("{!bbox"))) {
484 query.addFilterQuery("{!geofilt sfield=" + INDEX_GEOM_FIELD + "}");
485 }
486 query.add("pt", solrPoint);
487 query.add("d", solrDistance.toString());
488 }
489 query.set("q.op", "AND");
490 logger.info("Solr query: {}", query);
491
492 final QueryResponse response = solrClient.query(query);
493 logger.trace("response: {}", response);
494
495 final SolrDocumentList solrDocumentList = response.getResults();
496 logger.debug("Found {} solr documents", solrDocumentList.getNumFound());
497 final SearchResponse searchResponse = new SearchResponse()
498 .total(solrDocumentList.getNumFound())
499 .start(response.getResults().getStart())
500 .maxScore(solrDocumentList.getMaxScore());
501 response.getResults().forEach(solrDocument -> {
502 List<String> displayValues = solrDocument.getFieldValues(INDEX_DISPLAY_FIELD).stream()
503 .map(Object::toString)
504 .toList();
505 searchResponse.addDocumentsItem(new SearchDocument()
506 .fid(solrDocument.getFieldValue(SEARCH_ID_FIELD).toString())
507 .geometry(solrDocument.getFieldValue(INDEX_GEOM_FIELD).toString())
508 .displayValues(displayValues));
509 });
510
511 return searchResponse;
512 }
513
514
515
516
517
518
519 @Override
520 public void close() throws IOException {
521 if (null != this.solrClient) this.solrClient.close();
522 }
523
524 private boolean checkSchemaIfFieldExists(String fieldName) {
525 SchemaRequest.Field fieldCheck = new SchemaRequest.Field(fieldName);
526 try {
527 SchemaResponse.FieldResponse isField = fieldCheck.process(solrClient);
528 logger.debug("Field {} exists", isField.getField());
529 return true;
530 } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException e) {
531 logger.debug("Field {} does not exist or could not be retrieved. Assuming it does not exist.", fieldName);
532 } catch (IOException e) {
533 logger.error("Tried getting field: {}, but failed.", fieldName, e);
534 }
535 return false;
536 }
537
538
539
540
541
542
543 private void createSchemaFieldIfNotExists(String fieldName) throws SolrServerException, IOException {
544 if (!checkSchemaIfFieldExists(fieldName)) {
545 logger.info("Creating Solr field {}.", fieldName);
546 SchemaRequest.AddField schemaRequest = solrSearchFields.get(fieldName);
547 SolrResponse response = schemaRequest.process(solrClient);
548 logger.debug("Field type {} created", response);
549 solrClient.commit();
550 }
551 }
552
553
554 private void createSchemaIfNotExists() {
555 solrSearchFields.forEach((key, value) -> {
556 try {
557 if (key.equals(INDEX_GEOM_FIELD)) {
558 createGeometryFieldTypeIfNotExists();
559 }
560 createSchemaFieldIfNotExists(key);
561 } catch (SolrServerException | IOException e) {
562 logger.error(
563 "Error creating schema field: {} indexing may fail. Details: {}",
564 key,
565 e.getLocalizedMessage(),
566 e);
567 }
568 });
569 }
570
571 private void createGeometryFieldTypeIfNotExists() throws SolrServerException, IOException {
572 SchemaRequest.FieldType fieldTypeCheck = new SchemaRequest.FieldType(SOLR_SPATIAL_FIELDNAME);
573 try {
574 SchemaResponse.FieldTypeResponse isFieldType = fieldTypeCheck.process(solrClient);
575 logger.debug("Field type {} exists", isFieldType.getFieldType());
576 return;
577 } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException e) {
578 logger.debug(
579 "Field type {} does not exist or could not be retrieved. Assuming it does not exist.",
580 SOLR_SPATIAL_FIELDNAME);
581 } catch (IOException e) {
582 logger.error("Tried getting field type: {}, but failed.", SOLR_SPATIAL_FIELDNAME, e);
583 }
584
585 logger.info(
586 "Creating Solr field type for {} with validation rule {}",
587 SOLR_SPATIAL_FIELDNAME,
588 solrGeometryValidationRule);
589 FieldTypeDefinition spatialFieldTypeDef = new FieldTypeDefinition();
590 Map<String, Object> spatialFieldAttributes = new HashMap<>(Map.of(
591 "name", SOLR_SPATIAL_FIELDNAME,
592 "class", "solr.SpatialRecursivePrefixTreeFieldType",
593 "spatialContextFactory", "JTS",
594 "geo", false,
595 "distanceUnits", "kilometers",
596 "distCalculator", "cartesian",
597 "format", "WKT",
598 "autoIndex", true,
599 "distErrPct", "0.025",
600 "maxDistErr", "0.001"));
601 spatialFieldAttributes.putAll(Map.of(
602 "prefixTree",
603 "packedQuad",
604
605
606 "validationRule",
607 this.solrGeometryValidationRule,
608
609
610 "worldBounds",
611
612 "ENVELOPE(-20037508.34, 20037508.34, 20048966.1, -20048966.1)"
613
614
615 ));
616 spatialFieldTypeDef.setAttributes(spatialFieldAttributes);
617 SchemaRequest.AddFieldType spatialFieldType = new SchemaRequest.AddFieldType(spatialFieldTypeDef);
618 spatialFieldType.process(solrClient);
619 solrClient.commit();
620 }
621 }