1
2
3
4
5
6 package org.tailormap.api.solr;
7
8 import jakarta.validation.constraints.NotNull;
9 import jakarta.validation.constraints.Positive;
10 import java.io.IOException;
11 import java.lang.invoke.MethodHandles;
12 import java.time.Duration;
13 import java.time.Instant;
14 import java.time.OffsetDateTime;
15 import java.time.ZoneId;
16 import java.util.ArrayList;
17 import java.util.HashMap;
18 import java.util.HashSet;
19 import java.util.List;
20 import java.util.Map;
21 import java.util.Set;
22 import java.util.UUID;
23 import java.util.function.Consumer;
24 import org.apache.solr.client.solrj.SolrClient;
25 import org.apache.solr.client.solrj.SolrQuery;
26 import org.apache.solr.client.solrj.SolrResponse;
27 import org.apache.solr.client.solrj.SolrServerException;
28 import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
29 import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
30 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
31 import org.apache.solr.client.solrj.response.QueryResponse;
32 import org.apache.solr.client.solrj.response.UpdateResponse;
33 import org.apache.solr.client.solrj.response.schema.SchemaResponse;
34 import org.apache.solr.common.SolrDocumentList;
35 import org.apache.solr.common.SolrException;
36 import org.geotools.api.data.Query;
37 import org.geotools.api.data.SimpleFeatureSource;
38 import org.geotools.api.feature.simple.SimpleFeature;
39 import org.geotools.data.simple.SimpleFeatureCollection;
40 import org.geotools.data.simple.SimpleFeatureIterator;
41 import org.locationtech.jts.geom.Geometry;
42 import org.slf4j.Logger;
43 import org.slf4j.LoggerFactory;
44 import org.springframework.lang.NonNull;
45 import org.springframework.lang.Nullable;
46 import org.tailormap.api.admin.model.TaskProgressEvent;
47 import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper;
48 import org.tailormap.api.geotools.processing.GeometryProcessor;
49 import org.tailormap.api.persistence.SearchIndex;
50 import org.tailormap.api.persistence.TMFeatureType;
51 import org.tailormap.api.repository.SearchIndexRepository;
52 import org.tailormap.api.scheduling.TaskType;
53 import org.tailormap.api.util.Constants;
54 import org.tailormap.api.viewer.model.SearchDocument;
55 import org.tailormap.api.viewer.model.SearchResponse;
56
57
58
59
60
61
62
63 public class SolrHelper implements AutoCloseable, Constants {
64 private static final Logger logger =
65 LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
66
67
68 private static final String SOLR_SPATIAL_FIELDNAME = "tm_geometry_rpt";
69
70 private final SolrClient solrClient;
71
72
73 private final Map<String, SchemaRequest.AddField> solrSearchFields =
74 Map.of(
75 SEARCH_LAYER,
76 new SchemaRequest.AddField(
77 Map.of(
78 "name", SEARCH_LAYER,
79 "type", "string",
80 "indexed", true,
81 "stored", true,
82 "multiValued", false,
83 "required", true,
84 "uninvertible", false)),
85 INDEX_GEOM_FIELD,
86 new SchemaRequest.AddField(
87 Map.of("name", INDEX_GEOM_FIELD, "type", SOLR_SPATIAL_FIELDNAME, "stored", true)),
88 INDEX_SEARCH_FIELD,
89 new SchemaRequest.AddField(
90 Map.of(
91 "name", INDEX_SEARCH_FIELD,
92 "type", "text_general",
93 "indexed", true,
94 "stored", true,
95 "multiValued", true,
96 "required", true,
97 "uninvertible", false)),
98 INDEX_DISPLAY_FIELD,
99 new SchemaRequest.AddField(
100 Map.of(
101 "name", INDEX_DISPLAY_FIELD,
102 "type", "text_general",
103 "indexed", false,
104 "stored", true,
105 "multiValued", true,
106 "required", true,
107 "uninvertible", false)));
108
109 private int solrQueryTimeout = 7000;
110 private int solrBatchSize = 1000;
111 private String solrGeometryValidationRule = "repairBuffer0";
112
113
114
115
116
117
118 public SolrHelper(@NotNull SolrClient solrClient) {
119 this.solrClient = solrClient;
120 }
121
122
123
124
125
126
127 public SolrHelper withQueryTimeout(
128 @Positive(message = "Must use a positive integer for query timeout") int solrQueryTimeout) {
129 this.solrQueryTimeout = solrQueryTimeout * 1000;
130 return this;
131 }
132
133
134
135
136
137
138
139 public SolrHelper withBatchSize(
140 @Positive(message = "Must use a positive integer for batching") int solrBatchSize) {
141 this.solrBatchSize = solrBatchSize;
142 return this;
143 }
144
145
146
147
148
149
150
151
152
153
154 public SolrHelper withGeometryValidationRule(@NonNull String solrGeometryValidationRule) {
155 if (List.of("error", "none", "repairBuffer0", "repairConvexHull")
156 .contains(solrGeometryValidationRule)) {
157 logger.trace(
158 "Setting geometry validation rule for Solr geometry field to {}",
159 solrGeometryValidationRule);
160 this.solrGeometryValidationRule = solrGeometryValidationRule;
161 }
162 return this;
163 }
164
165
166
167
168
169
170
171
172
173
174
175
176
177 @SuppressWarnings("FromTemporalAccessor")
178 public SearchIndex addFeatureTypeIndex(
179 @NotNull SearchIndex searchIndex,
180 @NotNull TMFeatureType tmFeatureType,
181 @NotNull FeatureSourceFactoryHelper featureSourceFactoryHelper,
182 @NotNull SearchIndexRepository searchIndexRepository)
183 throws IOException, SolrServerException {
184
185 Consumer<TaskProgressEvent> progressListener =
186 (event) -> {
187 logger.debug("Progress event: {}", event);
188 };
189
190 return this.addFeatureTypeIndex(
191 searchIndex,
192 tmFeatureType,
193 featureSourceFactoryHelper,
194 searchIndexRepository,
195 progressListener,
196 null);
197 }
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214 @SuppressWarnings("FromTemporalAccessor")
215 public SearchIndex addFeatureTypeIndex(
216 @NotNull SearchIndex searchIndex,
217 @NotNull TMFeatureType tmFeatureType,
218 @NotNull FeatureSourceFactoryHelper featureSourceFactoryHelper,
219 @NotNull SearchIndexRepository searchIndexRepository,
220 @NotNull Consumer<TaskProgressEvent> progressListener,
221 @Nullable UUID taskUuid)
222 throws IOException, SolrServerException {
223
224 createSchemaIfNotExists();
225
226 final Instant startedAt = Instant.now();
227 final OffsetDateTime startedAtOffset =
228 startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt));
229
230 if (null == taskUuid && null != searchIndex.getSchedule()) {
231
232
233
234 taskUuid = searchIndex.getSchedule().getUuid();
235 }
236
237 if (null == searchIndex.getSearchFieldsUsed()) {
238 logger.warn(
239 "No search fields configured for search index: {}, bailing out.", searchIndex.getName());
240 return searchIndexRepository.save(
241 searchIndex
242 .setStatus(SearchIndex.Status.ERROR)
243 .setComment("No search fields configured"));
244 }
245
246 progressListener.accept(
247 new TaskProgressEvent()
248 .type(TaskType.INDEX.getValue())
249 .uuid(taskUuid)
250 .startedAt(startedAtOffset)
251 .progress(0));
252
253
254 List<String> searchFields =
255 searchIndex.getSearchFieldsUsed().stream()
256 .filter(s -> !tmFeatureType.getSettings().getHideAttributes().contains(s))
257 .toList();
258 List<String> displayFields =
259 searchIndex.getSearchDisplayFieldsUsed().stream()
260 .filter(s -> !tmFeatureType.getSettings().getHideAttributes().contains(s))
261 .toList();
262
263 if (searchFields.isEmpty()) {
264 logger.warn(
265 "No valid search fields configured for featuretype: {}, bailing out.",
266 tmFeatureType.getName());
267 return searchIndexRepository.save(
268 searchIndex
269 .setStatus(SearchIndex.Status.ERROR)
270 .setComment("No search fields configured"));
271 }
272
273
274 Set<String> propertyNames = new HashSet<>();
275
276 propertyNames.add(tmFeatureType.getPrimaryKeyAttribute());
277 propertyNames.add(tmFeatureType.getDefaultGeometryAttribute());
278 propertyNames.addAll(searchFields);
279
280 if (!displayFields.isEmpty()) {
281 propertyNames.addAll(displayFields);
282 }
283
284 clearIndexForLayer(searchIndex.getId());
285
286 logger.info(
287 "Indexing started for index id: {}, feature type: {}",
288 searchIndex.getId(),
289 tmFeatureType.getName());
290 searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXING));
291
292
293 SimpleFeatureSource fs = featureSourceFactoryHelper.openGeoToolsFeatureSource(tmFeatureType);
294 Query q = new Query(fs.getName().toString());
295
296 tmFeatureType.getSettings().getHideAttributes().forEach(propertyNames::remove);
297 q.setPropertyNames(List.copyOf(propertyNames));
298 q.setStartIndex(0);
299
300
301 logger.trace("Indexing query: {}", q);
302 SimpleFeatureCollection simpleFeatureCollection = fs.getFeatures(q);
303 final int total = simpleFeatureCollection.size();
304 List<FeatureIndexingDocument> docsBatch = new ArrayList<>(solrBatchSize);
305
306
307 UpdateResponse updateResponse;
308 int indexCounter = 0;
309 int indexSkippedCounter = 0;
310 try (SimpleFeatureIterator iterator = simpleFeatureCollection.features()) {
311 while (iterator.hasNext()) {
312 indexCounter++;
313 SimpleFeature feature = iterator.next();
314
315 FeatureIndexingDocument doc =
316 new FeatureIndexingDocument(feature.getID(), searchIndex.getId());
317 List<String> searchValues = new ArrayList<>();
318 List<String> displayValues = new ArrayList<>();
319 propertyNames.forEach(
320 propertyName -> {
321 Object value = feature.getAttribute(propertyName);
322 if (value != null) {
323 if (value instanceof Geometry
324 && propertyName.equals(tmFeatureType.getDefaultGeometryAttribute())) {
325
326
327 doc.setGeometry(GeometryProcessor.processGeometry(value, true, true, null));
328 } else {
329 if (searchFields.contains(propertyName)) {
330 searchValues.add(value.toString());
331 }
332 if (displayFields.contains(propertyName)) {
333 displayValues.add(value.toString());
334 }
335 }
336 }
337 });
338 if (searchValues.isEmpty() || displayValues.isEmpty()) {
339
340 logger.trace(
341 "No search or display values found for feature: {} in featuretype: {}, skipped for indexing",
342 feature.getID(),
343 tmFeatureType.getName());
344 indexSkippedCounter++;
345 } else {
346 doc.setSearchFields(searchValues.toArray(new String[0]));
347 doc.setDisplayFields(displayValues.toArray(new String[0]));
348 docsBatch.add(doc);
349 }
350
351 if (indexCounter % solrBatchSize == 0) {
352 updateResponse = solrClient.addBeans(docsBatch, solrQueryTimeout);
353 logger.info(
354 "Added {} documents of {} to index, result status: {}",
355 indexCounter - indexSkippedCounter,
356 total,
357 updateResponse.getStatus());
358 progressListener.accept(
359 new TaskProgressEvent()
360 .type(TaskType.INDEX.getValue())
361 .uuid(taskUuid)
362 .startedAt(startedAtOffset)
363 .progress((indexCounter - indexSkippedCounter))
364 .total(total));
365 docsBatch.clear();
366 }
367 }
368 } finally {
369 if (fs.getDataStore() != null) fs.getDataStore().dispose();
370 }
371
372 if (!docsBatch.isEmpty()) {
373 solrClient.addBeans(docsBatch, solrQueryTimeout);
374 logger.info("Added last {} documents of {} to index", docsBatch.size(), total);
375 progressListener.accept(
376 new TaskProgressEvent()
377 .type(TaskType.INDEX.getValue())
378 .uuid(taskUuid)
379 .startedAt(startedAtOffset)
380 .progress((indexCounter - indexSkippedCounter))
381 .total(total));
382 }
383 final Instant finishedAt = Instant.now();
384 final OffsetDateTime finishedAtOffset =
385 finishedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(finishedAt));
386 Duration processTime = Duration.between(startedAt, finishedAt).abs();
387 logger.info(
388 "Indexing finished for index id: {}, featuretype: {} at {} in {}",
389 searchIndex.getId(),
390 tmFeatureType.getName(),
391 finishedAtOffset,
392 processTime);
393 updateResponse = this.solrClient.commit();
394 logger.debug("Update response commit status: {}", updateResponse.getStatus());
395
396 if (indexSkippedCounter > 0) {
397 logger.warn(
398 "{} features were skipped because no search or display values were found.",
399 indexSkippedCounter);
400 searchIndex =
401 searchIndex.setComment(
402 "Indexed %s features in %s.%s seconds, started at %s. %s features were skipped because no search or display values were found."
403 .formatted(
404 total,
405 processTime.getSeconds(),
406 processTime.getNano(),
407 startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt)),
408 indexSkippedCounter));
409 } else {
410 searchIndex =
411 searchIndex.setComment(
412 "Indexed %s features in %s.%s seconds, started at %s."
413 .formatted(
414 total,
415 processTime.getSeconds(),
416 processTime.getNano(),
417 startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt))));
418 }
419
420 return searchIndexRepository.save(
421 searchIndex.setLastIndexed(finishedAtOffset).setStatus(SearchIndex.Status.INDEXED));
422 }
423
424
425
426
427
428
429
430
431 public void clearIndexForLayer(@NotNull Long searchLayerId)
432 throws IOException, SolrServerException {
433
434 QueryResponse response =
435 solrClient.query(
436 new SolrQuery("exists(query(" + SEARCH_LAYER + ":" + searchLayerId + "))"));
437 if (response.getResults().getNumFound() > 0) {
438 logger.info("Clearing index for searchLayer {}", searchLayerId);
439 UpdateResponse updateResponse = solrClient.deleteByQuery(SEARCH_LAYER + ":" + searchLayerId);
440 logger.debug("Delete response status: {}", updateResponse.getStatus());
441 updateResponse = solrClient.commit();
442 logger.debug("Commit response status: {}", updateResponse.getStatus());
443 } else {
444 logger.info("No index to clear for layer {}", searchLayerId);
445 }
446 }
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463 public SearchResponse findInIndex(
464 @NotNull SearchIndex searchIndex,
465 String solrQuery,
466 String solrFilterQuery,
467 String solrPoint,
468 Double solrDistance,
469 int start,
470 int numResultsToReturn)
471 throws IOException, SolrServerException, SolrException {
472
473 logger.info("Find in index for {}", searchIndex.getId());
474 if (null == solrQuery || solrQuery.isBlank()) {
475 solrQuery = "*";
476 }
477
478
479
480
481 final SolrQuery query =
482 new SolrQuery(INDEX_SEARCH_FIELD + ":" + solrQuery)
483 .setShowDebugInfo(logger.isDebugEnabled())
484 .setTimeAllowed(solrQueryTimeout)
485 .setIncludeScore(true)
486 .setFields(SEARCH_ID_FIELD, INDEX_DISPLAY_FIELD, INDEX_GEOM_FIELD)
487 .addFilterQuery(SEARCH_LAYER + ":" + searchIndex.getId())
488 .setSort("score", SolrQuery.ORDER.desc)
489 .addSort(SEARCH_ID_FIELD, SolrQuery.ORDER.asc)
490 .setRows(numResultsToReturn)
491 .setStart(start);
492
493 if (null != solrFilterQuery && !solrFilterQuery.isBlank()) {
494 query.addFilterQuery(solrFilterQuery);
495 }
496 if (null != solrPoint && null != solrDistance) {
497 if (null == solrFilterQuery
498 || !(solrFilterQuery.startsWith("{!geofilt") || solrFilterQuery.startsWith("{!bbox"))) {
499 query.addFilterQuery("{!geofilt sfield=" + INDEX_GEOM_FIELD + "}");
500 }
501 query.add("pt", solrPoint);
502 query.add("d", solrDistance.toString());
503 }
504 query.set("q.op", "AND");
505 logger.debug("Solr query: {}", query);
506
507 final QueryResponse response = solrClient.query(query);
508 logger.debug("response: {}", response);
509
510 final SolrDocumentList solrDocumentList = response.getResults();
511 logger.debug("Found {} solr documents", solrDocumentList.getNumFound());
512 final SearchResponse searchResponse =
513 new SearchResponse()
514 .total(solrDocumentList.getNumFound())
515 .start(response.getResults().getStart())
516 .maxScore(solrDocumentList.getMaxScore());
517 response
518 .getResults()
519 .forEach(
520 solrDocument -> {
521 List<String> displayValues =
522 solrDocument.getFieldValues(INDEX_DISPLAY_FIELD).stream()
523 .map(Object::toString)
524 .toList();
525 searchResponse.addDocumentsItem(
526 new SearchDocument()
527 .fid(solrDocument.getFieldValue(SEARCH_ID_FIELD).toString())
528 .geometry(solrDocument.getFieldValue(INDEX_GEOM_FIELD).toString())
529 .displayValues(displayValues));
530 });
531
532 return searchResponse;
533 }
534
535
536
537
538
539
540 @Override
541 public void close() throws IOException {
542 if (null != this.solrClient) this.solrClient.close();
543 }
544
545 private boolean checkSchemaIfFieldExists(String fieldName) {
546 SchemaRequest.Field fieldCheck = new SchemaRequest.Field(fieldName);
547 try {
548 SchemaResponse.FieldResponse isField = fieldCheck.process(solrClient);
549 logger.debug("Field {} exists", isField.getField());
550 return true;
551 } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException e) {
552 logger.debug(
553 "Field {} does not exist or could not be retrieved. Assuming it does not exist.",
554 fieldName);
555 } catch (IOException e) {
556 logger.error("Tried getting field: {}, but failed.", fieldName, e);
557 }
558 return false;
559 }
560
561
562
563
564
565
566 private void createSchemaFieldIfNotExists(String fieldName)
567 throws SolrServerException, IOException {
568 if (!checkSchemaIfFieldExists(fieldName)) {
569 logger.info("Creating Solr field {}.", fieldName);
570 SchemaRequest.AddField schemaRequest = solrSearchFields.get(fieldName);
571 SolrResponse response = schemaRequest.process(solrClient);
572 logger.debug("Field type {} created", response);
573 solrClient.commit();
574 }
575 }
576
577
578 private void createSchemaIfNotExists() {
579 solrSearchFields.forEach(
580 (key, value) -> {
581 try {
582 if (key.equals(INDEX_GEOM_FIELD)) {
583 createGeometryFieldTypeIfNotExists();
584 }
585 createSchemaFieldIfNotExists(key);
586 } catch (SolrServerException | IOException e) {
587 logger.error(
588 "Error creating schema field: {} indexing may fail. Details: {}",
589 key,
590 e.getLocalizedMessage(),
591 e);
592 }
593 });
594 }
595
596 private void createGeometryFieldTypeIfNotExists() throws SolrServerException, IOException {
597 SchemaRequest.FieldType fieldTypeCheck = new SchemaRequest.FieldType(SOLR_SPATIAL_FIELDNAME);
598 try {
599 SchemaResponse.FieldTypeResponse isFieldType = fieldTypeCheck.process(solrClient);
600 logger.debug("Field type {} exists", isFieldType.getFieldType());
601 return;
602 } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException e) {
603 logger.debug(
604 "Field type {} does not exist or could not be retrieved. Assuming it does not exist.",
605 SOLR_SPATIAL_FIELDNAME);
606 } catch (IOException e) {
607 logger.error("Tried getting field type: {}, but failed.", SOLR_SPATIAL_FIELDNAME, e);
608 }
609
610 logger.info(
611 "Creating Solr field type for {} with validation rule {}",
612 SOLR_SPATIAL_FIELDNAME,
613 solrGeometryValidationRule);
614 FieldTypeDefinition spatialFieldTypeDef = new FieldTypeDefinition();
615 Map<String, Object> spatialFieldAttributes =
616 new HashMap<>(
617 Map.of(
618 "name", SOLR_SPATIAL_FIELDNAME,
619 "class", "solr.SpatialRecursivePrefixTreeFieldType",
620 "spatialContextFactory", "JTS",
621 "geo", false,
622 "distanceUnits", "kilometers",
623 "distCalculator", "cartesian",
624 "format", "WKT",
625 "autoIndex", true,
626 "distErrPct", "0.025",
627 "maxDistErr", "0.001"));
628 spatialFieldAttributes.putAll(
629 Map.of(
630 "prefixTree",
631 "packedQuad",
632
633
634 "validationRule",
635 this.solrGeometryValidationRule,
636
637
638 "worldBounds",
639
640 "ENVELOPE(-20037508.34, 20037508.34, 20048966.1, -20048966.1)"
641
642
643 ));
644 spatialFieldTypeDef.setAttributes(spatialFieldAttributes);
645 SchemaRequest.AddFieldType spatialFieldType =
646 new SchemaRequest.AddFieldType(spatialFieldTypeDef);
647 spatialFieldType.process(solrClient);
648 solrClient.commit();
649 }
650 }