METRON-1503 Alerts are not getting populated in alerts UI when search engine is Solr...
[metron.git] / metron-platform / metron-solr / src / main / java / org / apache / metron / solr / dao / SolrColumnMetadataDao.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.metron.solr.dao;
19
20 import com.google.common.collect.Sets;
21 import java.io.IOException;
22 import java.lang.invoke.MethodHandles;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Set;
28 import org.apache.metron.indexing.dao.ColumnMetadataDao;
29 import org.apache.metron.indexing.dao.search.FieldType;
30 import org.apache.solr.client.solrj.SolrServerException;
31 import org.apache.solr.client.solrj.impl.CloudSolrClient;
32 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
33 import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
34 import org.apache.solr.common.SolrException;
35 import org.slf4j.Logger;
36 import org.slf4j.LoggerFactory;
37
38 public class SolrColumnMetadataDao implements ColumnMetadataDao {
39
40 private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
41
42 private static Map<String, FieldType> solrTypeMap;
43
44 static {
45 Map<String, FieldType> fieldTypeMap = new HashMap<>();
46 fieldTypeMap.put("string", FieldType.TEXT);
47 fieldTypeMap.put("pint", FieldType.INTEGER);
48 fieldTypeMap.put("plong", FieldType.LONG);
49 fieldTypeMap.put("pfloat", FieldType.FLOAT);
50 fieldTypeMap.put("pdouble", FieldType.DOUBLE);
51 fieldTypeMap.put("boolean", FieldType.BOOLEAN);
52 solrTypeMap = Collections.unmodifiableMap(fieldTypeMap);
53 }
54
55 private String zkHost;
56
57 public SolrColumnMetadataDao(String zkHost) {
58 this.zkHost = zkHost;
59 }
60
61 @Override
62 public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
63 Map<String, FieldType> indexColumnMetadata = new HashMap<>();
64 Map<String, String> previousIndices = new HashMap<>();
65 Set<String> fieldBlackList = Sets.newHashSet(SolrDao.ROOT_FIELD, SolrDao.VERSION_FIELD);
66
67 for (String index : indices) {
68 try {
69 getIndexFields(index).forEach(field -> {
70 String name = (String) field.get("name");
71 if (!fieldBlackList.contains(name)) {
72 FieldType type = toFieldType((String) field.get("type"));
73 if (!indexColumnMetadata.containsKey(name)) {
74 indexColumnMetadata.put(name, type);
75
76 // record the last index in which a field exists, to be able to print helpful error message on type mismatch
77 previousIndices.put(name, index);
78 } else {
79 FieldType previousType = indexColumnMetadata.get(name);
80 if (!type.equals(previousType)) {
81 String previousIndexName = previousIndices.get(name);
82 LOG.error(String.format(
83 "Field type mismatch: %s.%s has type %s while %s.%s has type %s. Defaulting type to %s.",
84 index, field, type.getFieldType(),
85 previousIndexName, field, previousType.getFieldType(),
86 FieldType.OTHER.getFieldType()));
87 indexColumnMetadata.put(name, FieldType.OTHER);
88
89 // the field is defined in multiple indices with different types; ignore the field as type has been set to OTHER
90 fieldBlackList.add(name);
91 }
92 }
93 }
94 });
95 } catch (SolrServerException e) {
96 throw new IOException(e);
97 } catch (SolrException e) {
98 // 400 means an index is missing so continue
99 if (e.code() != 400) {
100 throw new IOException(e);
101 }
102 }
103 }
104 return indexColumnMetadata;
105 }
106
107 protected List<Map<String, Object>> getIndexFields(String index) throws IOException, SolrServerException {
108 CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build();
109 client.setDefaultCollection(index);
110 SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client)
111 .getSchemaRepresentation();
112 return schemaRepresentation.getFields();
113 }
114
115 /**
116 * Converts a string type to the corresponding FieldType.
117 *
118 * @param type The type to convert.
119 * @return The corresponding FieldType or FieldType.OTHER, if no match.
120 */
121 private FieldType toFieldType(String type) {
122 return solrTypeMap.getOrDefault(type, FieldType.OTHER);
123 }
124 }