PHOENIX-4658 IllegalStateException: requestSeek cannot be called on ReversedKeyValueH...
[phoenix.git] / phoenix-core / src / main / java / org / apache / phoenix / util / ScanUtil.java
1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.phoenix.util;
19
20 import static org.apache.phoenix.compile.OrderByCompiler.OrderBy.FWD_ROW_KEY_ORDER_BY;
21 import static org.apache.phoenix.compile.OrderByCompiler.OrderBy.REV_ROW_KEY_ORDER_BY;
22 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.CUSTOM_ANNOTATIONS;
23 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_ACTUAL_START_ROW;
24 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_START_ROW_SUFFIX;
25 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STOP_ROW_SUFFIX;
26
27 import java.io.IOException;
28 import java.sql.SQLException;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collections;
32 import java.util.Iterator;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.NavigableSet;
36 import java.util.TreeMap;
37
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.client.Mutation;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.filter.Filter;
43 import org.apache.hadoop.hbase.filter.FilterList;
44 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
45 import org.apache.hadoop.hbase.io.TimeRange;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Pair;
48 import org.apache.hadoop.io.WritableComparator;
49 import org.apache.phoenix.compile.OrderByCompiler.OrderBy;
50 import org.apache.phoenix.compile.ScanRanges;
51 import org.apache.phoenix.compile.StatementContext;
52 import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
53 import org.apache.phoenix.coprocessor.MetaDataProtocol;
54 import org.apache.phoenix.exception.SQLExceptionCode;
55 import org.apache.phoenix.exception.SQLExceptionInfo;
56 import org.apache.phoenix.execute.DescVarLengthFastByteComparisons;
57 import org.apache.phoenix.filter.BooleanExpressionFilter;
58 import org.apache.phoenix.filter.DistinctPrefixFilter;
59 import org.apache.phoenix.filter.MultiEncodedCQKeyValueComparisonFilter;
60 import org.apache.phoenix.filter.SkipScanFilter;
61 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
62 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
63 import org.apache.phoenix.query.KeyRange;
64 import org.apache.phoenix.query.KeyRange.Bound;
65 import org.apache.phoenix.query.QueryConstants;
66 import org.apache.phoenix.query.QueryServices;
67 import org.apache.phoenix.query.QueryServicesOptions;
68 import org.apache.phoenix.schema.IllegalDataException;
69 import org.apache.phoenix.schema.PColumn;
70 import org.apache.phoenix.schema.PName;
71 import org.apache.phoenix.schema.PTable;
72 import org.apache.phoenix.schema.PTable.IndexType;
73 import org.apache.phoenix.schema.RowKeySchema;
74 import org.apache.phoenix.schema.SortOrder;
75 import org.apache.phoenix.schema.ValueSchema.Field;
76 import org.apache.phoenix.schema.types.PDataType;
77 import org.apache.phoenix.schema.types.PVarbinary;
78
79 import com.google.common.collect.Iterators;
80 import com.google.common.collect.Lists;
81
82 /**
83 *
84 * Various utilities for scans
85 *
86 *
87 * @since 0.1
88 */
89 public class ScanUtil {
90 public static final int[] SINGLE_COLUMN_SLOT_SPAN = new int[1];
91 /*
92 * Max length that we fill our key when we turn an inclusive key
93 * into a exclusive key.
94 */
95 private static final byte[] MAX_FILL_LENGTH_FOR_PREVIOUS_KEY = new byte[16];
96 static {
97 Arrays.fill(MAX_FILL_LENGTH_FOR_PREVIOUS_KEY, (byte)-1);
98 }
99 private static final byte[] ZERO_BYTE_ARRAY = new byte[1024];
100
101 private ScanUtil() {
102 }
103
104 public static void setTenantId(Scan scan, byte[] tenantId) {
105 scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
106 }
107
108 public static void setLocalIndex(Scan scan) {
109 scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX, PDataType.TRUE_BYTES);
110 }
111
112 public static boolean isLocalIndex(Scan scan) {
113 return scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX) != null;
114 }
115
116 public static boolean isNonAggregateScan(Scan scan) {
117 return scan.getAttribute(BaseScannerRegionObserver.NON_AGGREGATE_QUERY) != null;
118 }
119
120 // Use getTenantId and pass in column name to match against
121 // in as PSchema attribute. If column name matches in
122 // KeyExpressions, set on scan as attribute
123 public static ImmutableBytesPtr getTenantId(Scan scan) {
124 // Create Scan with special aggregation column over which to aggregate
125 byte[] tenantId = scan.getAttribute(PhoenixRuntime.TENANT_ID_ATTRIB);
126 if (tenantId == null) {
127 return null;
128 }
129 return new ImmutableBytesPtr(tenantId);
130 }
131
132 public static void setCustomAnnotations(Scan scan, byte[] annotations) {
133 scan.setAttribute(CUSTOM_ANNOTATIONS, annotations);
134 }
135
136 public static byte[] getCustomAnnotations(Scan scan) {
137 return scan.getAttribute(CUSTOM_ANNOTATIONS);
138 }
139
140 public static Scan newScan(Scan scan) {
141 try {
142 Scan newScan = new Scan(scan);
143 // Clone the underlying family map instead of sharing it between
144 // the existing and cloned Scan (which is the retarded default
145 // behavior).
146 TreeMap<byte [], NavigableSet<byte []>> existingMap = (TreeMap<byte[], NavigableSet<byte[]>>)scan.getFamilyMap();
147 Map<byte [], NavigableSet<byte []>> clonedMap = new TreeMap<byte [], NavigableSet<byte []>>(existingMap);
148 newScan.setFamilyMap(clonedMap);
149 // Carry over the reversed attribute
150 newScan.setReversed(scan.isReversed());
151 newScan.setSmall(scan.isSmall());
152 return newScan;
153 } catch (IOException e) {
154 throw new RuntimeException(e);
155 }
156 }
157 /**
158 * Intersects the scan start/stop row with the startKey and stopKey
159 * @param scan
160 * @param startKey
161 * @param stopKey
162 * @return false if the Scan cannot possibly return rows and true otherwise
163 */
164 public static boolean intersectScanRange(Scan scan, byte[] startKey, byte[] stopKey) {
165 return intersectScanRange(scan, startKey, stopKey, false);
166 }
167
168 public static boolean intersectScanRange(Scan scan, byte[] startKey, byte[] stopKey, boolean useSkipScan) {
169 boolean mayHaveRows = false;
170 int offset = 0;
171 if (ScanUtil.isLocalIndex(scan)) {
172 offset = startKey.length != 0 ? startKey.length : stopKey.length;
173 }
174 byte[] existingStartKey = scan.getStartRow();
175 byte[] existingStopKey = scan.getStopRow();
176 if (existingStartKey.length > 0) {
177 if (startKey.length == 0 || Bytes.compareTo(existingStartKey, startKey) > 0) {
178 startKey = existingStartKey;
179 }
180 } else {
181 mayHaveRows = true;
182 }
183 if (existingStopKey.length > 0) {
184 if (stopKey.length == 0 || Bytes.compareTo(existingStopKey, stopKey) < 0) {
185 stopKey = existingStopKey;
186 }
187 } else {
188 mayHaveRows = true;
189 }
190 scan.setStartRow(startKey);
191 scan.setStopRow(stopKey);
192 if (offset > 0 && useSkipScan) {
193 byte[] temp = null;
194 if (startKey.length != 0) {
195 temp =new byte[startKey.length - offset];
196 System.arraycopy(startKey, offset, temp, 0, startKey.length - offset);
197 startKey = temp;
198 }
199 if (stopKey.length != 0) {
200 temp = new byte[stopKey.length - offset];
201 System.arraycopy(stopKey, offset, temp, 0, stopKey.length - offset);
202 stopKey = temp;
203 }
204 }
205 mayHaveRows = mayHaveRows || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) < 0;
206
207 // If the scan is using skip scan filter, intersect and replace the filter.
208 if (mayHaveRows && useSkipScan) {
209 Filter filter = scan.getFilter();
210 if (filter instanceof SkipScanFilter) {
211 SkipScanFilter oldFilter = (SkipScanFilter)filter;
212 SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
213 if (newFilter == null) {
214 return false;
215 }
216 // Intersect found: replace skip scan with intersected one
217 scan.setFilter(newFilter);
218 } else if (filter instanceof FilterList) {
219 FilterList oldList = (FilterList)filter;
220 FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
221 for (Filter f : oldList.getFilters()) {
222 if (f instanceof SkipScanFilter) {
223 SkipScanFilter newFilter = ((SkipScanFilter)f).intersect(startKey, stopKey);
224 if (newFilter == null) {
225 return false;
226 }
227 newList.addFilter(newFilter);
228 } else {
229 newList.addFilter(f);
230 }
231 }
232 scan.setFilter(newList);
233 }
234 }
235 return mayHaveRows;
236 }
237
238 public static void andFilterAtBeginning(Scan scan, Filter andWithFilter) {
239 if (andWithFilter == null) {
240 return;
241 }
242 Filter filter = scan.getFilter();
243 if (filter == null) {
244 scan.setFilter(andWithFilter);
245 } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
246 FilterList filterList = (FilterList)filter;
247 List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
248 allFilters.add(andWithFilter);
249 allFilters.addAll(filterList.getFilters());
250 scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
251 } else {
252 scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(andWithFilter, filter)));
253 }
254 }
255
256 public static void andFilterAtEnd(Scan scan, Filter andWithFilter) {
257 if (andWithFilter == null) {
258 return;
259 }
260 Filter filter = scan.getFilter();
261 if (filter == null) {
262 scan.setFilter(andWithFilter);
263 } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
264 FilterList filterList = (FilterList)filter;
265 List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
266 allFilters.addAll(filterList.getFilters());
267 allFilters.add(andWithFilter);
268 scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
269 } else {
270 scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(filter, andWithFilter)));
271 }
272 }
273
274 public static void setQualifierRangesOnFilter(Scan scan, Pair<Integer, Integer> minMaxQualifiers) {
275 Filter filter = scan.getFilter();
276 if (filter != null) {
277 if (filter instanceof FilterList) {
278 for (Filter f : ((FilterList)filter).getFilters()) {
279 if (f instanceof MultiEncodedCQKeyValueComparisonFilter) {
280 ((MultiEncodedCQKeyValueComparisonFilter)f).setMinMaxQualifierRange(minMaxQualifiers);
281 }
282 }
283 } else if (filter instanceof MultiEncodedCQKeyValueComparisonFilter) {
284 ((MultiEncodedCQKeyValueComparisonFilter)filter).setMinMaxQualifierRange(minMaxQualifiers);
285 }
286 }
287 }
288
289 public static void setTimeRange(Scan scan, long ts) {
290 try {
291 scan.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, ts);
292 } catch (IOException e) {
293 throw new RuntimeException(e);
294 }
295 }
296
297 public static void setTimeRange(Scan scan, TimeRange range) {
298 try {
299 scan.setTimeRange(range.getMin(), range.getMax());
300 } catch (IOException e) {
301 throw new RuntimeException(e);
302 }
303 }
304
305 public static void setTimeRange(Scan scan, long minStamp, long maxStamp) {
306 try {
307 scan.setTimeRange(minStamp, maxStamp);
308 } catch (IOException e) {
309 throw new RuntimeException(e);
310 }
311 }
312
313 public static byte[] getMinKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan) {
314 return getKey(schema, slots, slotSpan, Bound.LOWER);
315 }
316
317 public static byte[] getMaxKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan) {
318 return getKey(schema, slots, slotSpan, Bound.UPPER);
319 }
320
321 private static byte[] getKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan, Bound bound) {
322 if (slots.isEmpty()) {
323 return KeyRange.UNBOUND;
324 }
325 int[] position = new int[slots.size()];
326 int maxLength = 0;
327 for (int i = 0; i < position.length; i++) {
328 position[i] = bound == Bound.LOWER ? 0 : slots.get(i).size()-1;
329 KeyRange range = slots.get(i).get(position[i]);
330 Field field = schema.getField(i + slotSpan[i]);
331 int keyLength = range.getRange(bound).length;
332 if (!field.getDataType().isFixedWidth()) {
333 keyLength++;
334 if (range.isUnbound(bound) && !range.isInclusive(bound) && field.getSortOrder() == SortOrder.DESC) {
335 keyLength++;
336 }
337 }
338 maxLength += keyLength;
339 }
340 byte[] key = new byte[maxLength];
341 int length = setKey(schema, slots, slotSpan, position, bound, key, 0, 0, position.length);
342 if (length == 0) {
343 return KeyRange.UNBOUND;
344 }
345 if (length == maxLength) {
346 return key;
347 }
348 byte[] keyCopy = new byte[length];
349 System.arraycopy(key, 0, keyCopy, 0, length);
350 return keyCopy;
351 }
352
353 /*
354 * Set the key by appending the keyRanges inside slots at positions as specified by the position array.
355 *
356 * We need to increment part of the key range, or increment the whole key at the end, depending on the
357 * bound we are setting and whether the key range is inclusive or exclusive. The logic for determining
358 * whether to increment or not is:
359 * range/single boundary bound increment
360 * range inclusive lower no
361 * range inclusive upper yes, at the end if occurs at any slots.
362 * range exclusive lower yes
363 * range exclusive upper no
364 * single inclusive lower no
365 * single inclusive upper yes, at the end if it is the last slots.
366 */
367 public static int setKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan, int[] position,
368 Bound bound, byte[] key, int byteOffset, int slotStartIndex, int slotEndIndex) {
369 return setKey(schema, slots, slotSpan, position, bound, key, byteOffset, slotStartIndex, slotEndIndex, slotStartIndex);
370 }
371
372 public static int setKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan, int[] position,
373 Bound bound, byte[] key, int byteOffset, int slotStartIndex, int slotEndIndex, int schemaStartIndex) {
374 int offset = byteOffset;
375 boolean lastInclusiveUpperSingleKey = false;
376 boolean anyInclusiveUpperRangeKey = false;
377 boolean lastUnboundUpper = false;
378 // The index used for slots should be incremented by 1,
379 // but the index for the field it represents in the schema
380 // should be incremented by 1 + value in the current slotSpan index
381 // slotSpan stores the number of columns beyond one that the range spans
382 Field field = null;
383 int i = slotStartIndex, fieldIndex = ScanUtil.getRowKeyPosition(slotSpan, slotStartIndex);
384 for (i = slotStartIndex; i < slotEndIndex; i++) {
385 // Build up the key by appending the bound of each key range
386 // from the current position of each slot.
387 KeyRange range = slots.get(i).get(position[i]);
388 // Use last slot in a multi-span column to determine if fixed width
389 field = schema.getField(fieldIndex + slotSpan[i]);
390 boolean isFixedWidth = field.getDataType().isFixedWidth();
391 /*
392 * If the current slot is unbound then stop if:
393 * 1) setting the upper bound. There's no value in
394 * continuing because nothing will be filtered.
395 * 2) setting the lower bound when the type is fixed length
396 * for the same reason. However, if the type is variable width
397 * continue building the key because null values will be filtered
398 * since our separator byte will be appended and incremented.
399 * 3) if the range includes everything as we cannot add any more useful
400 * information to the key after that.
401 */
402 lastUnboundUpper = false;
403 if ( range.isUnbound(bound) &&
404 ( bound == Bound.UPPER || isFixedWidth || range == KeyRange.EVERYTHING_RANGE) ){
405 lastUnboundUpper = (bound == Bound.UPPER);
406 break;
407 }
408 byte[] bytes = range.getRange(bound);
409 System.arraycopy(bytes, 0, key, offset, bytes.length);
410 offset += bytes.length;
411
412 /*
413 * We must add a terminator to a variable length key even for the last PK column if
414 * the lower key is non inclusive or the upper key is inclusive. Otherwise, we'd be
415 * incrementing the key value itself, and thus bumping it up too much.
416 */
417 boolean inclusiveUpper = range.isUpperInclusive() && bound == Bound.UPPER;
418 boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER && range != KeyRange.EVERYTHING_RANGE;
419 boolean exclusiveUpper = !range.isUpperInclusive() && bound == Bound.UPPER;
420 // If we are setting the upper bound of using inclusive single key, we remember
421 // to increment the key if we exit the loop after this iteration.
422 //
423 // We remember to increment the last slot if we are setting the upper bound with an
424 // inclusive range key.
425 //
426 // We cannot combine the two flags together in case for single-inclusive key followed
427 // by the range-exclusive key. In that case, we do not need to increment the end at the
428 // end. But if we combine the two flag, the single inclusive key in the middle of the
429 // key slots would cause the flag to become true.
430 lastInclusiveUpperSingleKey = range.isSingleKey() && inclusiveUpper;
431 anyInclusiveUpperRangeKey |= !range.isSingleKey() && inclusiveUpper;
432 // A null or empty byte array is always represented as a zero byte
433 byte sepByte = SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), bytes.length == 0, field);
434
435 if ( !isFixedWidth && ( sepByte == QueryConstants.DESC_SEPARATOR_BYTE
436 || ( !exclusiveUpper
437 && (fieldIndex < schema.getMaxFields() || inclusiveUpper || exclusiveLower) ) ) ) {
438 key[offset++] = sepByte;
439 // Set lastInclusiveUpperSingleKey back to false if this is the last pk column
440 // as we don't want to increment the null byte in this case
441 lastInclusiveUpperSingleKey &= i < schema.getMaxFields()-1;
442 }
443 if (exclusiveUpper) {
444 // Cannot include anything else on the key, as otherwise
445 // keys that match the upper range will be included. For example WHERE k1 < 2 and k2 = 3
446 // would match k1 = 2, k2 = 3 which is wrong.
447 break;
448 }
449 // If we are setting the lower bound with an exclusive range key, we need to bump the
450 // slot up for each key part. For an upper bound, we bump up an inclusive key, but
451 // only after the last key part.
452 if (exclusiveLower) {
453 if (!ByteUtil.nextKey(key, offset)) {
454 // Special case for not being able to increment.
455 // In this case we return a negative byteOffset to
456 // remove this part from the key being formed. Since the
457 // key has overflowed, this means that we should not
458 // have an end key specified.
459 return -byteOffset;
460 }
461 // We're filtering on values being non null here, but we still need the 0xFF
462 // terminator, since DESC keys ignore the last byte as it's expected to be
463 // the terminator. Without this, we'd ignore the separator byte that was
464 // just added and incremented.
465 if (!isFixedWidth && bytes.length == 0
466 && SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, field) == QueryConstants.DESC_SEPARATOR_BYTE) {
467 key[offset++] = QueryConstants.DESC_SEPARATOR_BYTE;
468 }
469 }
470
471 fieldIndex += slotSpan[i] + 1;
472 }
473 if (lastInclusiveUpperSingleKey || anyInclusiveUpperRangeKey || lastUnboundUpper) {
474 if (!ByteUtil.nextKey(key, offset)) {
475 // Special case for not being able to increment.
476 // In this case we return a negative byteOffset to
477 // remove this part from the key being formed. Since the
478 // key has overflowed, this means that we should not
479 // have an end key specified.
480 return -byteOffset;
481 }
482 }
483 // Remove trailing separator bytes, since the columns may have been added
484 // after the table has data, in which case there won't be a separator
485 // byte.
486 if (bound == Bound.LOWER) {
487 while (--i >= schemaStartIndex && offset > byteOffset &&
488 !(field=schema.getField(--fieldIndex)).getDataType().isFixedWidth() &&
489 field.getSortOrder() == SortOrder.ASC &&
490 key[offset-1] == QueryConstants.SEPARATOR_BYTE) {
491 offset--;
492 fieldIndex -= slotSpan[i];
493 }
494 }
495 return offset - byteOffset;
496 }
497
498 public static interface BytesComparator {
499 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2);
500 };
501
502 private static final BytesComparator DESC_VAR_WIDTH_COMPARATOR = new BytesComparator() {
503
504 @Override
505 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
506 return DescVarLengthFastByteComparisons.compareTo(b1, s1, l1, b2, s2, l2);
507 }
508
509 };
510
511 private static final BytesComparator ASC_FIXED_WIDTH_COMPARATOR = new BytesComparator() {
512
513 @Override
514 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
515 return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
516 }
517
518 };
519 public static BytesComparator getComparator(boolean isFixedWidth, SortOrder sortOrder) {
520 return isFixedWidth || sortOrder == SortOrder.ASC ? ASC_FIXED_WIDTH_COMPARATOR : DESC_VAR_WIDTH_COMPARATOR;
521 }
522 public static BytesComparator getComparator(Field field) {
523 return getComparator(field.getDataType().isFixedWidth(),field.getSortOrder());
524 }
525 /**
526 * Perform a binary lookup on the list of KeyRange for the tightest slot such that the slotBound
527 * of the current slot is higher or equal than the slotBound of our range.
528 * @return the index of the slot whose slot bound equals or are the tightest one that is
529 * smaller than rangeBound of range, or slots.length if no bound can be found.
530 */
531 public static int searchClosestKeyRangeWithUpperHigherThanPtr(List<KeyRange> slots, ImmutableBytesWritable ptr, int lower, Field field) {
532 int upper = slots.size() - 1;
533 int mid;
534 BytesComparator comparator = ScanUtil.getComparator(field.getDataType().isFixedWidth(), field.getSortOrder());
535 while (lower <= upper) {
536 mid = (lower + upper) / 2;
537 int cmp = slots.get(mid).compareUpperToLowerBound(ptr, true, comparator);
538 if (cmp < 0) {
539 lower = mid + 1;
540 } else if (cmp > 0) {
541 upper = mid - 1;
542 } else {
543 return mid;
544 }
545 }
546 mid = (lower + upper) / 2;
547 if (mid == 0 && slots.get(mid).compareUpperToLowerBound(ptr, true, comparator) > 0) {
548 return mid;
549 } else {
550 return ++mid;
551 }
552 }
553
554 public static ScanRanges newScanRanges(List<? extends Mutation> mutations) throws SQLException {
555 List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
556 for (Mutation m : mutations) {
557 keys.add(PVarbinary.INSTANCE.getKeyRange(m.getRow()));
558 }
559 ScanRanges keyRanges = ScanRanges.createPointLookup(keys);
560 return keyRanges;
561 }
562
563 /**
564 * Converts a partially qualified KeyRange into a KeyRange with a
565 * inclusive lower bound and an exclusive upper bound, widening
566 * as necessary.
567 */
568 public static KeyRange convertToInclusiveExclusiveRange (KeyRange partialRange, RowKeySchema schema, ImmutableBytesWritable ptr) {
569 // Ensure minMaxRange is lower inclusive and upper exclusive, as that's
570 // what we need to intersect against for the HBase scan.
571 byte[] lowerRange = partialRange.getLowerRange();
572 if (!partialRange.lowerUnbound()) {
573 if (!partialRange.isLowerInclusive()) {
574 lowerRange = ScanUtil.nextKey(lowerRange, schema, ptr);
575 }
576 }
577
578 byte[] upperRange = partialRange.getUpperRange();
579 if (!partialRange.upperUnbound()) {
580 if (partialRange.isUpperInclusive()) {
581 upperRange = ScanUtil.nextKey(upperRange, schema, ptr);
582 }
583 }
584 if (partialRange.getLowerRange() != lowerRange || partialRange.getUpperRange() != upperRange) {
585 partialRange = KeyRange.getKeyRange(lowerRange, upperRange);
586 }
587 return partialRange;
588 }
589
590 private static byte[] nextKey(byte[] key, RowKeySchema schema, ImmutableBytesWritable ptr) {
591 int pos = 0;
592 int maxOffset = schema.iterator(key, ptr);
593 while (schema.next(ptr, pos, maxOffset) != null) {
594 pos++;
595 }
596 Field field = schema.getField(pos - 1);
597 if (!field.getDataType().isFixedWidth()) {
598 byte[] newLowerRange = new byte[key.length + 1];
599 System.arraycopy(key, 0, newLowerRange, 0, key.length);
600 newLowerRange[key.length] = SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), key.length==0, field);
601 key = newLowerRange;
602 } else {
603 key = Arrays.copyOf(key, key.length);
604 }
605 ByteUtil.nextKey(key, key.length);
606 return key;
607 }
608
609 public static boolean isReversed(Scan scan) {
610 return scan.getAttribute(BaseScannerRegionObserver.REVERSE_SCAN) != null;
611 }
612
613 public static void setReversed(Scan scan) {
614 scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.TRUE_BYTES);
615 scan.setLoadColumnFamiliesOnDemand(false);
616 }
617
618 public static void unsetReversed(Scan scan) {
619 scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES);
620 scan.setLoadColumnFamiliesOnDemand(true);
621 }
622
623 private static byte[] getReversedRow(byte[] startRow) {
624 /*
625 * Must get previous key because this is going from an inclusive start key to an exclusive stop key, and we need
626 * the start key to be included. We get the previous key by decrementing the last byte by one. However, with
627 * variable length data types, we need to fill with the max byte value, otherwise, if the start key is 'ab', we
628 * lower it to 'aa' which would cause 'aab' to be included (which isn't correct). So we fill with a 0xFF byte to
629 * prevent this. A single 0xFF would be enough for our primitive types (as that byte wouldn't occur), but for an
630 * arbitrary VARBINARY key we can't know how many bytes to tack on. It's lame of HBase to force us to do this.
631 */
632 byte[] newStartRow = startRow;
633 if (startRow.length != 0) {
634 newStartRow = Arrays.copyOf(startRow, startRow.length + MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
635 if (ByteUtil.previousKey(newStartRow, startRow.length)) {
636 System.arraycopy(MAX_FILL_LENGTH_FOR_PREVIOUS_KEY, 0, newStartRow, startRow.length,
637 MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
638 } else {
639 newStartRow = HConstants.EMPTY_START_ROW;
640 }
641 }
642 return newStartRow;
643 }
644
645 // Start/stop row must be swapped if scan is being done in reverse
646 public static void setupReverseScan(Scan scan) {
647 if (isReversed(scan)) {
648 byte[] newStartRow = getReversedRow(scan.getStartRow());
649 byte[] newStopRow = getReversedRow(scan.getStopRow());
650 scan.setStartRow(newStopRow);
651 scan.setStopRow(newStartRow);
652 scan.setReversed(true);
653 }
654 }
655
656 /**
657 * Start key and stop key of the original scan from client are regions start and end keys so
658 * prefix scan start/stop key to the start row/stop row suffix and set them as scan boundaries.
659 * @param scan
660 */
661 public static void setupLocalIndexScan(Scan scan) {
662 byte[] prefix = scan.getStartRow().length == 0 ? new byte[scan.getStopRow().length]: scan.getStartRow();
663 int prefixLength = scan.getStartRow().length == 0? scan.getStopRow().length: scan.getStartRow().length;
664 if(scan.getAttribute(SCAN_START_ROW_SUFFIX)!=null) {
665 scan.setStartRow(ScanRanges.prefixKey(scan.getAttribute(SCAN_START_ROW_SUFFIX), 0, prefix, prefixLength));
666 }
667 if(scan.getAttribute(SCAN_STOP_ROW_SUFFIX)!=null) {
668 scan.setStopRow(ScanRanges.prefixKey(scan.getAttribute(SCAN_STOP_ROW_SUFFIX), 0, prefix, prefixLength));
669 }
670 }
671
672 public static byte[] getActualStartRow(Scan localIndexScan, HRegionInfo regionInfo) {
673 return localIndexScan.getAttribute(SCAN_START_ROW_SUFFIX) == null ? localIndexScan
674 .getStartRow() : ScanRanges.prefixKey(localIndexScan.getAttribute(SCAN_START_ROW_SUFFIX), 0 ,
675 regionInfo.getStartKey().length == 0 ? new byte[regionInfo.getEndKey().length]
676 : regionInfo.getStartKey(),
677 regionInfo.getStartKey().length == 0 ? regionInfo.getEndKey().length : regionInfo
678 .getStartKey().length);
679 }
680
681 /**
682 * Set all attributes required and boundaries for local index scan.
683 * @param keyOffset
684 * @param regionStartKey
685 * @param regionEndKey
686 * @param newScan
687 */
688 public static void setLocalIndexAttributes(Scan newScan, int keyOffset, byte[] regionStartKey, byte[] regionEndKey, byte[] startRowSuffix, byte[] stopRowSuffix) {
689 if(ScanUtil.isLocalIndex(newScan)) {
690 newScan.setAttribute(SCAN_ACTUAL_START_ROW, regionStartKey);
691 newScan.setStartRow(regionStartKey);
692 newScan.setStopRow(regionEndKey);
693 if (keyOffset > 0 ) {
694 newScan.setAttribute(SCAN_START_ROW_SUFFIX, ScanRanges.stripPrefix(startRowSuffix, keyOffset));
695 } else {
696 newScan.setAttribute(SCAN_START_ROW_SUFFIX, startRowSuffix);
697 }
698 if (keyOffset > 0) {
699 newScan.setAttribute(SCAN_STOP_ROW_SUFFIX, ScanRanges.stripPrefix(stopRowSuffix, keyOffset));
700 } else {
701 newScan.setAttribute(SCAN_STOP_ROW_SUFFIX, stopRowSuffix);
702 }
703 }
704 }
705
706 public static boolean isContextScan(Scan scan, StatementContext context) {
707 return Bytes.compareTo(context.getScan().getStartRow(), scan.getStartRow()) == 0 && Bytes
708 .compareTo(context.getScan().getStopRow(), scan.getStopRow()) == 0;
709 }
710 public static int getRowKeyOffset(byte[] regionStartKey, byte[] regionEndKey) {
711 return regionStartKey.length > 0 ? regionStartKey.length : regionEndKey.length;
712 }
713
714 private static void setRowKeyOffset(Filter filter, int offset) {
715 if (filter instanceof BooleanExpressionFilter) {
716 BooleanExpressionFilter boolFilter = (BooleanExpressionFilter)filter;
717 IndexUtil.setRowKeyExpressionOffset(boolFilter.getExpression(), offset);
718 } else if (filter instanceof SkipScanFilter) {
719 SkipScanFilter skipScanFilter = (SkipScanFilter)filter;
720 skipScanFilter.setOffset(offset);
721 } else if (filter instanceof DistinctPrefixFilter) {
722 DistinctPrefixFilter prefixFilter = (DistinctPrefixFilter) filter;
723 prefixFilter.setOffset(offset);
724 }
725 }
726
727 public static void setRowKeyOffset(Scan scan, int offset) {
728 Filter filter = scan.getFilter();
729 if (filter == null) {
730 return;
731 }
732 if (filter instanceof FilterList) {
733 FilterList filterList = (FilterList)filter;
734 for (Filter childFilter : filterList.getFilters()) {
735 setRowKeyOffset(childFilter, offset);
736 }
737 } else {
738 setRowKeyOffset(filter, offset);
739 }
740 }
741
742 public static int[] getDefaultSlotSpans(int nSlots) {
743 return new int[nSlots];
744 }
745
746 /**
747 * Finds the position in the row key schema for a given position in the scan slots.
748 * For example, with a slotSpan of {0, 1, 0}, the slot at index 1 spans an extra column in the row key. This means
749 * that the slot at index 2 has a slot index of 2 but a row key index of 3.
750 * To calculate the "adjusted position" index, we simply add up the number of extra slots spanned and offset
751 * the slotPosition by that much.
752 * @param slotSpan the extra span per skip scan slot. corresponds to {@link ScanRanges#slotSpan}
753 * @param slotPosition the index of a slot in the SkipScan slots list.
754 * @return the equivalent row key position in the RowKeySchema
755 */
756 public static int getRowKeyPosition(int[] slotSpan, int slotPosition) {
757 int offset = 0;
758
759 for(int i = 0; i < slotPosition; i++) {
760 offset += slotSpan[i];
761 }
762
763 return offset + slotPosition;
764 }
765
766 public static boolean isAnalyzeTable(Scan scan) {
767 return scan.getAttribute((BaseScannerRegionObserver.ANALYZE_TABLE)) != null;
768 }
769
770 public static boolean crossesPrefixBoundary(byte[] key, byte[] prefixBytes, int prefixLength) {
771 if (key.length < prefixLength) {
772 return true;
773 }
774 if (prefixBytes.length >= prefixLength) {
775 return Bytes.compareTo(prefixBytes, 0, prefixLength, key, 0, prefixLength) != 0;
776 }
777 return hasNonZeroLeadingBytes(key, prefixLength);
778 }
779
780 public static byte[] getPrefix(byte[] startKey, int prefixLength) {
781 // If startKey is at beginning, then our prefix will be a null padded byte array
782 return startKey.length >= prefixLength ? startKey : ByteUtil.EMPTY_BYTE_ARRAY;
783 }
784
785 private static boolean hasNonZeroLeadingBytes(byte[] key, int nBytesToCheck) {
786 if (nBytesToCheck > ZERO_BYTE_ARRAY.length) {
787 do {
788 if (Bytes.compareTo(key, nBytesToCheck - ZERO_BYTE_ARRAY.length, ZERO_BYTE_ARRAY.length, ScanUtil.ZERO_BYTE_ARRAY, 0, ScanUtil.ZERO_BYTE_ARRAY.length) != 0) {
789 return true;
790 }
791 nBytesToCheck -= ZERO_BYTE_ARRAY.length;
792 } while (nBytesToCheck > ZERO_BYTE_ARRAY.length);
793 }
794 return Bytes.compareTo(key, 0, nBytesToCheck, ZERO_BYTE_ARRAY, 0, nBytesToCheck) != 0;
795 }
796
797 public static byte[] getTenantIdBytes(RowKeySchema schema, boolean isSalted, PName tenantId, boolean isMultiTenantTable, boolean isSharedIndex)
798 throws SQLException {
799 return isMultiTenantTable ?
800 getTenantIdBytes(schema, isSalted, tenantId, isSharedIndex)
801 : tenantId.getBytes();
802 }
803
804 public static byte[] getTenantIdBytes(RowKeySchema schema, boolean isSalted, PName tenantId, boolean isSharedIndex)
805 throws SQLException {
806 int pkPos = (isSalted ? 1 : 0) + (isSharedIndex ? 1 : 0);
807 Field field = schema.getField(pkPos);
808 PDataType dataType = field.getDataType();
809 byte[] convertedValue;
810 try {
811 Object value = dataType.toObject(tenantId.getString());
812 convertedValue = dataType.toBytes(value);
813 ImmutableBytesWritable ptr = new ImmutableBytesWritable(convertedValue);
814 dataType.pad(ptr, field.getMaxLength(), field.getSortOrder());
815 convertedValue = ByteUtil.copyKeyBytesIfNecessary(ptr);
816 } catch(IllegalDataException ex) {
817 throw new SQLExceptionInfo.Builder(SQLExceptionCode.TENANTID_IS_OF_WRONG_TYPE)
818 .build().buildException();
819 }
820 return convertedValue;
821 }
822
823 public static Iterator<Filter> getFilterIterator(Scan scan) {
824 Iterator<Filter> filterIterator;
825 Filter topLevelFilter = scan.getFilter();
826 if (topLevelFilter == null) {
827 filterIterator = Collections.emptyIterator();
828 } else if (topLevelFilter instanceof FilterList) {
829 filterIterator = ((FilterList) topLevelFilter).getFilters().iterator();
830 } else {
831 filterIterator = Iterators.singletonIterator(topLevelFilter);
832 }
833 return filterIterator;
834 }
835
836 /**
837 * Selecting underlying scanners in a round-robin fashion is possible if there is no ordering of
838 * rows needed, not even row key order. Also no point doing round robin of scanners if fetch
839 * size is 1.
840 */
841 public static boolean isRoundRobinPossible(OrderBy orderBy, StatementContext context)
842 throws SQLException {
843 int fetchSize = context.getStatement().getFetchSize();
844 return fetchSize > 1 && !shouldRowsBeInRowKeyOrder(orderBy, context)
845 && orderBy.getOrderByExpressions().isEmpty();
846 }
847
848 public static boolean forceRowKeyOrder(StatementContext context) {
849 return context.getConnection().getQueryServices().getProps()
850 .getBoolean(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, QueryServicesOptions.DEFAULT_FORCE_ROW_KEY_ORDER);
851 }
852
853 public static boolean shouldRowsBeInRowKeyOrder(OrderBy orderBy, StatementContext context) {
854 return forceRowKeyOrder(context) || orderBy == FWD_ROW_KEY_ORDER_BY || orderBy == REV_ROW_KEY_ORDER_BY;
855 }
856
857 public static TimeRange intersectTimeRange(TimeRange rowTimestampColRange, TimeRange scanTimeRange, Long scn) throws IOException, SQLException {
858 long scnToUse = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
859 long lowerRangeToBe = 0;
860 long upperRangeToBe = scnToUse;
861 if (rowTimestampColRange != null) {
862 long minRowTimestamp = rowTimestampColRange.getMin();
863 long maxRowTimestamp = rowTimestampColRange.getMax();
864 if ((lowerRangeToBe > maxRowTimestamp) || (upperRangeToBe < minRowTimestamp)) {
865 return null; // degenerate
866 } else {
867 // there is an overlap of ranges
868 lowerRangeToBe = Math.max(lowerRangeToBe, minRowTimestamp);
869 upperRangeToBe = Math.min(upperRangeToBe, maxRowTimestamp);
870 }
871 }
872 if (scanTimeRange != null) {
873 long minScanTimeRange = scanTimeRange.getMin();
874 long maxScanTimeRange = scanTimeRange.getMax();
875 if ((lowerRangeToBe > maxScanTimeRange) || (upperRangeToBe < lowerRangeToBe)) {
876 return null; // degenerate
877 } else {
878 // there is an overlap of ranges
879 lowerRangeToBe = Math.max(lowerRangeToBe, minScanTimeRange);
880 upperRangeToBe = Math.min(upperRangeToBe, maxScanTimeRange);
881 }
882 }
883 return new TimeRange(lowerRangeToBe, upperRangeToBe);
884 }
885
886 public static boolean isDefaultTimeRange(TimeRange range) {
887 return range.getMin() == 0 && range.getMax() == Long.MAX_VALUE;
888 }
889
890 /**
891 * @return true if scanners could be left open and records retrieved by simply advancing them on
892 * the server side. To make sure HBase doesn't cancel the leases and close the open
893 * scanners, we need to periodically renew leases. To look at the earliest HBase version
894 * that supports renewing leases, see
895 * {@link PhoenixDatabaseMetaData#MIN_RENEW_LEASE_VERSION}
896 */
897 public static boolean isPacingScannersPossible(StatementContext context) {
898 return context.getConnection().getQueryServices().isRenewingLeasesEnabled();
899 }
900
901 public static void addOffsetAttribute(Scan scan, Integer offset) {
902 scan.setAttribute(BaseScannerRegionObserver.SCAN_OFFSET, Bytes.toBytes(offset));
903 }
904
905 public static final boolean canQueryBeExecutedSerially(PTable table, OrderBy orderBy, StatementContext context) {
906 /*
907 * If ordering by columns not on the PK axis, we can't execute a query serially because we
908 * need to do a merge sort across all the scans which isn't possible with SerialIterators.
909 * Similar reasoning follows for salted and local index tables when ordering rows in a row
910 * key order. Serial execution is OK in other cases since SerialIterators will execute scans
911 * in the correct order.
912 */
913 if (!orderBy.getOrderByExpressions().isEmpty()
914 || ((table.getBucketNum() != null || table.getIndexType() == IndexType.LOCAL) && shouldRowsBeInRowKeyOrder(
915 orderBy, context))) {
916 return false;
917 }
918 return true;
919 }
920
921 public static boolean hasDynamicColumns(PTable table) {
922 for (PColumn col : table.getColumns()) {
923 if (col.isDynamic()) {
924 return true;
925 }
926 }
927 return false;
928 }
929
930 public static boolean isIndexRebuild(Scan scan) {
931 return scan.getAttribute((BaseScannerRegionObserver.REBUILD_INDEXES)) != null;
932 }
933
934 }