diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java index efe2d10dd2..582e60b0b0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSet.java @@ -29,11 +29,13 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import javax.annotation.Nonnull; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -167,30 +169,67 @@ public PartiallyOrderedSet mapAll(@Nonnull final Function PartiallyOrderedSet mapAll(@Nonnull final Map map) { - final var mappedElements = Sets.newLinkedHashSet(map.values()); - - final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); - for (final var entry : getTransitiveClosure().entries()) { - final var key = entry.getKey(); - final var value = entry.getValue(); - - if (map.containsKey(key) && map.containsKey(value)) { - resultDependencyMapBuilder.put(map.get(key), map.get(value)); - } else { - if (!map.containsKey(value)) { - // if key depends on value that does not exist -- do not insert the dependency and also remove key - final var mappedKey = map.get(key); - if (mappedKey != null) { - mappedElements.remove(mappedKey); + final var allRemovedBuilder = ImmutableSet.builder(); + final var leftToRemove = new HashSet<>(Sets.difference(set, map.keySet())); + + final var degreeMap = dependencyMap.entries().stream() + .collect(Collectors.groupingBy( + Map.Entry::getKey, + HashMap::new, + Collectors.summingInt(e -> 1))); + + while (!leftToRemove.isEmpty()) { + final T toRemove = leftToRemove.iterator().next(); + for (final var entry: dependencyMap.entries()) { + final var key = entry.getKey(); + if (entry.getValue().equals(toRemove) && degreeMap.containsKey(key)) { + final var updatedDegree = degreeMap.get(key) - 1; + if (updatedDegree == 0) { + leftToRemove.add(key); + degreeMap.remove(key); + } else { + degreeMap.put(key, updatedDegree); } } } + leftToRemove.remove(toRemove); + allRemovedBuilder.add(toRemove); } + final var allRemovedElements = allRemovedBuilder.build(); + + final var mappedElements = set.stream() + .filter(value -> !allRemovedElements.contains(value)) + .map(map::get) + .collect(Collectors.toSet()); + final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); + dependencyMap.entries().stream() + .filter(entry -> !allRemovedElements.contains(entry.getKey()) && !allRemovedElements.contains(entry.getValue())) + .map(entry -> Map.entry(map.get(entry.getKey()), map.get(entry.getValue()))) + .forEach(resultDependencyMapBuilder::put); // this needs the dependency map to be cleansed (which is done in the constructor) return PartiallyOrderedSet.of(mappedElements, resultDependencyMapBuilder.build()); } + @Nonnull + public PartiallyOrderedSet mapAll(@Nonnull final Multimap map) { + final var identityMapped = this.filterElements(map::containsKey); + + final var resultSet = identityMapped.getSet().stream() + .flatMap(value -> map.get(value).stream()) + .collect(Collectors.toSet()); + + final var resultDependencyMapBuilder = ImmutableSetMultimap.builder(); + for (final var entry: identityMapped.dependencyMap.entries()) { + Verify.verify(map.containsKey(entry.getKey()) && map.containsKey(entry.getValue())); + map.get(entry.getKey()).forEach(mappedKey -> resultDependencyMapBuilder.putAll(mappedKey, map.get(entry.getValue()))); + resultSet.addAll(map.get(entry.getKey())); + } + + // this needs the dependency map to be cleansed (which is done in the constructor) + return PartiallyOrderedSet.of(resultSet, resultDependencyMapBuilder.build()); + } + /** * Method that computes a new partially-ordered set that only retains elements that pass the filtering predicate. * The filtering is applied in a way that we do not break dependencies. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java index 86fe88a819..2a7dea349e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java @@ -53,6 +53,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -246,7 +247,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouped value " + groupedValue) .addLogInfo(LogMessageKeys.VALUE, groupedValue); } - argument = result.get(groupedValue); + argument = Iterables.getOnlyElement(result.get(groupedValue)); } else { throw new RecordCoreException("unable to plan group by with non-field value") .addLogInfo(LogMessageKeys.VALUE, groupedValue); @@ -272,7 +273,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouping value " + groupingValue) .addLogInfo(LogMessageKeys.VALUE, groupingValue); } - return pulledUpGroupingValuesMap.get(groupingValue); + return Iterables.getOnlyElement(pulledUpGroupingValuesMap.get(groupingValue)); }).collect(ImmutableList.toImmutableList()); final var groupingColsValue = RecordConstructorValue.ofUnnamed(pulledUpGroupingValues); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java index 44b5ffb846..d173e65d27 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java @@ -41,6 +41,7 @@ import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import javax.annotation.Nonnull; @@ -82,7 +83,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouped value " + groupedValue) .addLogInfo(LogMessageKeys.VALUE, groupedValue); } - argument = result.get(groupedValue); + argument = Iterables.getOnlyElement(result.get(groupedValue)); } else { throw new UnsupportedOperationException("unable to plan group by with non-field value " + groupedValue); } @@ -114,7 +115,7 @@ protected NonnullPair> constructGroupBy(@Nonnull f throw new RecordCoreException("could not pull grouping value " + groupingValue) .addLogInfo(LogMessageKeys.VALUE, groupingValue); } - return pulledUpGroupingValuesMap.get(groupingValue); + return Iterables.getOnlyElement(pulledUpGroupingValuesMap.get(groupingValue)); }).collect(ImmutableList.toImmutableList()); final var pulledUpGroupingValues = ImmutableList.builder().addAll(explicitPulledUpGroupingValues).add(implicitGroupingValue).build(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java index 19b2666024..c21d9d2937 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java @@ -44,6 +44,7 @@ import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import javax.annotation.Nonnull; @@ -349,7 +350,7 @@ private static ImmutableList pullUpPlaceholders(@Nonnull final Grap final var pulledUpValue = pulledUpPlaceholderValuesMap.get(value); final var parameterAlias = Objects.requireNonNull(childExpansionPlaceholderValuesMap.get(value)); - return Placeholder.newInstanceWithoutRanges(pulledUpValue, parameterAlias); + return Placeholder.newInstanceWithoutRanges(Iterables.getOnlyElement(pulledUpValue), parameterAlias); }) .collect(ImmutableList.toImmutableList()); } @@ -372,7 +373,7 @@ private static ImmutableList> pullUpResultColumns(@Nonnu throw new RecordCoreException("could not pull expansion value " + value) .addLogInfo(LogMessageKeys.VALUE, value); } - return pulledUpValuesMap.get(value); + return Iterables.getOnlyElement(pulledUpValuesMap.get(value)); }) .map(Column::unnamedOf) .collect(ImmutableList.toImmutableList()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java index 17b6ba8842..434f34002e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java @@ -121,8 +121,8 @@ static ImmutableBiMap adjustMatchedValueMap(@Nonnull final Correla candidateLowerExpression.getCorrelatedTo()), candidateAlias); final var pulledUpCandidateAggregateValue = candidatePullUpMap.get(candidateAggregateValue); - if (pulledUpCandidateAggregateValue != null) { - adjustedMatchedAggregateMapBuilder.put(queryAggregateValue, pulledUpCandidateAggregateValue); + if (!pulledUpCandidateAggregateValue.isEmpty()) { + adjustedMatchedAggregateMapBuilder.put(queryAggregateValue, Iterables.getOnlyElement(pulledUpCandidateAggregateValue)); } } return adjustedMatchedAggregateMapBuilder.build(); @@ -506,10 +506,10 @@ public static GroupByMappings pullUpGroupByMappings(@Nonnull final PartialMatch Sets.difference(queryAggregateValue.getCorrelatedToWithoutChildren(), queryExpression.getCorrelatedTo()), queryAlias); final var pulledUpQueryAggregateValue = pullUpMap.get(queryAggregateValue); - if (pulledUpQueryAggregateValue == null) { + if (pulledUpQueryAggregateValue.isEmpty()) { return GroupByMappings.empty(); } - unmatchedAggregateMapBuilder.put(unmatchedAggregateMapEntry.getKey(), pulledUpQueryAggregateValue); + unmatchedAggregateMapBuilder.put(unmatchedAggregateMapEntry.getKey(), Iterables.getOnlyElement(pulledUpQueryAggregateValue)); } return GroupByMappings.of(matchedGroupingsMap, matchedAggregatesMap, unmatchedAggregateMapBuilder.build()); @@ -527,8 +527,8 @@ private static ImmutableBiMap pullUpMatchedValueMap(@Nonnull final final var pullUpMap = queryResultValue.pullUp(ImmutableList.of(queryValue), EvaluationContext.empty(), AliasMap.emptyMap(), constantAliases, queryAlias); - final Value pulledUpQueryValue = pullUpMap.get(queryValue); - if (pulledUpQueryValue == null) { + final var pulledUpQueryValue = pullUpMap.get(queryValue); + if (pulledUpQueryValue.isEmpty()) { continue; } @@ -544,10 +544,10 @@ private static ImmutableBiMap pullUpMatchedValueMap(@Nonnull final candidateLowerExpression.getCorrelatedTo()), candidateAlias); final var pulledUpCandidateAggregateValue = candidatePullUpMap.get(candidateAggregateValue); - if (pulledUpCandidateAggregateValue == null) { + if (pulledUpCandidateAggregateValue.isEmpty()) { continue; } - matchedAggregatesMapBuilder.put(pulledUpQueryValue, pulledUpCandidateAggregateValue); + matchedAggregatesMapBuilder.put(Iterables.getOnlyElement(pulledUpQueryValue), Iterables.getOnlyElement(pulledUpCandidateAggregateValue)); } return matchedAggregatesMapBuilder.build(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java index e372da639a..768f416597 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Ordering.java @@ -37,9 +37,11 @@ import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; @@ -57,6 +59,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * This class captures an ordering property. @@ -289,15 +292,67 @@ public Set deriveRequestedOrderings(@Nonnull final RequestedO .collect(ImmutableSet.toImmutableSet()); } + /** + * Method to verify that a given {@link RequestedOrdering} can be used to construct an ordering sequence that is + * supported by the current ordering from the list of {@link Value}s that composes the {@link RequestedOrdering}. + * This method is similar to {@link Ordering#satisfiesGroupingValues(Set)} in the sense that then both check for + * existence of a sequence given a bunch of {@link Value}. They differ on the satisfiability criteria such that + * current method requires the candidate sequence's prefix to match the ordering of {@link Value} in the + * {@link RequestedOrdering}. + * Example 1: + * 
+     *     {@code
+     *     requested ordering:  a↑, e↑, c↑
+     *     this ordering:
+     *         partially ordered set:
+     *             members: a, b, x, c, d, e
+     *             dependencies: a ← c, b ← c, c ← d, e ← d, x ← d
+     *         bindings a↑, b↑, x↑, c↑, d↑, e↑
+     *
+     *     It should be possible to satisfy the requested ordering from the induced sub-poset of the original partially
+     *     ordered set over elements {a, e, c}:
+     *     induced sub-poset:
+     *         members: a, c, e
+     *         dependencies: a ← c
+     *
+     *     enumerated orderings (exhaustive) of the induced sub-poset:
+     *         a↑, c↑, e↑
+     *         a↑, e↑, c↑
+     *         e↑, a↑, c↑
+     *     }
+     *
+ * @param requestedOrdering the set of values which needs to match a prefix of a valid sequence + * @return the result of satisfiability of set of values in the ordering + */ public boolean satisfies(@Nonnull RequestedOrdering requestedOrdering) { - return !Iterables.isEmpty(enumerateCompatibleRequestedOrderings(requestedOrdering)); + if (requestedOrdering.isDistinct() && !isDistinct()) { + return false; + } + + for (final var requestedOrderingPart : requestedOrdering.getOrderingParts()) { + if (!bindingMap.containsKey(requestedOrderingPart.getValue())) { + return false; + } + final var bindings = bindingMap.get(requestedOrderingPart.getValue()); + final var sortOrder = sortOrder(bindings); + if (!sortOrder.isCompatibleWithRequestedSortOrder(requestedOrderingPart.getSortOrder())) { + return false; + } + } + final var requestedValuesSet = ImmutableSet.copyOf(requestedOrdering.getOrderingParts().stream().map(OrderingPart::getValue).iterator()); + return !Iterables.isEmpty( + TopologicalSort.satisfyingPermutations( + getOrderingSet().filterElements(requestedValuesSet::contains), + requestedOrdering.getOrderingParts().stream().map(OrderingPart::getValue).collect(Collectors.toList()), + Function.identity(), + permutation -> requestedOrdering.getOrderingParts().size())); } /** * Method to, given a constraining {@link RequestedOrdering}, enumerates all ordering sequences supported by this - * ordering that are compatible with the given {@link RequestedOrdering}. This functionality is needed when a - * provided order is used to again drive requested orderings for another quantifier participating in e.g. a set - * operation or similar. + * ordering that are compatible with the given {@link RequestedOrdering} having all the elements of the partially + * ordered set. This functionality is needed when a provided order is used to again drive requested orderings for + * another quantifier participating in e.g. a set operation or similar. * Example: * 
      *     {@code
@@ -309,14 +364,16 @@ public boolean satisfies(@Nonnull RequestedOrdering requestedOrdering) {
      *         bindings a↑, b=, x=, c↑, d↑, e↑
      *
      *     enumerated requested orderings (among others):
-     *         a↑, b↓, c↑
-     *         a↑, x↕, b↓, c↑
      *         a↑, x↕, b↓, c↑, d↕, e↕
      *         a↑, x↕, b↓, c↑, e↕, d↕
+     *
+     *     however, the following won't be in the enumerated sequences:
+     *         a↑, b↓, c↑
+     *         a↑, x↕, b↓, c↑
      *     }
      *
* @param requestedOrdering the {@link RequestedOrdering} this ordering needs to be compatible with - * @return an iterable of all compatible {@link RequestedOrdering}s + * @return boolean result of the compatibility check. */ @Nonnull public Iterable> enumerateCompatibleRequestedOrderings(@Nonnull final RequestedOrdering requestedOrdering) { @@ -324,7 +381,6 @@ public Iterable> enumerateCompatibleRequestedOrderin return ImmutableList.of(); } - final var requestedOrderingValuesBuilder = ImmutableList.builder(); final var requestedOrderingValuesMapBuilder = ImmutableMap.builder(); for (final var requestedOrderingPart : requestedOrdering.getOrderingParts()) { if (!bindingMap.containsKey(requestedOrderingPart.getValue())) { @@ -335,8 +391,6 @@ public Iterable> enumerateCompatibleRequestedOrderin if (!sortOrder.isCompatibleWithRequestedSortOrder(requestedOrderingPart.getSortOrder())) { return ImmutableList.of(); } - - requestedOrderingValuesBuilder.add(requestedOrderingPart.getValue()); requestedOrderingValuesMapBuilder.put(requestedOrderingPart.getValue(), requestedOrderingPart); } final var requestedOrderingValuesMap = requestedOrderingValuesMapBuilder.build(); @@ -359,6 +413,25 @@ public Iterable> enumerateCompatibleRequestedOrderin .collect(ImmutableList.toImmutableList())); } + /** + * Method to verify that a given set of grouping values forms a valid prefix of an enumerated sequence that is + * supported by this ordering. + * Example: + * 
+     *     {@code
+     *     requested grouping values set:  {a, e, c}
+     *     this ordering:
+     *         partially ordered set:
+     *             members: a, b, x, c, d, e
+     *             dependencies: a ← c, b ← c, c ← d, e ← d, x ← d
+     *         bindings a↑, b↑, x↑, c↑, d↑, e↑
+     *
+     *     It should be possible to satisfy {a, c, e} as a and e are independent and c depends on a.
+     *     }
+     *
+ * @param requestedGroupingValues the set of values which needs to match a prefix of a valid sequence + * @return the result of satisfiability of set of values in the ordering + */ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupingValues) { // no ordering left worth further considerations if (requestedGroupingValues.isEmpty()) { @@ -380,12 +453,11 @@ public boolean satisfiesGroupingValues(@Nonnull final Set requestedGroupi })) { return false; } - - final var permutations = TopologicalSort.topologicalOrderPermutations(orderingSet); + final var permutations = TopologicalSort.topologicalOrderPermutations(getOrderingSet().filterElements(requestedGroupingValues::contains)); for (final var permutation : permutations) { - final var containsAll = + final var satisfies = permutation.size() >= requestedGroupingValues.size() && requestedGroupingValues.containsAll(permutation.subList(0, requestedGroupingValues.size())); - if (containsAll) { + if (satisfies) { return true; } } @@ -401,23 +473,25 @@ public Ordering pullUp(@Nonnull final Value value, @Nonnull EvaluationContext ev translateBindings(entry.getValue(), toBePulledUpValues -> value.pullUp(toBePulledUpValues, evaluationContext, aliasMap, constantAliases, Quantifier.current())); - pulledUpBindingMapBuilder.putAll(entry.getKey(), pulledUpBindings); + pulledUpBindingMapBuilder.putAll(entry.getKey() /* old value*/, pulledUpBindings); } // pull up the values we actually could also pull up some of the bindings for final var pulledUpBindingMap = pulledUpBindingMapBuilder.build(); - final var pulledUpValuesMap = + final var pulledUpValuesMultimap = value.pullUp(pulledUpBindingMap.keySet(), evaluationContext, aliasMap, constantAliases, Quantifier.current()); - final var mappedOrderingSet = getOrderingSet().mapAll(pulledUpValuesMap); + final var mappedOrderingSet = getOrderingSet().mapAll(pulledUpValuesMultimap); final var mappedValues = mappedOrderingSet.getSet(); final var bindingMapBuilder = ImmutableSetMultimap.builder(); - for (final var entry : pulledUpValuesMap.entrySet()) { - if (mappedValues.contains(entry.getValue())) { - Verify.verify(pulledUpBindingMap.containsKey(entry.getKey())); - bindingMapBuilder.putAll(entry.getValue(), pulledUpBindingMap.get(entry.getKey())); + for (final var entry : pulledUpValuesMultimap.asMap().entrySet()) { + for (final var pulledUpValue: entry.getValue()) { + if (mappedValues.contains(pulledUpValue)) { + Verify.verify(pulledUpBindingMap.containsKey(entry.getKey())); + bindingMapBuilder.putAll(pulledUpValue, pulledUpBindingMap.get(entry.getKey())); + } } } @@ -436,18 +510,18 @@ public Ordering pushDown(@Nonnull final Value value, @Nonnull final EvaluationCo value.pushDown(toBePushedValues, DefaultValueSimplificationRuleSet.instance(), evaluationContext, aliasMap, constantAliases, Quantifier.current()); - final var resultMap = new LinkedIdentityMap(); + final var resultMapBuilder = ImmutableMultimap.builder(); for (int i = 0; i < toBePushedValues.size(); i++) { final Value toBePushedValue = toBePushedValues.get(i); final Value pushedValue = Objects.requireNonNull(pushedDownValues.get(i)); - resultMap.put(toBePushedValue, pushedValue); + resultMapBuilder.put(toBePushedValue, pushedValue); } - return resultMap; + return resultMapBuilder.build(); }); pushedBindingMapBuilder.putAll(entry.getKey(), pushedBindings); } - // pull up the values we actually could also pull up some of the the bindings for + // push down the values for which we actually could push down some of the bindings final var pushedBindingMap = pushedBindingMapBuilder.build(); final var values = pushedBindingMap.keySet(); final var pushedValues = @@ -505,7 +579,7 @@ public boolean isSingularFixedValue(@Nonnull final Value value) { @Nonnull private static Set translateBindings(@Nonnull final Collection bindings, - @Nonnull final Function, Map> translateFunction) { + @Nonnull final Function, Multimap> translateFunction) { final var translatedBindingsBuilder = ImmutableSet.builder(); if (areAllBindingsFixed(bindings)) { @@ -523,10 +597,9 @@ private static Set translateBindings(@Nonnull final Collection if (comparison instanceof Comparisons.ValueComparison) { final var valueComparison = (Comparisons.ValueComparison)comparison; if (translationMap.containsKey(valueComparison.getValue())) { - final var translatedComparison = - new Comparisons.ValueComparison(valueComparison.getType(), - translationMap.get(valueComparison.getValue())); - translatedBindingsBuilder.add(Binding.fixed(translatedComparison)); + translationMap.get(valueComparison.getValue()).stream() + .map(value -> new Comparisons.ValueComparison(valueComparison.getType(), Objects.requireNonNull(value))) + .forEach(translatedValueComparison -> translatedBindingsBuilder.add(Binding.fixed(translatedValueComparison))); } } else { translatedBindingsBuilder.add(binding); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java index d4fa8e00f2..65ee51d1d1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QuantifiedValue.java @@ -26,12 +26,12 @@ import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Streams; import javax.annotation.Nonnull; -import java.util.Map; import java.util.Set; /** @@ -58,11 +58,11 @@ default ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { @Nonnull @Override - default Map pullUp(@Nonnull final Iterable toBePulledUpValues, - @Nonnull final EvaluationContext evaluationContext, - @Nonnull final AliasMap aliasMap, - @Nonnull final Set constantAliases, - @Nonnull final CorrelationIdentifier upperBaseAlias) { + default Multimap pullUp(@Nonnull final Iterable toBePulledUpValues, + @Nonnull final EvaluationContext evaluationContext, + @Nonnull final AliasMap aliasMap, + @Nonnull final Set constantAliases, + @Nonnull final CorrelationIdentifier upperBaseAlias) { final var alias = getAlias(); final var areSimpleReferences = Streams.stream(toBePulledUpValues) @@ -71,7 +71,7 @@ default Map pullUp(@Nonnull final Iterable toBePu if (areSimpleReferences) { final var translationMap = TranslationMap.rebaseWithAliasMap(AliasMap.ofAliases(alias, upperBaseAlias)); - final var translatedMapBuilder = ImmutableMap.builder(); + final var translatedMapBuilder = ImmutableMultimap.builder(); for (final var toBePulledUpValue : toBePulledUpValues) { translatedMapBuilder.put(toBePulledUpValue, toBePulledUpValue.translateCorrelations(translationMap)); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java index 0b5da02fef..1c63a1dfe8 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java @@ -69,9 +69,10 @@ import com.google.common.base.Functions; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; import com.google.common.collect.Streams; import com.google.common.primitives.ImmutableIntArray; import com.google.protobuf.Message; @@ -81,7 +82,6 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -508,30 +508,27 @@ default Value simplify(@Nonnull final EvaluationContext evaluationContext, * resulting values of the pull-up logic */ @Nonnull - default Map pullUp(@Nonnull final Iterable toBePulledUpValues, - @Nonnull final EvaluationContext evaluationContext, - @Nonnull final AliasMap aliasMap, - @Nonnull final Set constantAliases, - @Nonnull final CorrelationIdentifier upperBaseAlias) { + default Multimap pullUp(@Nonnull final Iterable toBePulledUpValues, + @Nonnull final EvaluationContext evaluationContext, + @Nonnull final AliasMap aliasMap, + @Nonnull final Set constantAliases, + @Nonnull final CorrelationIdentifier upperBaseAlias) { final var resultPair = Simplification.compute(this, evaluationContext, toBePulledUpValues, aliasMap, constantAliases, PullUpValueRuleSet.ofPullUpValueRules()); if (resultPair == null) { - return ImmutableMap.of(); + return ImmutableMultimap.of(); } final var matchedValuesMap = resultPair.getRight(); - final var resultsMapBuilder = ImmutableMap.builder(); + final var resultsMapBuilder = ImmutableMultimap.builder(); for (final var toBePulledUpValue : toBePulledUpValues) { - final var compensation = matchedValuesMap.get(toBePulledUpValue); - if (compensation != null) { - resultsMapBuilder.put(toBePulledUpValue, - compensation.compensate( - QuantifiedObjectValue.of(upperBaseAlias, this.getResultType()))); - } + matchedValuesMap.getOrDefault(toBePulledUpValue, ImmutableList.of()) + .forEach(compensation -> resultsMapBuilder.put(toBePulledUpValue, + compensation.compensate(QuantifiedObjectValue.of(upperBaseAlias, this.getResultType())))); } - return resultsMapBuilder.buildKeepingLast(); + return resultsMapBuilder.build(); } /** @@ -547,8 +544,7 @@ default Map pullUp(@Nonnull final Iterable toBePu * @param aliasMap an alias map of equalities * @param constantAliases a set of aliases that are considered to be constant * @param upperBaseAlias an alias to be treated as current alias - * @return a map from {@link Value} to {@link Value} that related the values that the called passed in with the - * resulting values of the pull-up logic + * @return a list of resulting {@link Value}s of the push-down logic */ @Nonnull default List pushDown(@Nonnull final Iterable toBePushedDownValues, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java index e8c4a1e3a4..1e9d28ab1e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/CompensateRecordConstructorRule.java @@ -26,8 +26,10 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.MultiMatcher.all; @@ -44,7 +46,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class CompensateRecordConstructorRule extends ValueComputationRule, Map, RecordConstructorValue> { +public class CompensateRecordConstructorRule extends ValueComputationRule, Map>, RecordConstructorValue> { @Nonnull private static final BindingMatcher rootMatcher = recordConstructorValue(all(anyValue())); @@ -54,17 +56,18 @@ public CompensateRecordConstructorRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var recordConstructorValue = bindings.get(rootMatcher); - final var resultingMatchedValuesMap = new LinkedIdentityMap(); + final var resultingMatchedValuesMap = new LinkedIdentityMap>(); final var recordConstructorValueResult = call.getResult(recordConstructorValue); - final var matchedCompensation = recordConstructorValueResult == null + final var matchedCompensations = recordConstructorValueResult == null ? null : recordConstructorValueResult.getValue().get(recordConstructorValue); - if (matchedCompensation != null) { - resultingMatchedValuesMap.put(recordConstructorValue, matchedCompensation); + if (matchedCompensations != null) { + resultingMatchedValuesMap.put(recordConstructorValue, matchedCompensations); } else { + final var compensationsBuilderMap = new LinkedIdentityMap>(); for (int i = 0; i < recordConstructorValue.getColumns().size(); ++i) { final var column = recordConstructorValue.getColumns().get(i); final var childResultPair = call.getResult(column.getValue()); @@ -78,14 +81,19 @@ public void onMatch(@Nonnull final ValueComputationRuleCall new ImmutableList.Builder<>()) + .addAll(argumentValueCompensations + .stream() + .map(compensation -> new FieldValueCompensation(FieldValue.FieldPath.ofSingle(field, columnIdx), compensation)) + .iterator()); } } + compensationsBuilderMap.forEach((key, value) -> resultingMatchedValuesMap.put(key, value.build())); } - call.yieldValue(recordConstructorValue, resultingMatchedValuesMap); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java index e347050ef2..f8839b0379 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchConstantValueRule.java @@ -27,8 +27,10 @@ import com.apple.foundationdb.record.query.plan.cascades.values.AggregateValue; import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -38,7 +40,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchConstantValueRule extends ValueComputationRule, Map, Value> { +public class MatchConstantValueRule extends ValueComputationRule, Map>, Value> { @Nonnull private static final BindingMatcher rootMatcher = ValueMatchers.anyValue(); @@ -54,7 +56,7 @@ public Optional> getRootOperator() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { if (!call.isRoot()) { return; } @@ -62,7 +64,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); final var resultPair = call.getResult(value); final var matchedValuesMap = resultPair == null ? null : resultPair.getRight(); if (matchedValuesMap != null) { @@ -79,7 +81,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall toBePulledUpValue); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(ignored -> toBePulledUpValue)); } } call.yieldValue(value, newMatchedValuesMap); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java index df8e40208a..f74e9575ee 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchFieldValueAgainstQuantifiedObjectValueRule.java @@ -27,9 +27,11 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,7 +42,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchFieldValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map, QuantifiedObjectValue> { +public class MatchFieldValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map>, QuantifiedObjectValue> { @Nonnull private static final BindingMatcher rootMatcher = ValueMatchers.quantifiedObjectValue(); @@ -50,7 +52,7 @@ public MatchFieldValueAgainstQuantifiedObjectValueRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var quantifiedObjectValue = bindings.get(rootMatcher); final var toBePulledUpValues = Objects.requireNonNull(call.getArgument()); @@ -59,14 +61,14 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); for (final var toBePulledUpValue : toBePulledUpValues) { if (toBePulledUpValue instanceof FieldValue) { final var toBePulledUpFieldValue = (FieldValue)toBePulledUpValue; if (quantifiedObjectValue.semanticEquals(toBePulledUpFieldValue.getChild(), equivalenceMap)) { - newMatchedValuesMap.put(toBePulledUpValue, new FieldValueCompensation(toBePulledUpFieldValue.getFieldPath())); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(new FieldValueCompensation(toBePulledUpFieldValue.getFieldPath()))); } } else { inheritMatchedMapEntry(matchedValuesMap, newMatchedValuesMap, toBePulledUpValue); @@ -75,8 +77,8 @@ public void onMatch(@Nonnull final ValueComputationRuleCall matchedValuesMap, - @Nonnull final Map newMatchedValuesMap, + private static void inheritMatchedMapEntry(@Nullable final Map> matchedValuesMap, + @Nonnull final Map> newMatchedValuesMap, @Nonnull final Value toBePulledUpValue) { if (matchedValuesMap != null && matchedValuesMap.containsKey(toBePulledUpValue)) { newMatchedValuesMap.put(toBePulledUpValue, matchedValuesMap.get(toBePulledUpValue)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java index 1fd0267101..b99f7ba1c6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchOrCompensateFieldValueRule.java @@ -28,8 +28,10 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -42,7 +44,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchOrCompensateFieldValueRule extends ValueComputationRule, Map, FieldValue> { +public class MatchOrCompensateFieldValueRule extends ValueComputationRule, Map>, FieldValue> { @Nonnull private static final CollectionMatcher fieldPathOrdinalsMatcher = all(anyObject()); @@ -58,7 +60,7 @@ public MatchOrCompensateFieldValueRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var fieldValue = bindings.get(rootMatcher); @@ -67,7 +69,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); for (final var toBePulledUpValue : toBePulledUpValues) { if (toBePulledUpValue instanceof FieldValue) { @@ -83,19 +85,21 @@ public void onMatch(@Nonnull final ValueComputationRuleCall { if (pathSuffix.isEmpty()) { - newMatchedValuesMap.put(toBePulledUpValue, ValueCompensation.noCompensation()); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(ValueCompensation.noCompensation())); } else { - newMatchedValuesMap.put(toBePulledUpValue, new FieldValueCompensation(pathSuffix)); + newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(new FieldValueCompensation(pathSuffix))); } }); } } else { // there already is a matched field value - final var compensation = matchedValuesMap.get(toBePulledUpValue); - if (compensation instanceof FieldValueCompensation) { - final var fieldValueCompensation = (FieldValueCompensation)compensation; - final var pathSuffixOptional = FieldValue.stripFieldPrefixMaybe(fieldValueCompensation.getFieldPath(), fieldValue.getFieldPath()); - pathSuffixOptional.ifPresent(pathSuffix -> newMatchedValuesMap.put(toBePulledUpValue, fieldValueCompensation.withSuffix(pathSuffix))); + final var compensations = matchedValuesMap.get(toBePulledUpValue); + for (var compensation: compensations) { + if (compensation instanceof FieldValueCompensation) { + final var fieldValueCompensation = (FieldValueCompensation)compensation; + final var pathSuffixOptional = FieldValue.stripFieldPrefixMaybe(fieldValueCompensation.getFieldPath(), fieldValue.getFieldPath()); + pathSuffixOptional.ifPresent(pathSuffix -> newMatchedValuesMap.put(toBePulledUpValue, ImmutableList.of(fieldValueCompensation.withSuffix(pathSuffix)))); + } } } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java index 548f1a7665..89b7c72c69 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueAgainstQuantifiedObjectValueRule.java @@ -22,16 +22,19 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.LinkedIdentityMap; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.ValueMatchers; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -42,7 +45,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map, QuantifiedObjectValue> { +public class MatchValueAgainstQuantifiedObjectValueRule extends ValueComputationRule, Map>, QuantifiedObjectValue> { @Nonnull private static final BindingMatcher rootMatcher = ValueMatchers.quantifiedObjectValue(); @@ -52,7 +55,7 @@ public MatchValueAgainstQuantifiedObjectValueRule() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var quantifiedObjectValue = bindings.get(rootMatcher); final var toBePulledUpValues = Objects.requireNonNull(call.getArgument()); @@ -60,7 +63,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); for (final var toBePulledUpValue : toBePulledUpValues) { if (toBePulledUpValue instanceof FieldValue || @@ -82,18 +85,22 @@ public void onMatch(@Nonnull final ValueComputationRuleCall { final var translationMapBuilder = TranslationMap.regularBuilder(); translationMapBuilder.when(alias).then(((sourceAlias, leafValue) -> value)); return toBePulledUpValue.translateCorrelations(translationMapBuilder.build()); - })); + }))); } call.yieldValue(quantifiedObjectValue, newMatchedValuesMap); } - private static void inheritMatchedMapEntry(@Nullable final Map matchedValuesMap, - @Nonnull final Map newMatchedValuesMap, + private static void inheritMatchedMapEntry(@Nullable final Map> matchedValuesMap, + @Nonnull final Map> newMatchedValuesMap, @Nonnull final Value toBePulledUpValue) { if (matchedValuesMap != null && matchedValuesMap.containsKey(toBePulledUpValue)) { newMatchedValuesMap.put(toBePulledUpValue, matchedValuesMap.get(toBePulledUpValue)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java index 8c0f4b9069..3e31a793df 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/MatchValueRule.java @@ -25,8 +25,10 @@ import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -38,7 +40,7 @@ */ @API(API.Status.EXPERIMENTAL) @SuppressWarnings("PMD.TooManyStaticImports") -public class MatchValueRule extends ValueComputationRule, Map, Value> { +public class MatchValueRule extends ValueComputationRule, Map>, Value> { @Nonnull private static final BindingMatcher rootMatcher = anyValue(); @@ -53,12 +55,12 @@ public Optional> getRootOperator() { } @Override - public void onMatch(@Nonnull final ValueComputationRuleCall, Map> call) { + public void onMatch(@Nonnull final ValueComputationRuleCall, Map>> call) { final var bindings = call.getBindings(); final var value = bindings.get(rootMatcher); final var toBePulledUpValues = Objects.requireNonNull(call.getArgument()); - final var newMatchedValuesMap = new LinkedIdentityMap(); + final var newMatchedValuesMap = new LinkedIdentityMap>(); final var resultPair = call.getResult(value); final var matchedValuesMap = resultPair == null ? null : resultPair.getRight(); @@ -69,7 +71,7 @@ public void onMatch(@Nonnull final ValueComputationRuleCall, Map> { - protected static final ValueComputationRule, Map, ? extends Value> matchValueRule = new MatchValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchValueAgainstQuantifiedObjectValueRule = new MatchValueAgainstQuantifiedObjectValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchFieldValueAgainstQuantifiedObjectValueRule = new MatchFieldValueAgainstQuantifiedObjectValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchOrCompensateFieldValueRule = new MatchOrCompensateFieldValueRule(); - protected static final ValueComputationRule, Map, ? extends Value> compensateRecordConstructorRule = new CompensateRecordConstructorRule(); - protected static final ValueComputationRule, Map, ? extends Value> matchConstantValueRule = new MatchConstantValueRule(); +public class PullUpValueRuleSet extends ValueComputationRuleSet, Map>> { + protected static final ValueComputationRule, Map>, ? extends Value> matchValueRule = new MatchValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchValueAgainstQuantifiedObjectValueRule = new MatchValueAgainstQuantifiedObjectValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchFieldValueAgainstQuantifiedObjectValueRule = new MatchFieldValueAgainstQuantifiedObjectValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchOrCompensateFieldValueRule = new MatchOrCompensateFieldValueRule(); + protected static final ValueComputationRule, Map>, ? extends Value> compensateRecordConstructorRule = new CompensateRecordConstructorRule(); + protected static final ValueComputationRule, Map>, ? extends Value> matchConstantValueRule = new MatchConstantValueRule(); - protected static final Set, Map, ? extends Value>> PULL_UP_RULES = + protected static final Set, Map>, ? extends Value>> PULL_UP_RULES = ImmutableSet.of(matchValueRule, matchValueAgainstQuantifiedObjectValueRule, matchFieldValueAgainstQuantifiedObjectValueRule, @@ -50,11 +51,11 @@ public class PullUpValueRuleSet extends ValueComputationRuleSet, Map, ? extends Value>, ValueComputationRule, Map, ? extends Value>> PULL_UP_DEPENDS_ON; + protected static final SetMultimap, Map>, ? extends Value>, ValueComputationRule, Map>, ? extends Value>> PULL_UP_DEPENDS_ON; static { final var dependsOnBuilder = - ImmutableSetMultimap., Map, ? extends Value>, ValueComputationRule, Map, ? extends Value>>builder(); + ImmutableSetMultimap., Map>, ? extends Value>, ValueComputationRule, Map>, ? extends Value>>builder(); PULL_UP_RULES.forEach(rule -> { if (rule != matchConstantValueRule) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java index e25e27eb78..89aef3047c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/MaxMatchMap.java @@ -94,7 +94,7 @@ * correspondences between subtrees that include this subtree. *
* In the example above we can say that the query side value and the candidate value define maximum matches between - * {@code q.a -> q.a}, {@code q.b -> q.b}, and {@code q.b -> q.b}. If the candidate side had been + * {@code q.a -> q.a}, {@code q.b -> q.b}, and {@code q.c -> q.c}. If the candidate side had been * {@code rcv(q.a as a, q.b as b, q.c as c)}, the maximum matches between query side and candidate side would just * have been {@code rcv(q.a as a, q.b as b, q.c as c) -> rcv(q.a as a, q.b as b, q.c as c)} as the entire tree under * the root would have matched. Even though {@code q.a -> q.a}, {@code q.b -> q.b}, and {@code q.c -> q.c} are also @@ -255,10 +255,10 @@ public Optional translateQueryValueMaybe(@Nonnull final CorrelationIdenti final var queryPart = entry.getKey(); final var candidatePart = entry.getValue(); final var pulledUpdateCandidatePart = pulledUpCandidateValueMap.get(candidatePart); - if (pulledUpdateCandidatePart == null) { + if (pulledUpdateCandidatePart.isEmpty()) { return Optional.empty(); } - pulledUpMaxMatchMapBuilder.put(queryPart, pulledUpdateCandidatePart); + pulledUpMaxMatchMapBuilder.put(queryPart, Iterables.getOnlyElement(pulledUpdateCandidatePart)); } final var pulledUpMaxMatchMap = pulledUpMaxMatchMapBuilder.build(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java index 29e100b2f0..cbc3b74b58 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/translation/PullUp.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import javax.annotation.Nonnull; @@ -177,10 +178,10 @@ public Optional pullUpCandidateValueMaybe(@Nonnull final Value value) { currentRangedOverAliases), currentCandidateAlias); final var pulledUpCandidateAggregateValue = candidatePullUpMap.get(currentValue); - if (pulledUpCandidateAggregateValue == null) { + if (pulledUpCandidateAggregateValue.isEmpty()) { return Optional.empty(); } - currentValue = pulledUpCandidateAggregateValue; + currentValue = Iterables.getOnlyElement(pulledUpCandidateAggregateValue); if (currentPullUp.getParentPullUp() == null) { return Optional.of(currentValue); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java index b43482cdf1..2cfc7cb0d7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBInQueryTest.java @@ -2804,8 +2804,8 @@ void testInUnionWithIntersectionOnTwoPredicates(int replans, boolean dropNumValu ) ).where(inUnionValuesSources(only(inUnionInParameter(equalsObject("str_list"))))) ).where(inUnionValuesSources(only(inUnionInParameter(equalsObject("nv2_list")))))); - assertEquals(-1515083625, plan.planHash(PlanHashable.CURRENT_LEGACY)); - assertEquals(-1918411257, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + assertEquals(-1515112545, plan.planHash(PlanHashable.CURRENT_LEGACY)); + assertEquals(-1918440177, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); singleIndexScan = false; } else if (replans < 0 || dropNumValue3Index) { // Before replanning, we always end up with an intersection. Neither index alone is enough to satisfy both diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index 000321fe1f..368264f92e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -1197,7 +1197,7 @@ void testOrderedOrQueryWithMultipleValuesForEarlierColumn(OrQueryParams orQueryP // Index(multi_index [[odd, 0],[odd, 0]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'rec_no' None}] Index(multi_index [[odd, 2],[odd, 2]]) orQueryParams.setPlannerConfiguration(this); RecordQueryPlan plan = planQuery(query); - final KeyExpression comparisonKey = planner instanceof CascadesPlanner ? concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"), field("num_value_2")) : concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")); + final KeyExpression comparisonKey = planner instanceof CascadesPlanner ? concat(field("num_value_3_indexed"), field("num_value_2"), primaryKey("MySimpleRecord")) : concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")); final BindingMatcher planMatcher = queryPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[odd, 0],[odd, 0]]"))), indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[odd, 2],[odd, 2]]"))) @@ -1213,8 +1213,8 @@ void testOrderedOrQueryWithMultipleValuesForEarlierColumn(OrQueryParams orQueryP assertEquals(-521036388, plan.planHash(CURRENT_FOR_CONTINUATION)); } } else { - assertEquals(-2004621044, plan.planHash(CURRENT_LEGACY)); - assertEquals(661700575, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(-2004620414, plan.planHash(CURRENT_LEGACY)); + assertEquals(661700665, plan.planHash(CURRENT_FOR_CONTINUATION)); } try (FDBRecordContext context = openContext()) { @@ -1288,7 +1288,7 @@ void testOrderedOrQueryWithIntermediateUnorderedColumn(OrQueryParams orQueryPara final BindingMatcher planMatcher = queryPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[odd],[odd]]"))), indexPlan().where(indexName("multi_index")).and(scanComparisons(range("[[even],[even]]"))) - ), planner instanceof CascadesPlanner ? concat(field("num_value_2"), field("num_value_3_indexed"), primaryKey("MySimpleRecord"), field("str_value_indexed")) : concat(field("num_value_2"), field("num_value_3_indexed"), primaryKey("MySimpleRecord"))); + ), planner instanceof CascadesPlanner ? concat(field("num_value_2"), field("num_value_3_indexed"), field("str_value_indexed"), primaryKey("MySimpleRecord")) : concat(field("num_value_2"), field("num_value_3_indexed"), primaryKey("MySimpleRecord"))); assertMatchesExactly(plan, planMatcher); if (planner instanceof RecordQueryPlanner) { @@ -1300,8 +1300,8 @@ void testOrderedOrQueryWithIntermediateUnorderedColumn(OrQueryParams orQueryPara assertEquals(1294497974, plan.planHash(CURRENT_FOR_CONTINUATION)); } } else { - assertEquals(471565558, plan.planHash(CURRENT_LEGACY)); - assertEquals(1167320211, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(471565768, plan.planHash(CURRENT_LEGACY)); + assertEquals(1167320241, plan.planHash(CURRENT_FOR_CONTINUATION)); } try (FDBRecordContext context = openContext()) { @@ -1365,12 +1365,12 @@ void testNestedPredicates(OrQueryParams orQueryParams) throws Exception { indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town1]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town2]"))) ), - isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("name"), field("id"), field("stats").nest("hometown")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); + isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("name"), field("stats").nest("hometown"), field("id")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 1539206407 : 1539206374, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? 1716842733 : 1722562791, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 1539265987 : 1539265954, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? 1716902313 : 1722622371, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { assertEquals(orQueryParams.isSortReverse() ? 1766220 : 1766187, plan.planHash(CURRENT_LEGACY)); if (orQueryParams.shouldDeferFetch()) { @@ -1413,7 +1413,7 @@ void testNestedPredicatesWithExtraUnorderedColumns(OrQueryParams orQueryParams) indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town1]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS 0, EQUALS $town2]"))) ), - isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("name"), field("id"), field("stats.hometown")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); + isUseCascadesPlanner() ? concat(field("stats").nest("start_date"), field("stats.hometown"), field("name"), field("id")) : concat(field("stats").nest("start_date"), field("name"), primaryKey("RestaurantReviewer"))); } else if (orQueryParams.shouldNormalizeNestedFields()) { planMatcher = filterPlan( indexPlan() @@ -1440,8 +1440,8 @@ void testNestedPredicatesWithExtraUnorderedColumns(OrQueryParams orQueryParams) assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 1539206407 : 1539206374, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? 1716842733 : 1722562791, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 1541112037 : 1541112004, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? 1718748363 : 1724468421, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { if (orQueryParams.shouldNormalizeNestedFields()) { if (orQueryParams.shouldOmitPrimaryKeyInOrderingKey()) { @@ -1577,12 +1577,12 @@ void testOrderedUnionHasRepeatedColumnsInPrefixAndOrdering(OrQueryParams orQuery final BindingMatcher planMatcher = unionPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + value2Param + ", EQUALS $" + strValue1Param + "]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + value2Param + ", EQUALS $" + strValue2Param + "]"))) - ), isUseCascadesPlanner() ? concatenateFields("num_value_3_indexed", "rec_no", "str_value_indexed") : concatenateFields("num_value_3_indexed", "rec_no")); + ), isUseCascadesPlanner() ? concatenateFields("num_value_3_indexed", "str_value_indexed", "rec_no") : concatenateFields("num_value_3_indexed", "rec_no")); assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 973132071L : 973132038L, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? 1977118957L : 1982839015L, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 973132101L : 973132068L, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? 1977118987L : 1982839045L, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { assertEquals(orQueryParams.isSortReverse() ? -2058994186L : -2058994219L, plan.planHash(CURRENT_LEGACY)); if (orQueryParams.shouldDeferFetch()) { @@ -1666,12 +1666,12 @@ void testOrderedUnionHasRepeatedColumnsNotInPrefix(OrQueryParams orQueryParams) final BindingMatcher planMatcher = unionPlanMatcher(orQueryParams, List.of( indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + strValue1Param + "]"))), indexPlan().where(indexName(index.getName())).and(scanComparisons(range("[EQUALS $" + strValue2Param + "]"))) - ), isUseCascadesPlanner() ? concatenateFields("num_value_2", "num_value_3_indexed", "rec_no", "str_value_indexed") : concatenateFields("num_value_2", "num_value_3_indexed", "rec_no")); + ), isUseCascadesPlanner() ? concatenateFields("num_value_2", "num_value_3_indexed", "str_value_indexed", "rec_no") : concatenateFields("num_value_2", "num_value_3_indexed", "rec_no")); assertMatchesExactly(plan, planMatcher); assertEquals(orQueryParams.isSortReverse(), plan.isReverse()); if (isUseCascadesPlanner()) { - assertEquals(orQueryParams.isSortReverse() ? 1380805606 : 1380805573, plan.planHash(CURRENT_LEGACY)); - assertEquals(orQueryParams.isSortReverse() ? -1854527892 : -1848807834, plan.planHash(CURRENT_FOR_CONTINUATION)); + assertEquals(orQueryParams.isSortReverse() ? 1380805636 : 1380805603, plan.planHash(CURRENT_LEGACY)); + assertEquals(orQueryParams.isSortReverse() ? -1854527862 : -1848807804, plan.planHash(CURRENT_FOR_CONTINUATION)); } else { assertEquals(orQueryParams.isSortReverse() ? 336481815 : 336481782, plan.planHash(CURRENT_LEGACY)); if (orQueryParams.shouldDeferFetch()) { @@ -1759,7 +1759,7 @@ void orderByIncludingValuePortion(@Nonnull OrQueryParams orQueryParams, @Nonnull if (sortKeyContainsPrimaryKey) { expectedComparisonKey = concatenateFields("num_value_2", "rec_no", "num_value_3_indexed"); } else { - expectedComparisonKey = concatenateFields("num_value_2", "rec_no", "num_value_3_indexed"); + expectedComparisonKey = concatenateFields("num_value_2", "num_value_3_indexed", "rec_no"); } } else { expectedComparisonKey = concatenateFields("num_value_2", "rec_no"); @@ -1778,8 +1778,8 @@ void orderByIncludingValuePortion(@Nonnull OrQueryParams orQueryParams, @Nonnull assertEquals(orQueryParams.isSortReverse() ? 951372169 : 951372136, legacyHash); assertEquals(orQueryParams.isSortReverse() ? 477401359 : 483121417, continuationHash); } else { - assertEquals(orQueryParams.isSortReverse() ? 951372169 : 951372136, legacyHash); - assertEquals(orQueryParams.isSortReverse() ? 477401359 : 483121417, continuationHash); + assertEquals(orQueryParams.isSortReverse() ? 951372289 : 951372256, legacyHash); + assertEquals(orQueryParams.isSortReverse() ? 477401479 : 483121537, continuationHash); } } else { if (orQueryParams.shouldDeferFetch()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java index b2189a897f..3846aec861 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/combinatorics/PartiallyOrderedSetTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; import org.junit.jupiter.api.Test; import java.util.Map; @@ -160,4 +161,75 @@ void testEligibleSetsSingle() { eligibleSet = eligibleSet.removeEligibleElements(eligibleSet.eligibleElements()); assertTrue(eligibleSet.eligibleElements().isEmpty()); } + + @Test + void testFilterElements() { + final var a = CorrelationIdentifier.of("a"); + final var b = CorrelationIdentifier.of("b"); + final var c = CorrelationIdentifier.of("c"); + final var d = CorrelationIdentifier.of("d"); + final var e = CorrelationIdentifier.of("e"); + final var f = CorrelationIdentifier.of("f"); + final var g = CorrelationIdentifier.of("g"); + + // a < c, b < c, c < d, d < e, d < f, e < g, f < g + final var dependencyMapBuilder = ImmutableSetMultimap.builder(); + dependencyMapBuilder.putAll(c, b, a); + dependencyMapBuilder.putAll(d, c); + dependencyMapBuilder.putAll(e, d); + dependencyMapBuilder.putAll(f, d); + dependencyMapBuilder.putAll(g, e, f); + final var partialOrder = PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, f, g), dependencyMapBuilder.build()); + + // a < c, b < c + var filteredSet = ImmutableSet.of(a, b, c); + var actualSubset = partialOrder.filterElements(filteredSet::contains); + var expectedDependencyEntries = ImmutableSetMultimap.of(c, a, c, b); + actualSubset.getDependencyMap().entries().forEach(entry -> assertTrue(expectedDependencyEntries.get(entry.getKey()).contains(entry.getValue()))); + + // {a} + filteredSet = ImmutableSet.of(a); + actualSubset = partialOrder.filterElements(filteredSet::contains); + assertTrue(actualSubset.getDependencyMap().isEmpty()); + assertTrue(actualSubset.getSet().containsAll(filteredSet)); + + filteredSet = ImmutableSet.of(c, d, e, f, g); + actualSubset = partialOrder.filterElements(filteredSet::contains); + assertTrue(actualSubset.getDependencyMap().isEmpty()); + assertTrue(actualSubset.getSet().isEmpty()); + + filteredSet = ImmutableSet.of(a, e, g); + actualSubset = partialOrder.filterElements(filteredSet::contains); + assertTrue(actualSubset.getDependencyMap().isEmpty()); + assertTrue(actualSubset.getSet().containsAll(ImmutableSet.of(a))); + } + + @Test + void testMapAllWithMultiMap() { + final var a = CorrelationIdentifier.of("a"); + final var b = CorrelationIdentifier.of("b"); + final var c = CorrelationIdentifier.of("c"); + final var d = CorrelationIdentifier.of("d"); + + // a < b, a < c, c < d, b < d + final var dependencyMapBuilder = ImmutableSetMultimap.builder(); + dependencyMapBuilder.putAll(c, a); + dependencyMapBuilder.putAll(b, a); + dependencyMapBuilder.putAll(d, b, c); + final var partialOrder = PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d), dependencyMapBuilder.build()); + + var mapToBuilder = ImmutableSetMultimap.builder(); + final var a_1 = CorrelationIdentifier.of("a_1"); + final var a_2 = CorrelationIdentifier.of("a_2"); + final var b_1 = CorrelationIdentifier.of("b_1"); + final var d_1 = CorrelationIdentifier.of("d_1"); + final var d_2 = CorrelationIdentifier.of("d_2"); + mapToBuilder.putAll(a, a_1, a_2); + mapToBuilder.putAll(b, b_1); + mapToBuilder.putAll(d, d_1, d_2); + + var actualSubset = partialOrder.mapAll(mapToBuilder.build()); + var expectedDependencyEntries = ImmutableSetMultimap.of(b_1, a_1, b_1, a_2, d_1, b_1, d_2, b_1); + actualSubset.getDependencyMap().entries().forEach(entry -> assertTrue(expectedDependencyEntries.get(entry.getKey()).contains(entry.getValue()))); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java index 29e2f500e5..caca69fd93 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/OrderingTest.java @@ -47,9 +47,11 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; class OrderingTest { @Test @@ -327,6 +329,320 @@ void testPullUp4() { assertEquals(expectedOrdering, pulledUpOrdering); } + @Test + void testPullUp5() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var innerOrderedSet = PartiallyOrderedSet.of(ImmutableSet.of(a, b, c), ImmutableSetMultimap.of(b, a)); + final var innerOrdering = + Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING), innerOrderedSet, false); + + final var rcv2 = RecordConstructorValue.ofColumns(ImmutableList.of( + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("a_1")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("a"))), + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("b_1")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("b"))), + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("a_2")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("a"))), + Column.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("b_2")), FieldValue.ofFieldNames(ValueTestHelpers.qov(), ImmutableList.of("b"))) + )); + final var pulledUpOrdering = + innerOrdering.pullUp(rcv2, EvaluationContext.empty(), AliasMap.emptyMap(), Set.of()); + + final var qovCurrent = QuantifiedObjectValue.of(Quantifier.current(), rcv2.getResultType()); + final var ap1 = ValueTestHelpers.field(qovCurrent, "a_1"); + final var bp1 = ValueTestHelpers.field(qovCurrent, "b_1"); + final var ap2 = ValueTestHelpers.field(qovCurrent, "a_2"); + final var bp2 = ValueTestHelpers.field(qovCurrent, "b_2"); + + final var expectedOrdering = + Ordering.ofOrderingSet(bindingMap(ap1, ProvidedSortOrder.ASCENDING, + bp1, ProvidedSortOrder.ASCENDING, + ap2, ProvidedSortOrder.ASCENDING, + bp2, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(ap1, ap2, bp1, bp2), ImmutableSetMultimap.of(bp2, ap1, bp2, ap2, bp1, ap1, bp1, ap2)), + false); + + assertEquals(expectedOrdering, pulledUpOrdering); + } + + @Test + void testSatisfiesGroupingValues() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, d < e, d < x + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, e, d, x, d)), + false); + + Stream.of( + ImmutableSet.of(a), + ImmutableSet.of(b), + ImmutableSet.of(a, c), + ImmutableSet.of(a, b), + ImmutableSet.of(b, c), + ImmutableSet.of(a, b, c), + ImmutableSet.of(a, c, d), + ImmutableSet.of(a, c, d, e), + ImmutableSet.of(b, c, d, x), + ImmutableSet.of(b, c, d, x, e) + ).forEach(valueSet -> assertTrue(ordering.satisfiesGroupingValues(valueSet))); + + Stream.of( + ImmutableSet.of(c), + ImmutableSet.of(d), + ImmutableSet.of(e), + ImmutableSet.of(x), + ImmutableSet.of(a, d, x), + ImmutableSet.of(c, d, x), + ImmutableSet.of(e, d, x), + ImmutableSet.of(a, c, e), + ImmutableSet.of(b, c, x), + ImmutableSet.of(a, b, e, x) + ).forEach(valueSet -> assertFalse(ordering.satisfiesGroupingValues(valueSet))); + } + + @Test + void testSatisfiesGroupingValues2() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, e < d, x < d + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), + false); + + Stream.of( + ImmutableSet.of(a), + ImmutableSet.of(b), + ImmutableSet.of(e), + ImmutableSet.of(x), + ImmutableSet.of(a, c), + ImmutableSet.of(a, b), + ImmutableSet.of(b, c), + ImmutableSet.of(a, b, c), + ImmutableSet.of(a, c, d), + ImmutableSet.of(a, d, x), + ImmutableSet.of(e, d, x), + ImmutableSet.of(a, c, d, e), + ImmutableSet.of(b, c, d, x) + ).forEach(valueSet -> assertTrue(ordering.satisfiesGroupingValues(valueSet))); + + Stream.of( + ImmutableSet.of(c), + ImmutableSet.of(d), + ImmutableSet.of(c, d), + ImmutableSet.of(c, d, x) + ).forEach(valueSet -> assertFalse(ordering.satisfiesGroupingValues(valueSet))); + } + + @Test + void testSatisfiesRequiredOrdering() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, d < e, d < x + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, e, d, x, d)), + false); + + Stream.of( + RequestedOrdering.ofParts(requested(a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, d, e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, d, x, e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertTrue(ordering.satisfies(requestedOrdering))); + + Stream.of( + RequestedOrdering.ofParts(requested(c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, d, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, x, d, e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertFalse(ordering.satisfies(requestedOrdering))); + } + + @Test + void testSatisfiesRequiredOrdering2() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, e < d, x < d + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), + false); + + Stream.of( + RequestedOrdering.ofParts(requested(a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, b, c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x, a, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(x, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, c, e, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(a, e, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(e, a, c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(b, c, x, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertTrue(ordering.satisfies(requestedOrdering))); + + Stream.of( + RequestedOrdering.ofParts(requested(c), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, d), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()), + RequestedOrdering.ofParts(requested(c, d, x), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()) + ).forEach(requestedOrdering -> assertFalse(ordering.satisfies(requestedOrdering))); + } + + @Test + void testEnumerateCompatibleRequestedOrderings() { + final var qov = ValueTestHelpers.qov(); + final var a = ValueTestHelpers.field(qov, "a"); + final var b = ValueTestHelpers.field(qov, "b"); + final var c = ValueTestHelpers.field(qov, "c"); + final var d = ValueTestHelpers.field(qov, "d"); + final var e = ValueTestHelpers.field(qov, "e"); + final var x = ValueTestHelpers.field(qov, "x"); + + // a < c, b < c, c < d, e < d, x < d + final var ordering = Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING, + c, ProvidedSortOrder.ASCENDING, + d, ProvidedSortOrder.ASCENDING, + e, ProvidedSortOrder.ASCENDING, + x, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b, c, d, e, x), ImmutableSetMultimap.of(c, a, c, b, d, c, d, e, d, x)), + false); + + + var requestedOrdering = RequestedOrdering.ofParts(requested(), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()); + // #pattern of _, _, c, _, _, d = 2!*2! = 4 + // #pattern of _, _, _, c, _, d = 2*3!*1! = 12 + // #pattern of _, _, _, _, c, d = 4! = 24 + assertEquals(40, ImmutableList.copyOf(ordering.enumerateCompatibleRequestedOrderings(requestedOrdering)).size()); + + requestedOrdering = RequestedOrdering.ofParts(requested(a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()); + // #pattern of a, _, c, _, _, d = 1!*2! = 2 + // #pattern of a, _, _, c, _, d = 2*2!*1! = 4 + // #pattern of a, _, _, _, c, d = 3! = 6 + assertEquals(12, ImmutableList.copyOf(ordering.enumerateCompatibleRequestedOrderings(requestedOrdering)).size()); + + requestedOrdering = RequestedOrdering.ofParts(requested(e, a), RequestedOrdering.Distinctness.NOT_DISTINCT, false, Set.of()); + // #pattern of e, a, _, c, _, d = 1 + // #pattern of e, a, _, _, c, d = 2! = 2 + assertEquals(3, ImmutableList.copyOf(ordering.enumerateCompatibleRequestedOrderings(requestedOrdering)).size()); + } + + @Test + void testPushDown1() { + final var rcv = select("a", "b", "c"); + + final var qovCurrent = QuantifiedObjectValue.of(Quantifier.current(), rcv.getResultType()); + final var ap = ValueTestHelpers.field(qovCurrent, "ap"); + final var ordering = + Ordering.ofOrderingSet(bindingMap(ap, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(ap), ImmutableSetMultimap.of()), + false); + + final var pushedDownOrdering = ordering.pushDown(rcv, EvaluationContext.empty(), AliasMap.emptyMap(), Set.of()); + + final var a = ValueTestHelpers.field(ValueTestHelpers.qov(), "a"); + final var expectedOrdering = + Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a), ImmutableSetMultimap.of()), + false); + + assertEquals(expectedOrdering, pushedDownOrdering); + } + + @Test + void testPushDown2() { + final var rcv = select("a", "b", "c"); + + final var qovCurrent = QuantifiedObjectValue.of(Quantifier.current(), rcv.getResultType()); + final var ap = ValueTestHelpers.field(qovCurrent, "ap"); + final var bp = ValueTestHelpers.field(qovCurrent, "bp"); + final var ordering = + Ordering.ofOrderingSet(bindingMap(ap, ProvidedSortOrder.ASCENDING, + bp, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(ap, bp), ImmutableSetMultimap.of(bp, ap)), + false); + + final var pushedDownOrdering = ordering.pushDown(rcv, EvaluationContext.empty(), AliasMap.emptyMap(), Set.of()); + + final var a = ValueTestHelpers.field(ValueTestHelpers.qov(), "a"); + final var b = ValueTestHelpers.field(ValueTestHelpers.qov(), "b"); + final var expectedOrdering = + Ordering.ofOrderingSet(bindingMap(a, ProvidedSortOrder.ASCENDING, + b, ProvidedSortOrder.ASCENDING), + PartiallyOrderedSet.of(ImmutableSet.of(a, b), ImmutableSetMultimap.of(b, a)), + false); + + assertEquals(expectedOrdering, pushedDownOrdering); + } + @Test void testMergePartialOrdersNAry() { final var qov = ValueTestHelpers.qov(); @@ -540,7 +856,7 @@ void testCommonOrdering4() { RequestedOrdering.ofPrimitiveParts(requested(a, b, x), RequestedOrdering.Distinctness.PRESERVE_DISTINCTNESS, false); - assertFalse(mergedOrdering.satisfies(requestedOrdering)); + assertTrue(mergedOrdering.satisfies(requestedOrdering)); } @Nonnull diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java index 56857d9c5d..4283f79681 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ValueTranslationTest.java @@ -252,7 +252,6 @@ public void testMultiLevelValueTranslation() { fv(t_, "b", "m")), fv(t_, "j", "s"), fv(t_, "j", "q"), - fv(t_, "b", "t"), fv(t_, "b", "m") ); @@ -471,7 +470,7 @@ void maxMatchValueWithUnmatchableArithmeticOperationCase2() { public void maxMatchValueWithMatchableArithmeticOperationAndOtherConstantCorrelations() { /* 1st level: - (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | | T | T' | | | @@ -491,7 +490,6 @@ public void maxMatchValueWithMatchableArithmeticOperationAndOtherConstantCorrela fv(t_, "b", "m")), fv(t_, "j", "s"), fv(t_, "j", "q"), - fv(t_, "b", "t"), fv(t_, "b", "m") ); @@ -534,7 +532,7 @@ translation of (t.a.q + s, t.a.r, (t.b.t), t.j.s) with correlation mapping of t | | P | p' | | | - (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + (t.a.q + s, t.a.r, (t.b.t), t.j.s) ((t'.a.q + s', t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | | T | T' | | | @@ -600,7 +598,7 @@ private Type.Record getNType() { public void maxMatchValueWithCompositionOfTranslationMaps() { /* 1st level: - (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t) + (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q) | | T | T' | | | @@ -631,8 +629,7 @@ public void maxMatchValueWithCompositionOfTranslationMaps() { rcv(fv(t_, "b", "t"), fv(t_, "b", "m")), fv(t_, "j", "s"), - fv(t_, "j", "q"), - fv(t_, "b", "t") + fv(t_, "j", "q") ); final var mv = rcv( @@ -748,7 +745,7 @@ translation of (t.a.q, t.a.r, (t.b.t), t.j.s) with correlation mapping of t -> t | P' | | - ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | T' | | @@ -892,8 +889,7 @@ public void maxMatchDifferentCompositions() { rcv(fv(t_, "b", "t"), fv(t_, "b", "m")), fv(t_, "j", "s"), - fv(t_, "j", "q"), - fv(t_, "b", "t") + fv(t_, "j", "q") ); final var mv = rcv( @@ -920,7 +916,7 @@ public void maxMatchDifferentCompositions() { /* 1st level: - (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t) + (t.a.q, t.a.r, (t.b.t), t.j.s) ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q) | | T | T' | | | @@ -988,7 +984,7 @@ translation of (t.a.q, t.a.r, (t.b.t), t.j.s) with correlation mapping of t -> t | P' | | - ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.t, t'.b.m) + ((t'.a.q, t'.a.r), (t'.b.t, t'.b.m), t'.j.s, t'.j.q, t'.b.m) | T' | | diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java index bdcb8ee0a0..3176bcce4b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueComputationTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.Column; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; @@ -37,12 +38,13 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.tuple.TupleOrdering; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import javax.annotation.Nonnull; +import java.util.List; import java.util.Optional; /** @@ -89,13 +91,40 @@ void testPullUpFieldValue1() { final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); - final var expectedMap = ImmutableMap.of( + final var expectedMap = ImmutableMultimap.of( _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "xa")), _z, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1"))); Assertions.assertEquals(expectedMap, resultsMap); } + @Test + void testPullUpAmbiguous() { + final var someCurrentValue = ObjectValue.of(ALIAS, someRecordType()); + // (_.x as _0, _.x.xa as _1) + final var pulledThroughValue = RecordConstructorValue.ofColumns(ImmutableList.of( + Column.unnamedOf(FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x"))), + Column.unnamedOf(FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x", "xa"))) + )); + + // _.x.xa + final var _x_xa = FieldValue.ofFieldNames(someCurrentValue, ImmutableList.of("x", "xa")); + final var toBePulledUpValues = ImmutableList.of(_x_xa); + + final var resultsMap = pulledThroughValue.pullUp(toBePulledUpValues, + EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of(), ALIAS); + + final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); + + // _.x.xa -> { _0.xa, _1 } + final var expectedMap = ImmutableMultimap.of( + _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "xa")), + _x_xa, FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1")) + ); + + Assertions.assertEquals(expectedMap, resultsMap); + } + @Test void testPullUpFieldValueThroughStreamingAggregation() { // _ @@ -122,7 +151,7 @@ void testPullUpFieldValueThroughStreamingAggregation() { // _ final var someNewCurrentValue = QuantifiedObjectValue.of(ALIAS, completeResultValue.getResultType()); - final var expectedResult = ImmutableMap.of( + final var expectedResult = ImmutableMultimap.of( _x, FieldValue.ofOrdinalNumber(someNewCurrentValue, 0), _z, FieldValue.ofOrdinalNumber(someNewCurrentValue, 1)); Assertions.assertEquals(expectedResult, resultsMap); @@ -150,7 +179,7 @@ void testPullUpValue1() { final var new_a_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("a", "ab")); final var new_x_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("x", "xb")); - final var expectedMap = ImmutableMap.of( + final var expectedMap = ImmutableMultimap.of( record_type, new RecordTypeValue(QuantifiedObjectValue.of(UPPER_ALIAS, new Type.AnyRecord(true))), _a_ab__plus__x_xb, (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(new_a_b, new_x_b))); Assertions.assertEquals(expectedMap, resultsMap); @@ -178,12 +207,40 @@ void testPullUpValue2() { final var new_a_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1", "a", "ab")); final var new_x_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_1", "x", "xb")); - final var expectedMap = ImmutableMap.of( + final var expectedMap = ImmutableMultimap.of( record_type, new RecordTypeValue(FieldValue.ofFieldName(upperCurrentValue, "_1")), _a_ab__plus__x_xb, (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(new_a_b, new_x_b))); Assertions.assertEquals(expectedMap, resultsMap); } + @Test + void testPullUpValue3() { + final var alias1 = CorrelationIdentifier.of("blah1"); + final var qov1 = QuantifiedObjectValue.of(alias1, someRecordType()); + final var alias2 = CorrelationIdentifier.of("blah2"); + final var qov2 = QuantifiedObjectValue.of(alias2, Type.Record.fromFields(List.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("r2_a"))))); + + final var pulledThroughValue = RecordConstructorValue.ofUnnamed(ImmutableList.of(qov1, qov2)); + + final var _a_b = FieldValue.ofFieldNames(qov1, ImmutableList.of("a", "ab")); + final var _x_b = FieldValue.ofFieldNames(qov1, ImmutableList.of("x", "xb")); + // _.a.ab + _.x.xb + final var _a_ab__plus__x_xb = (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(_a_b, _x_b)); + final var toBePulledUpValues = ImmutableList.of(_a_ab__plus__x_xb); + + final var resultsMap = pulledThroughValue.pullUp(toBePulledUpValues, + EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of(), ALIAS); + final var upperCurrentValue = QuantifiedObjectValue.of(ALIAS, pulledThroughValue.getResultType()); + + final var new_a_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "a", "ab")); + final var new_x_b = FieldValue.ofFieldNames(upperCurrentValue, ImmutableList.of("_0", "x", "xb")); + + final var expectedMap = ImmutableMultimap.of( + _a_ab__plus__x_xb, (Value)new ArithmeticValue.AddFn().encapsulate(ImmutableList.of(new_a_b, new_x_b))); + Assertions.assertEquals(expectedMap, resultsMap); + } + @Test void testMatchAndCompensateRemainder1() { final var someCurrentValue = ObjectValue.of(ALIAS, someRecordType()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java index c2d97cc5c6..64de387100 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/simplification/ValueSimplificationTest.java @@ -280,8 +280,8 @@ void testSimplificationPullUp() { // _._0.b final var bFieldPulledUp = FieldValue.ofFieldNameAndFuseIfPossible(zeroFieldPulledUp, "b"); - Assertions.assertEquals(iFieldPulledUp, pulledUpValuesMap.get(iField)); - Assertions.assertEquals(bFieldPulledUp, pulledUpValuesMap.get(bField)); + Assertions.assertEquals(ImmutableList.of(iFieldPulledUp), pulledUpValuesMap.get(iField)); + Assertions.assertEquals(ImmutableList.of(bFieldPulledUp), pulledUpValuesMap.get(bField)); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java index f73df9743f..c9457664c9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expression.java @@ -42,6 +42,7 @@ import com.google.common.base.Suppliers; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import javax.annotation.Nonnull; @@ -193,7 +194,7 @@ public Expression pullUp(@Nonnull Value value, @Nonnull CorrelationIdentifier co simplifiedValue.pullUp(List.of(subExpression), EvaluationContext.empty(), aliasMap, constantAliases, correlationIdentifier); if (pulledUpExpressionMap.containsKey(subExpression)) { - return pulledUpExpressionMap.get(subExpression); + return Iterables.getOnlyElement(pulledUpExpressionMap.get(subExpression)); } return subExpression; } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java index 93c0d4604c..418f459c25 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Expressions.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.util.Assert; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -101,7 +102,8 @@ public Expressions pullUp(@Nonnull Value value, @Nonnull CorrelationIdentifier c simplifiedValue.pullUp(List.of(subExpression), EvaluationContext.empty(), aliasMap, constantAliases, correlationIdentifier); if (pulledUpExpressionMap.containsKey(subExpression)) { - return pulledUpExpressionMap.get(subExpression); + Assert.thatUnchecked(pulledUpExpressionMap.get(subExpression).size() == 1, ErrorCode.AMBIGUOUS_COLUMN, "Ambiguous columns for " + subExpression); + return Iterables.getOnlyElement(pulledUpExpressionMap.get(subExpression)); } return subExpression; } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java index 728c6883c4..8908663d7a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderByExpression.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.query.plan.cascades.OrderingPart; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.util.Assert; +import com.google.common.collect.Iterables; import javax.annotation.Nonnull; import java.util.List; @@ -91,8 +92,8 @@ public static Stream pullUp(@Nonnull final Stream BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET) +=g )( 08@hAISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET) digraph G { fontname=courier; rankdir=BT; @@ -11,35 +11,10 @@ digraph G { 3 [ label=<
Index
BITMAPINDEX1
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS CATEGORY)" ]; 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} +}  -bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY category, bitmap_bucket_offset(id) -O ̡B(408%@AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET) digraph G { - fontname=courier; - rankdir=BT; - splines=polyline; - 1 [ label=<
Value Computation
MAP (q6._2 AS BITMAP, q6._0 AS CATEGORY, q6._1 AS OFFSET)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(BYTES AS BITMAP, STRING AS CATEGORY, LONG AS OFFSET)" ]; - 2 [ label=<
Index Scan
scan type: BY_GROUP
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING AS _0, LONG AS _1, BYTES AS _2)" ]; - 3 [ label=<
Index
BITMAPINDEX2
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS CATEGORY)" ]; - 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} - -bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), bitmap_bucket_offset(id), bitmap_bucket_offset(id) -פ ُ(408%@hAISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET) -digraph G { - fontname=courier; - rankdir=BT; - splines=polyline; - 1 [ label=<
Value Computation
MAP (q6._1 AS BITMAP, q6._0 AS OFFSET)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(BYTES AS BITMAP, LONG AS OFFSET)" ]; - 2 [ label=<
Index Scan
scan type: BY_GROUP
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, BYTES AS _1)" ]; - 3 [ label=<
Index
BITMAPINDEX1
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS CATEGORY)" ]; - 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} - -bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), category, bitmap_bucket_offset(id) -O B(408%@AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET) digraph G { +bitmap-agg-index-testsEXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY category, bitmap_bucket_offset(id) +g ( 0٣L8@AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET) digraph G { fontname=courier; rankdir=BT; splines=polyline; diff --git a/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml b/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml index 6cadf1e00d..f26d326477 100644 --- a/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml +++ b/yaml-tests/src/test/resources/bitmap-aggregate-index.metrics.yaml @@ -3,52 +3,26 @@ bitmap-agg-index-tests: bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id) explain: 'AISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET)' - task_count: 483 - task_total_time_ms: 165 - transform_count: 148 - transform_time_ms: 135 - transform_yield_count: 52 - insert_time_ms: 1 - insert_new_count: 37 - insert_reused_count: 2 + task_count: 332 + task_total_time_ms: 128 + transform_count: 103 + transform_time_ms: 87 + transform_yield_count: 32 + insert_time_ms: 3 + insert_new_count: 24 + insert_reused_count: 1 - query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY category, bitmap_bucket_offset(id) explain: 'AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET)' - task_count: 483 - task_total_time_ms: 166 - transform_count: 148 - transform_time_ms: 139 - transform_yield_count: 52 - insert_time_ms: 2 - insert_new_count: 37 - insert_reused_count: 2 -- query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, - bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), - bitmap_bucket_offset(id), bitmap_bucket_offset(id) - explain: 'AISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | - MAP (_._1 AS BITMAP, _._0 AS OFFSET)' - task_count: 483 - task_total_time_ms: 57 - transform_count: 148 - transform_time_ms: 39 - transform_yield_count: 52 - insert_time_ms: 2 - insert_new_count: 37 - insert_reused_count: 2 -- query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, - category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), - category, bitmap_bucket_offset(id) - explain: 'AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) - | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET)' - task_count: 483 - task_total_time_ms: 167 - transform_count: 148 - transform_time_ms: 140 - transform_yield_count: 52 - insert_time_ms: 3 - insert_new_count: 37 - insert_reused_count: 2 + task_count: 332 + task_total_time_ms: 60 + transform_count: 103 + transform_time_ms: 38 + transform_yield_count: 32 + insert_time_ms: 1 + insert_new_count: 24 + insert_reused_count: 1 - query: EXPLAIN SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T2 GROUP BY bitmap_bucket_offset(id) explain: ISCAN(AGG_INDEX_1 <,>) | MAP (_ AS _0) | AGG (bitmap_construct_agg_l((_._0.ID) diff --git a/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql b/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql index 0aedc30da1..1840d92f66 100644 --- a/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql +++ b/yaml-tests/src/test/resources/bitmap-aggregate-index.yamsql @@ -22,8 +22,6 @@ schema_template: CREATE TABLE T1(id bigint, category string, PRIMARY KEY(id)) create index bitmapIndex1 as select bitmap_construct_agg(bitmap_bit_position(id)), bitmap_bucket_offset(id) from T1 group by bitmap_bucket_offset(id) create index bitmapIndex2 as select bitmap_construct_agg(bitmap_bit_position(id)), category, bitmap_bucket_offset(id) from T1 group by category, bitmap_bucket_offset(id) - create index bitmapIndex3 as select bitmap_construct_agg(bitmap_bit_position(id)), bitmap_bucket_offset(id), bitmap_bucket_offset(id), bitmap_bucket_offset(id) from T1 group by bitmap_bucket_offset(id), bitmap_bucket_offset(id), bitmap_bucket_offset(id) - create index bitmapIndex4 as select bitmap_construct_agg(bitmap_bit_position(id)), bitmap_bucket_offset(id), category, bitmap_bucket_offset(id) from T1 group by bitmap_bucket_offset(id), category, bitmap_bucket_offset(id) CREATE TABLE T2(id bigint, category string, PRIMARY KEY(id)) create index agg_index_1 as select bitmap_bucket_offset(id) from T2 order by bitmap_bucket_offset(id) @@ -59,14 +57,18 @@ test_block: {BITMAP: xStartsWith_1250'0400008', 'CATEGORY': 'world', 'OFFSET':0}] - - query: SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), bitmap_bucket_offset(id), bitmap_bucket_offset(id) - - explain: "AISCAN(BITMAPINDEX1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS BITMAP, _._0 AS OFFSET)" + - initialVersionLessThan: !current_version - unorderedResult: [{BITMAP: xStartsWith_1250'060000c', 'OFFSET':0}, {BITMAP: xStartsWith_1250'02', 'OFFSET':10000}] + - initialVersionAtLeast: !current_version + - error: "42702" - - query: SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, category, bitmap_bucket_offset(id) as offset FROM T1 GROUP BY bitmap_bucket_offset(id), category, bitmap_bucket_offset(id) - - explain: "AISCAN(BITMAPINDEX2 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS BITMAP, _._0 AS CATEGORY, _._1 AS OFFSET)" + - initialVersionLessThan: !current_version - unorderedResult: [{BITMAP: xStartsWith_1250'0200004', 'CATEGORY': 'hello', 'OFFSET':0}, {BITMAP: xStartsWith_1250'02', 'CATEGORY': 'hello', 'OFFSET':10000}, {BITMAP: xStartsWith_1250'0400008', 'CATEGORY': 'world', 'OFFSET':0}] + - initialVersionAtLeast: !current_version + - error: "42702" - - query: SELECT bitmap_construct_agg(bitmap_bit_position(id)) as bitmap, bitmap_bucket_offset(id) as offset FROM T2 GROUP BY bitmap_bucket_offset(id) - explain: "ISCAN(AGG_INDEX_1 <,>) | MAP (_ AS _0) | AGG (bitmap_construct_agg_l((_._0.ID) bitmap_bit_position 10000) AS _0) GROUP BY ((_._0.ID) bitmap_bucket_offset 10000 AS _0) | MAP (_._1._0 AS BITMAP, _._0._0 AS OFFSET)" diff --git a/yaml-tests/src/test/resources/orderby.metrics.binpb b/yaml-tests/src/test/resources/orderby.metrics.binpb index 3f8835fc98..4192a4266e 100644 --- a/yaml-tests/src/test/resources/orderby.metrics.binpb +++ b/yaml-tests/src/test/resources/orderby.metrics.binpb @@ -1,7 +1,8 @@  4 orderby-tests#EXPLAIN select c from t1 order by b -M »($08@cCOVERING(I1 <,> -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C) digraph G { + +M ($0,8@cCOVERING(I1 <,> -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C) digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -12,10 +13,10 @@ 3 -> 2 [ label=< q47> label="q47" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 4 -> 3 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q6> label="q6" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} +} 4 - orderby-tests#EXPLAIN select b from t1 order by c -^ Ҁ(+08)@cCOVERING(I2 <,> -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B) digraph G { + orderby-tests#EXPLAIN select b from t1 order by c +r^ ¸^(+08)@cCOVERING(I2 <,> -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B) digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -29,7 +30,7 @@ } 9 orderby-tests(EXPLAIN select c from t1 order by b desc -M ($0 8@kCOVERING(I1 <,> REVERSE -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)digraph G { +ȍM ($0)8@kCOVERING(I1 <,> REVERSE -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -43,7 +44,7 @@ } 9 orderby-tests(EXPLAIN select b from t1 order by c desc - ^ Ž(+0#8)@kCOVERING(I2 <,> REVERSE -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)digraph G { +IJ^ (+0r8)@kCOVERING(I2 <,> REVERSE -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)digraph G { fontname=courier; rankdir=BT; splines=polyline; @@ -57,7 +58,7 @@ } @ orderby-tests/EXPLAIN select c, b from t5 order by c, b desc; -5 (08@vCOVERING(I8 <,> -> [A: KEY[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY[0]]) | MAP (_.C AS C, _.B AS B) +5 (0'8@vCOVERING(I8 <,> -> [A: KEY[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY[0]]) | MAP (_.C AS C, _.B AS B) digraph G { fontname=courier; rankdir=BT; @@ -67,4 +68,43 @@ digraph G { 3 [ label=<
Index
I8
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B, LONG AS C)" ]; 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q32> label="q32" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +Z + orderby-testsIEXPLAIN select q as "nested", q.a as "ordered" from t2 order by "ordered" +5 (0%8@5ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered) +digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (q2.Q AS nested, q2.Q.A AS ordered)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B AS nested, LONG AS ordered)" ]; + 2 [ label=<
Index Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 [ label=<
Index
I4
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +s + orderby-testsbEXPLAIN select (T.*) from (select q as "nested", q.a as "ordered" from t2) as T order by "ordered" +Y9 E(0؏8@=ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0) +digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP ((q2.Q AS nested, q2.Q.A AS ordered) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B AS nested, LONG AS ordered AS _0)" ]; + 2 [ label=<
Index Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 [ label=<
Index
I4
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +E + orderby-tests4EXPLAIN select (*) from t2_func() order by "ordered" +9 ׽(0+8@>ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0) +digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP ((q2.Q AS nesting, q2.Q.A AS ordered) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, LONG AS B AS nesting, LONG AS ordered AS _0)" ]; + 2 [ label=<
Index Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 [ label=<
Index
I4
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS P, LONG AS A, LONG AS B AS Q)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; } \ No newline at end of file diff --git a/yaml-tests/src/test/resources/orderby.metrics.yaml b/yaml-tests/src/test/resources/orderby.metrics.yaml index 1671f87a02..19df8fa9d8 100644 --- a/yaml-tests/src/test/resources/orderby.metrics.yaml +++ b/yaml-tests/src/test/resources/orderby.metrics.yaml @@ -3,9 +3,9 @@ orderby-tests: explain: 'COVERING(I1 <,> -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)' task_count: 302 - task_total_time_ms: 12 + task_total_time_ms: 22 transform_count: 77 - transform_time_ms: 5 + transform_time_ms: 11 transform_yield_count: 36 insert_time_ms: 0 insert_new_count: 25 @@ -14,20 +14,20 @@ orderby-tests: explain: 'COVERING(I2 <,> -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)' task_count: 407 - task_total_time_ms: 7 + task_total_time_ms: 240 transform_count: 94 - transform_time_ms: 3 + transform_time_ms: 198 transform_yield_count: 43 - insert_time_ms: 0 + insert_time_ms: 7 insert_new_count: 41 insert_reused_count: 4 - query: EXPLAIN select c from t1 order by b desc explain: 'COVERING(I1 <,> REVERSE -> [A: KEY[2], B: KEY[0], C: VALUE[0]]) | MAP (_.C AS C, _.B AS B) | MAP (_.C AS C)' task_count: 302 - task_total_time_ms: 5 + task_total_time_ms: 14 transform_count: 77 - transform_time_ms: 2 + transform_time_ms: 8 transform_yield_count: 36 insert_time_ms: 0 insert_new_count: 25 @@ -36,21 +36,52 @@ orderby-tests: explain: 'COVERING(I2 <,> REVERSE -> [A: KEY[2], B: VALUE[0], C: KEY[0]]) | MAP (_.B AS B, _.C AS C) | MAP (_.B AS B)' task_count: 407 - task_total_time_ms: 19 + task_total_time_ms: 36 transform_count: 94 - transform_time_ms: 5 + transform_time_ms: 16 transform_yield_count: 43 - insert_time_ms: 0 + insert_time_ms: 1 insert_new_count: 41 insert_reused_count: 4 - query: EXPLAIN select c, b from t5 order by c, b desc; explain: 'COVERING(I8 <,> -> [A: KEY[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY[0]]) | MAP (_.C AS C, _.B AS B)' task_count: 210 - task_total_time_ms: 14 + task_total_time_ms: 18 transform_count: 53 - transform_time_ms: 6 + transform_time_ms: 9 transform_yield_count: 22 insert_time_ms: 0 insert_new_count: 20 insert_reused_count: 2 +- query: EXPLAIN select q as "nested", q.a as "ordered" from t2 order by "ordered" + explain: ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered) + task_count: 194 + task_total_time_ms: 11 + transform_count: 53 + transform_time_ms: 6 + transform_yield_count: 26 + insert_time_ms: 0 + insert_new_count: 17 + insert_reused_count: 2 +- query: EXPLAIN select (T.*) from (select q as "nested", q.a as "ordered" from + t2) as T order by "ordered" + explain: ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0) + task_count: 220 + task_total_time_ms: 187 + transform_count: 57 + transform_time_ms: 146 + transform_yield_count: 28 + insert_time_ms: 4 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select (*) from t2_func() order by "ordered" + explain: ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0) + task_count: 220 + task_total_time_ms: 11 + transform_count: 57 + transform_time_ms: 7 + transform_yield_count: 28 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 diff --git a/yaml-tests/src/test/resources/orderby.yamsql b/yaml-tests/src/test/resources/orderby.yamsql index 13c78c2ebe..2f6a5a6500 100644 --- a/yaml-tests/src/test/resources/orderby.yamsql +++ b/yaml-tests/src/test/resources/orderby.yamsql @@ -25,6 +25,9 @@ schema_template: create index i3 as select b, c from t1 order by c, b CREATE TYPE AS STRUCT st_1(a bigint, b bigint) create table t2(p bigint, q st_1, primary key(p)) + create function t2_func() as select q as "nesting", q.a as "ordered" from t2 + create function t2_func_2() as select q as "nesting", q.a as "ordered", q.b as "ordered" from t2 + create function t2_func_3() as select q as "nesting", q.a as "ordered1", q.b as "ordered2" from t2 create index i4 as select q.b, q.a from t2 order by q.a create index i5 as select q.b, q.a from t2 order by q.b, q.a create table t3(a bigint, b bigint, c bigint, d bigint, p bigint, primary key(p)) @@ -262,4 +265,41 @@ test_block: - query: select c, b from t5 order by c, b desc; - explain: "COVERING(I8 <,> -> [A: KEY:[3], B: from_ordered_bytes(KEY:[1], DESC_NULLS_LAST), C: KEY:[0]]) | MAP (_.C AS C, _.B AS B)" - result: [{0, 2}, {0, 1}, {1, 4}, {1, 3}, {5, 10}, {5, 9}, {5, 8}, {8, 7}, {8, 6}, {8, 5}] + - + # Ordering by a ambiguous projected column + - query: select q as "nested", q.a as "ordered" from t2 order by "ordered" + - supported_version: !current_version + - explain: "ISCAN(I4 <,>) | MAP (_.Q AS nested, _.Q.A AS ordered)" + - result: [ {{1, 2}, 1}, {{2, 1}, 2}, {{3, 2}, 3}, {{4, 1}, 4}, {{5, 2}, 5}, {{6, 1}, 6}, {{7, 2}, 7}, {{8, 1}, 8} ] + - + # Ordering by a ambiguous projected column + - query: select q as "nested", q.a as "ordered", q.b as "ordered" from t2 order by "ordered" + - error: "42702" + - + # Ordering by a ambiguous projected column in subquery + - query: select (T.*) from (select q as "nested", q.a as "ordered" from t2) as T order by "ordered" + - supported_version: !current_version + - explain: "ISCAN(I4 <,>) | MAP ((_.Q AS nested, _.Q.A AS ordered) AS _0)" + - result: [ {{{1, 2}, 1}}, {{{2, 1}, 2}}, {{{3, 2}, 3}}, {{{4, 1}, 4}}, {{{5, 2}, 5}}, {{{6, 1}, 6}}, {{{7, 2}, 7}}, {{{8, 1}, 8}} ] + - + # Ordering by a column in table function that has been projected in multiple ways + - query: select (*) from t2_func() order by "ordered" + - supported_version: !current_version + - explain: "ISCAN(I4 <,>) | MAP ((_.Q AS nesting, _.Q.A AS ordered) AS _0)" + - result: [ {{{1, 2}, 1}}, {{{2, 1}, 2}}, {{{3, 2}, 3}}, {{{4, 1}, 4}}, {{{5, 2}, 5}}, {{{6, 1}, 6}}, {{{7, 2}, 7}}, {{{8, 1}, 8}} ] + - + # Ordering by a column in table function that has been projected in multiple ways + - query: select (*) from t2_func_2() order by "ordered" + # This should probably return AMBIGUOUS_COLUMN, but currently it returns UNKNOWN_COLUMN. + - error: "42703" + - + # qualification on order by clause is currently not supported entirely + - query: select (q as "nested", q.a as "ordered") as "st" from t2 order by "st"."ordered" + - supported_version: !current_version + - error: "42703" + - + # Ordering by a column in table function that has been projected in multiple ways + - query: select (*) from t2_func_3() order by "ordered2", "ordered1" + - supported_version: !current_version + - result: [ {{{2, 1}, 2, 1}}, {{{4, 1}, 4, 1}}, {{{6, 1}, 6, 1}}, {{{8, 1}, 8, 1}}, {{{1, 2}, 1, 2}}, {{{3, 2}, 3, 2}}, {{{5, 2}, 5, 2}}, {{{7, 2}, 7, 2}} ] ...