IKKBZ Join ordering

src/engine/CMakeLists.txt

@@ -1,6 +1,8 @@
 add_subdirectory(sparqlExpressions)
+add_subdirectory(joinOrdering)
 add_library(SortPerformanceEstimator SortPerformanceEstimator.cpp)
 qlever_target_link_libraries(SortPerformanceEstimator)
+
 add_library(engine
         Engine.cpp QueryExecutionTree.cpp Operation.cpp Result.cpp LocalVocab.cpp
         IndexScan.cpp Join.cpp Sort.cpp

src/engine/joinOrdering/CMakeLists.txt

@@ -0,0 +1,10 @@
+add_library(joinOrdering
+        QueryGraph.cpp
+        JoinTree.cpp JoinNode.cpp
+        IKKBZ.cpp
+        LinearizedDP.cpp
+        RelationBasic.cpp
+        EdgeInfo.cpp
+        CostIKKBZ.cpp
+        CostCout.cpp)
+qlever_target_link_libraries(joinOrdering)

src/engine/joinOrdering/CostCout.cpp

@@ -0,0 +1,140 @@
+// Copyright 2024, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author:
+// Mahmoud Khalaf (2024-, [email protected])
+
+#include "CostCout.h"
+
+namespace JoinOrdering::Cost {
+
+template <typename N>
+requires RelationAble<N> float selectivity(
+    std::shared_ptr<JoinNode<N>> x, std::shared_ptr<JoinNode<N>> y,
+    const std::map<std::string, std::map<std::string, float>>& selectivities) {
+  if (!x) return 1.0;
+  if (!y) return 1.0;
+
+  if (x->isLeaf() && y->isLeaf())
+  //    return selectivities.at(x.get()->relation.getLabel())
+  //        .at(y.get()->relation.getLabel());
+  {
+    auto z = selectivities.at(x.get()->relation.getLabel());
+    auto kk = y.get()->relation.getLabel();
+    if (!z.contains(kk)) return 1.0;
+    auto zz = z.at(kk);  // TODO: get or default
+    return zz;
+  }
+
+  if (x->isLeaf() && !y->isLeaf())
+    return selectivity(x, y->left, selectivities) *
+           selectivity(x, y->right, selectivities);
+
+  if (!x->isLeaf() && y->isLeaf())
+    return selectivity(y, x->left, selectivities) *
+           selectivity(y, x->right, selectivities);
+
+  return selectivity(x->left, y->left, selectivities) *
+         selectivity(x->left, y->right, selectivities) *
+         selectivity(x->right, y->left, selectivities) *
+         selectivity(x->right, y->right, selectivities);
+}
+
+// assumes independence of the predicates
+// ref: 77/637
+template <typename N>
+requires RelationAble<N> unsigned long long cardinality(
+    std::shared_ptr<JoinNode<N>> n,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities) {
+  if (n == nullptr) return 1;
+
+  // TODO: log missing relation cardinality
+  if (n->isLeaf()) return cardinalities.at(n->relation.getLabel());
+
+  auto l = n->left;
+  auto r = n->right;
+
+  if (l && r)
+    return cardinality(l, cardinalities, selectivities) *
+           cardinality(r, cardinalities, selectivities) *
+           selectivity(l, r, selectivities);
+
+  if (l) return cardinality(n->left, cardinalities, selectivities);
+  //  if (r) return cardinality(n->right, cardinalities, selectivities);
+  return cardinality(n->right, cardinalities, selectivities);
+
+  //  AD_CONTRACT_CHECK("How Did We Get Here?");
+  //  return 0;
+}
+
+template <typename N>
+requires RelationAble<N>
+double Cout(const JoinTree<N>& t, const QueryGraph<N>& q) {
+  //  q.selectivity
+  std::map<std::string, std::map<std::string, float>> qselecm;
+  std::map<std::string, unsigned long long> qcards;
+
+  // FIXME: garbage!
+  // std::map<N, std::map<N, EdgeInfo>>
+  for (auto const& [k, xm] : q.edges_) {
+    auto l = k;
+    qcards[l.getLabel()] = l.getCardinality();
+    for (auto const& [x, xe] : xm) {
+      auto r = x;
+      if (!xe.hidden) {
+        auto s = xe.weight;
+        qselecm[l.getLabel()][r.getLabel()] = s;
+        qselecm[r.getLabel()][l.getLabel()] = s;
+      }
+    }
+  }
+
+  return Cout(t.root, qcards, qselecm);
+}
+
+template <typename N>
+requires RelationAble<N> double Cout(
+    const JoinTree<N>& t,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities) {
+  return Cout(t.root, cardinalities, selectivities);
+}
+
+// ref: 79/637
+template <typename N>
+requires RelationAble<N> double Cout(
+    std::shared_ptr<JoinNode<N>> n,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities) {
+  if (n == nullptr) return 0;  // empty join tree, DP table
+  if (n->isLeaf()) return 0;
+
+  auto l = n->left;
+  auto r = n->right;
+
+  if (l && r)
+    return cardinality(n, cardinalities, selectivities) +
+           Cout(l, cardinalities, selectivities) +
+           Cout(r, cardinalities, selectivities);
+
+  if (l) return Cout(l, cardinalities, selectivities);
+  //  if (r) return Cout(r, cardinalities, selectivities);
+  return Cout(r, cardinalities, selectivities);
+  //  AD_CONTRACT_CHECK("How Did We Get Here?");
+  //  return 0;
+}
+
+template double Cout(const JoinTree<RelationBasic>& t,
+                     const QueryGraph<RelationBasic>& q);
+
+template double Cout(
+    const JoinTree<RelationBasic>& t,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities);
+
+template double Cout(
+    std::shared_ptr<JoinNode<RelationBasic>> n,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities);
+
+}  // namespace JoinOrdering::Cost

src/engine/joinOrdering/CostCout.h

@@ -0,0 +1,74 @@
+// Copyright 2024, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author:
+// Mahmoud Khalaf (2024-, [email protected])
+
+#pragma once
+
+#include <map>
+#include <memory>
+
+#include "JoinNode.h"
+#include "JoinTree.h"
+#include "QueryGraph.h"
+
+namespace JoinOrdering::Cost {
+
+template <typename N>
+requires RelationAble<N>
+double Cout(const JoinTree<N>& t, const QueryGraph<N>& q);
+
+/**
+ *
+ * Basic Cost Function that returns an estimate on how expensive to
+ * evaluate a given JoinTree. low cost implies cheap execution plan.
+ *
+ * ref: 79/637
+ *
+ * // TODO: can be inferred by RelationAble::getCardinality
+ * // TODO: better use some sort of map of unordered pairs since selectivity is
+ * direction-less
+ * // TODO: default to 1.0 when the selectivity between 2 relations is not
+ * defined.
+ *
+ * @tparam N type that satisfies RelationAble concept
+ * @param t Linear JoinTree (left-deep, right-deep, zigzag, ...)
+ * @param cardinalities map of cardinality of each relation in the tree
+ * @param selectivities map of selectivity for each pair of relation in the tree
+ * @return Cost Evaluation for given JoinTree
+ */
+template <typename N>
+requires RelationAble<N> double Cout(
+    const JoinTree<N>& t,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities);
+
+/**
+ *
+ *
+ * @tparam N type that satisfies RelationAble concept
+ * @param r JoinNode that can be inner (join operators) or leaf node (relations)
+ * @param cardinalities map of cardinality of each relation in the tree
+ * @param selectivities map of selectivity for each pair of relation in the tree
+ * @return Cost Evaluation for given JoinNode
+ */
+template <typename N>
+requires RelationAble<N> double Cout(
+    std::shared_ptr<JoinNode<N>> r,
+    const std::map<std::string, unsigned long long>& cardinalities,
+    const std::map<std::string, std::map<std::string, float>>& selectivities);
+
+// template <typename N>
+// requires RelationAble<N> unsigned long long cardinality(
+//     std::shared_ptr<JoinNode<N>> r,
+//     const std::map<std::string, int>& cardinalities,
+//     const std::map<std::string, std::map<std::string, float>>&
+//     selectivities);
+//
+// template <typename N>
+// requires RelationAble<N> float selectivity(
+//     std::shared_ptr<JoinNode<N>> x, std::shared_ptr<JoinNode<N>> y,
+//     const std::map<std::string, std::map<std::string, float>>&
+//     selectivities);
+
+}  // namespace JoinOrdering::Cost

src/engine/joinOrdering/CostIKKBZ.cpp

@@ -0,0 +1,67 @@
+// Copyright 2024, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author:
+// Mahmoud Khalaf (2024-, [email protected])
+
+#include "CostIKKBZ.h"
+
+#include "RelationBasic.h"
+
+namespace JoinOrdering {
+
+template <typename N>
+requires RelationAble<N>
+float CostIKKBZ<N>::C(const QueryGraph<N>& g, std::span<N> seq) {
+  if (seq.empty()) return 0.0f;
+  auto s1 = seq.front();
+  //  auto s2 = seq | std::views::drop(1);
+  auto s2 = seq.subspan(1);
+  return C(g, s1) + T(g, s1) * C(g, s2);  // TODO: might overflow
+}
+
+template <typename N>
+requires RelationAble<N>
+float CostIKKBZ<N>::C(const QueryGraph<N>& g, const N& n) {
+  // return 0 if Ri is root 113/637
+  if (g.root == n) return 0;
+
+  // i.e: regular relation
+  if (!g.is_compound_relation(n)) return T(g, n);
+
+  auto const& [s1, s2] = g.hist.at(n).value();
+  return C(g, s1) + T(g, s1) * C(g, s2);  // TODO: might overflow
+}
+
+template <typename N>
+requires RelationAble<N>
+float CostIKKBZ<N>::T(const QueryGraph<N>& g, const N& n) {
+  // return 0 if Ri is root 113/637
+  if (g.root == n) return 1;
+  return g.selectivity.at(n) * static_cast<float>(n.getCardinality());
+}
+
+template <typename N>
+requires RelationAble<N>
+float CostIKKBZ<N>::rank(const QueryGraph<N>& g, const N& n) {
+  // memorize cost and rank
+  // avoid recomputing for long sequences
+  if (rank_m.contains(n)) return rank_m[n];     // important
+  auto c = C_m.contains(n) ? C_m[n] : C(g, n);  // important
+  auto t = T_m.contains(n) ? T_m[n] : T(g, n);  // maybe not important
+
+  if (c == 0) return 0;
+  auto r = (t - 1) / c;
+  AD_CONTRACT_CHECK(r >= 0 && r <= 1);
+
+  rank_m[n] = r;
+  C_m[n] = c;
+  T_m[n] = t;
+  return r;
+}
+
+template float CostIKKBZ<RelationBasic>::C(const QueryGraph<RelationBasic>& g,
+                                           std::span<RelationBasic> seq);
+
+template float CostIKKBZ<RelationBasic>::rank(
+    const QueryGraph<RelationBasic>& g, const RelationBasic& n);
+}  // namespace JoinOrdering

src/engine/joinOrdering/CostIKKBZ.h

@@ -0,0 +1,72 @@
+// Copyright 2024, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author:
+// Mahmoud Khalaf (2024-, [email protected])
+
+#pragma once
+
+#include <span>
+
+#include "ICostASI.h"
+// #include "util/HashMap.h"
+#include <map>
+
+namespace JoinOrdering {
+
+template <typename N>
+requires RelationAble<N> class CostIKKBZ : public ICostASI<N> {
+ public:
+  //  ad_utility::HashMap<N, float> rank_m;
+  //  ad_utility::HashMap<N, float> C_m;
+  //  ad_utility::HashMap<N, float> T_m;
+
+  std::map<N, float> rank_m;
+  std::map<N, float> C_m;
+  std::map<N, float> T_m;
+
+  float rank(const QueryGraph<N>& g, const N& n);
+
+  /**
+   *
+   * calculate T for an uncompound relation s_i * n_i
+   * (cardinality * selectivity)
+   *
+   *
+   * @param g precedence tree
+   * @param n Relation
+   * @return T(n)
+   */
+  float T(const QueryGraph<N>& g, const N& n);
+  /**
+   *
+   * a join is called increasing if cost > 1
+   * a join is called decreasing if cost < 1
+   *
+   * ref: 113/637
+   *
+   * @param g precedence tree
+   * @param n Relation
+   * @return C(n)
+   */
+  float C(const QueryGraph<N>& g, const N& n);
+
+  /**
+   *
+   * calculate cost for a sequence of relations
+   *
+   *
+   * C(eps) = 0
+   * C(R) = 0 (if R is root)
+   * C(R) = h_i * (n_i)
+   * C(S_1 S_2) = C(S1) + T(S1) * C(S2)
+   *
+   * ref: 113/637
+   *
+   * @param g precedence tree
+   * @param seq sequence of relations (may include compound relations)
+   * @return C(S_1 S_2)
+   */
+  float C(const QueryGraph<N>& g, std::span<N> seq);
+};
+
+}  // namespace JoinOrdering

src/engine/joinOrdering/EdgeInfo.cpp

@@ -0,0 +1,14 @@
+// Copyright 2024, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author:
+// Mahmoud Khalaf (2024-, [email protected])
+
+#include "EdgeInfo.h"
+
+namespace JoinOrdering {
+
+EdgeInfo::EdgeInfo() = default;
+// EdgeInfo::EdgeInfo(Direction dir) : direction(dir) {}
+EdgeInfo::EdgeInfo(Direction dir, float weight)
+    : direction(dir), weight(weight) {}
+}  // namespace JoinOrdering