FastADC implementation

examples/basic/README.md

@@ -20,6 +20,7 @@ These scenarios showcase a single pattern by discussing its definition and provi
 + [mining_set_od_1.py](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/mining_set_od_1.py) — a scenario showing how to discover order dependencies based on set axiomatization, part 1.
 + [mining_set_od_2.py](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/mining_set_od_2.py) — a scenario showing how to discover order dependencies based on set axiomatization, part 2.
 + [mining_ucc.py](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/mining_ucc.py) — a scenario showing how to discover exact unique column combinations.
++ [mining_adc.py](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/mining_adc.py) — a scenario showing how to discover an approximate denial constraints.
 + [verifying_aucc.py](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/verifying_aucc.py) — a scenario showing how to verify an approximate unique column combination.
 + [verifying_fd_afd.py](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/verifying_fd_afd.py) — a scenario showing how to verify exact and approximate functional dependencies.
 + [verifying_gfd](https://github.com/Desbordante/desbordante-core/tree/main/examples/basic/verifying_gfd) — a scenario showing how to verify a graph functional dependency.

examples/basic/mining_adc.py

@@ -0,0 +1,107 @@
+import desbordante as db
+import pandas as pd
+
+RED = '\033[31m'
+YELLOW = '\033[33m'
+GREEN = '\033[32m'
+CYAN = '\033[1m\033[36m'
+ENDC = '\033[0m'
+
+TABLE_1 = "examples/datasets/taxes.csv"
+TABLE_2 = "examples/datasets/taxes_2.csv"
+
+def print_table(filename: str, title: str = "") -> None:
+    if title:
+        print(f"{title}")
+    data = pd.read_csv(filename, header=0)
+    print(data, end="\n\n")
+
+def main():
+    print(f"""{YELLOW}This file demonstrates how to discover Approximate Denial Constraints (ADCs){ENDC}.
+
+DC {CYAN}φ{ENDC} is a conjunction of predicates of the following form:
+{CYAN}∀s, t ∈ R, s ≠ t: ¬(p_1 ∧ . . . ∧ p_m){ENDC}
+
+DCs involve comparisons between pairs of rows within a dataset.
+A typical DC example, derived from a Functional Dependency such as {CYAN}A -> B{ENDC},
+is expressed as: {CYAN}∀s, t ∈ R, s ≠ t, ¬(t.A == s.A and t.B ≠ s.B){ENDC}.
+This denotes that for any pair of rows in the relation, it should not be the case
+that while the values in column A are equal, the values in column B are unequal.
+
+{YELLOW}Let's begin by looking at TABLE_1:{ENDC}
+""")
+
+    print_table(TABLE_1, "TABLE_1 (examples/datasets/taxes.csv):")
+
+    print(f"""- The 'evidence_threshold' parameter specifies the fraction of row pairs that must satisfy the DC for it to be considered valid.
+  * evidence_threshold = 0 => exact DC mining, where all pairs must satisfy.
+  * evidence_threshold < 1.0 => approximate DC mining, which allows a fraction of violations.
+- The 'shard_length' parameter splits the dataset into row "shards" for parallelization. Here,
+  we set it to 0 so all rows are processed in one shard.
+""")
+
+    print(f"""{YELLOW}Mining exact DCs (evidence_threshold=0) on TABLE_1{ENDC}""")
+
+    algo = db.dc.algorithms.Default()
+    algo.load_data(table=(TABLE_1, ',', True))
+    algo.execute(evidence_threshold=0, shard_length=0)
+    dcs_table1_exact = algo.get_dcs()
+
+    print(f"{YELLOW}Discovered DCs:{ENDC}")
+    for dc in dcs_table1_exact:
+        print(f"  {CYAN}{dc}{ENDC}")
+    print()
+
+    print(f"""Note the following Denial Constraint: {GREEN}¬{{ t.State == s.State ∧ t.Salary <= s.Salary ∧ t.FedTaxRate >= s.FedTaxRate }}{ENDC}.
+It tells us that for all people in the same state the person with a higher salary has a higher tax rate.
+""")
+
+    print(f"""{YELLOW}Now let's lower the evidence_threshold to 0.5 on TABLE_1{ENDC}
+This means the DC only needs to hold for at least half of the row pairs, thus allowing more approximate constraints.
+""")
+
+    print(f"""{YELLOW}Mining ADCs (evidence_threshold=0.5) on TABLE_1{ENDC}""")
+
+    algo = db.dc.algorithms.Default()
+    algo.load_data(table=(TABLE_1, ',', True))
+    algo.execute(evidence_threshold=0.5, shard_length=0)
+    dcs_table1_approx = algo.get_dcs()
+
+    print(f"{YELLOW}Discovered ADCs:{ENDC}")
+    for dc in dcs_table1_approx:
+        print(f"  {CYAN}{dc}{ENDC}")
+    print()
+
+    print(f"""{YELLOW}Let's take a look at TABLE_2:{ENDC}""")
+
+    print_table(TABLE_2, "TABLE_2 (examples/datasets/taxes_2.csv):")
+
+
+    print(f"""TABLE_2 is almost the same as TABLE_1, but we added a new record for Texas:
+{GREEN}(State=Texas, Salary=5000, FedTaxRate=0.05){ENDC}.
+
+This additional record violates one of the DCs that was valid in TABLE_1,
+because it introduces a new pair of rows that breaks the constraint
+""")
+
+    print(f"""{YELLOW}Mining exact DCs (evidence_threshold=0) on TABLE_2{ENDC}""")
+
+    algo = db.dc.algorithms.Default()
+    algo.load_data(table=(TABLE_2, ',', True))
+    algo.execute(evidence_threshold=0, shard_length=0)
+    dcs_table2_exact = algo.get_dcs()
+
+    print(f"{YELLOW}Discovered DCs:{ENDC}")
+    for dc in dcs_table2_exact:
+        print(f"  {CYAN}{dc}{ENDC}")
+    print()
+
+    print(f"""Now we can see that the same DC we examined on the previous dataset doesn't hold on the new one.
+The thing is that for the last record {GREEN}(Texas, 5000, 0.05){ENDC} there are people in Texas with a lower salary
+but higher tax rate. Pairs of records like this that contradict a DC are called violations.
+In this case the following pairs are violations: {RED}(6, 9), (7, 9), (8, 9){ENDC}, where each number is an index of a record.
+""")
+
+if __name__ == "__main__":
+    main()
+

src/core/algorithms/dc/FastADC/fastadc.cpp

@@ -0,0 +1,147 @@
+#include "algorithms/dc/FastADC/fastadc.h"
+
+#include <easylogging++.h>
+
+#include "config/names_and_descriptions.h"
+#include "config/option.h"
+#include "config/option_using.h"
+#include "config/tabular_data/input_table/option.h"
+#include "dc/FastADC/model/pli_shard.h"
+#include "dc/FastADC/util/approximate_evidence_inverter.h"
+#include "dc/FastADC/util/evidence_aux_structures_builder.h"
+#include "dc/FastADC/util/evidence_set_builder.h"
+#include "dc/FastADC/util/predicate_builder.h"
+#include "descriptions.h"
+#include "names.h"
+
+namespace algos::dc {
+
+FastADC::FastADC() : Algorithm({}) {
+    RegisterOptions();
+    MakeOptionsAvailable({config::kTableOpt.GetName()});
+}
+
+void FastADC::RegisterOptions() {
+    DESBORDANTE_OPTION_USING;
+
+    config::InputTable default_table;
+
+    RegisterOption(config::kTableOpt(&input_table_));
+    RegisterOption(Option{&shard_length_, kShardLength, kDShardLength, 350U});
+    RegisterOption(Option{&allow_cross_columns_, kAllowCrossColumns, kDAllowCrossColumns, true});
+    RegisterOption(Option{&minimum_shared_value_, kMinimumSharedValue, kDMinimumSharedValue, 0.3});
+    RegisterOption(
+            Option{&comparable_threshold_, kComparableThreshold, kDComparableThreshold, 0.1});
+    RegisterOption(Option{&evidence_threshold_, kEvidenceThreshold, kDEvidenceThreshold, 0.01});
+}
+
+void FastADC::MakeExecuteOptsAvailable() {
+    using namespace config::names;
+
+    MakeOptionsAvailable({kShardLength, kAllowCrossColumns, kMinimumSharedValue,
+                          kComparableThreshold, kEvidenceThreshold});
+}
+
+void FastADC::LoadDataInternal() {
+    typed_relation_ = model::ColumnLayoutTypedRelationData::CreateFrom(
+            *input_table_, true, true);  // kMixed type will be treated as a string type
+
+    if (typed_relation_->GetColumnData().empty()) {
+        throw std::runtime_error("Got an empty dataset: DC mining is meaningless.");
+    }
+}
+
+void FastADC::SetLimits() {
+    unsigned all_rows_num = typed_relation_->GetNumRows();
+
+    if (shard_length_ > all_rows_num) {
+        throw std::invalid_argument(
+                "'shard_length' (" + std::to_string(shard_length_) +
+                ") must be less or equal to the number of rows in the table (total "
+                "rows: " +
+                std::to_string(all_rows_num) + ")");
+    }
+    if (shard_length_ == 0) shard_length_ = all_rows_num;
+}
+
+void FastADC::CheckTypes() {
+    model::ColumnIndex columns_num = typed_relation_->GetNumColumns();
+    unsigned rows_num = typed_relation_->GetNumRows();
+
+    for (model::ColumnIndex column_index = 0; column_index < columns_num; column_index++) {
+        model::TypedColumnData const& column = typed_relation_->GetColumnData(column_index);
+        model::TypeId type_id = column.GetTypeId();
+
+        if (type_id == +model::TypeId::kMixed) {
+            LOG(WARNING) << "Column with index \"" + std::to_string(column_index) +
+                                    "\" contains values of different types. Those values will be "
+                                    "treated as strings.";
+        } else if (!column.IsNumeric() && type_id != +model::TypeId::kString) {
+            throw std::invalid_argument(
+                    "Column with index \"" + std::to_string(column_index) +
+                    "\" is of unsupported type. Only numeric and string types are supported.");
+        }
+
+        for (std::size_t row_index = 0; row_index < rows_num; row_index++) {
+            if (column.IsNull(row_index)) {
+                throw std::runtime_error("Some of the value coordinates are nulls.");
+            }
+            if (column.IsEmpty(row_index)) {
+                throw std::runtime_error("Some of the value coordinates are empty.");
+            }
+        }
+    }
+}
+
+void FastADC::PrintResults() {
+    LOG(INFO) << "Total denial constraints: " << dcs_.TotalDCSize();
+    LOG(INFO) << "Minimal denial constraints: " << dcs_.MinDCSize();
+    LOG(DEBUG) << dcs_.ToString();
+}
+
+unsigned long long FastADC::ExecuteInternal() {
+    auto const start_time = std::chrono::system_clock::now();
+    LOG(DEBUG) << "Start";
+
+    SetLimits();
+    CheckTypes();
+
+    PredicateBuilder predicate_builder(&pred_provider_, &pred_index_provider_, allow_cross_columns_,
+                                       minimum_shared_value_, comparable_threshold_);
+    predicate_builder.BuildPredicateSpace(typed_relation_->GetColumnData());
+
+    PliShardBuilder pli_shard_builder(&int_prov_, &double_prov_, &string_prov_, shard_length_);
+    pli_shard_builder.BuildPliShards(typed_relation_->GetColumnData());
+
+    EvidenceAuxStructuresBuilder evidence_aux_structures_builder(predicate_builder);
+    evidence_aux_structures_builder.BuildAll();
+
+    EvidenceSetBuilder evidence_set_builder(pli_shard_builder.pli_shards,
+                                            evidence_aux_structures_builder.GetPredicatePacks());
+    evidence_set_builder.BuildEvidenceSet(evidence_aux_structures_builder.GetCorrectionMap(),
+                                          evidence_aux_structures_builder.GetCardinalityMask());
+
+    LOG(INFO) << "Built evidence set";
+    auto elapsed_milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(
+            std::chrono::system_clock::now() - start_time);
+    LOG(DEBUG) << "Current time: " << elapsed_milliseconds.count();
+
+    ApproxEvidenceInverter dcbuilder(predicate_builder, evidence_threshold_,
+                                     std::move(evidence_set_builder.evidence_set));
+
+    dcs_ = dcbuilder.BuildDenialConstraints();
+
+    PrintResults();
+
+    elapsed_milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(
+            std::chrono::system_clock::now() - start_time);
+    LOG(INFO) << "Algorithm time: " << elapsed_milliseconds.count();
+    return elapsed_milliseconds.count();
+}
+
+// TODO: mb make this a list?
+std::vector<DenialConstraint> const& FastADC::GetDCs() const {
+    return dcs_.GetResult();
+}
+
+}  // namespace algos::dc

src/core/algorithms/dc/FastADC/fastadc.h

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "algorithm.h"
+#include "dc/FastADC/providers/predicate_provider.h"
+#include "dc/FastADC/util/denial_constraint_set.h"
+#include "model/denial_constraint.h"
+#include "table/column_layout_typed_relation_data.h"
+#include "tabular_data/input_table_type.h"
+
+namespace algos::dc {
+
+using namespace fastadc;
+
+class FastADC : public Algorithm {
+private:
+    unsigned shard_length_;
+    bool allow_cross_columns_;
+    double minimum_shared_value_;
+    double comparable_threshold_;
+    double evidence_threshold_;
+
+    config::InputTable input_table_;
+    std::unique_ptr<model::ColumnLayoutTypedRelationData> typed_relation_;
+
+    PredicateIndexProvider pred_index_provider_;
+    PredicateProvider pred_provider_;
+    IntIndexProvider int_prov_;
+    DoubleIndexProvider double_prov_;
+    StringIndexProvider string_prov_;
+    DenialConstraintSet dcs_;
+
+    void MakeExecuteOptsAvailable() override;
+    void LoadDataInternal() override;
+
+    void SetLimits();
+    void CheckTypes();
+    void PrintResults();
+
+    void ResetState() final {
+        pred_index_provider_.Clear();
+        pred_provider_.Clear();
+        int_prov_.Clear();
+        double_prov_.Clear();
+        string_prov_.Clear();
+        dcs_.Clear();
+    }
+
+    unsigned long long ExecuteInternal() final;
+
+    void RegisterOptions();
+
+public:
+    FastADC();
+
+    std::vector<DenialConstraint> const& GetDCs() const;
+};
+
+}  // namespace algos::dc

src/core/algorithms/dc/FastADC/misc/misc.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "model/table/typed_column_data.h"
+
+namespace algos::fastadc {
+
+namespace details {
+// Helper to trigger a compile-time error for unsupported types
+template <typename T>
+struct DependentFalse : std::false_type {};
+}  // namespace details
+
+// TODO: look at performance, is returning by const reference here beneficial?
+template <typename T>
+[[nodiscard]] T GetValue(model::TypedColumnData const& column, size_t row) {
+    model::Type const& type = column.GetType();
+
+    if (!column.IsNullOrEmpty(row)) {
+        return type.GetValue<T>(column.GetValue(row));
+    }
+
+    /*
+     * Mimicking the Java behavior:
+     * https://github.com/RangerShaw/FastADC/blob/master/src/main/java/de/metanome/algorithms/dcfinder/input/Column.java#L71
+     *
+     * public Long getLong(int line) {
+     *     return values.get(line).isEmpty() ? Long.MIN_VALUE :
+     *             Long.parseLong(values.get(line));
+     * }
+     *
+     * public Double getDouble(int line) {
+     *     return values.get(line).isEmpty() ? Double.MIN_VALUE :
+     *            Double.parseDouble(values.get(line));
+     * }
+     *
+     * public String getString(int line) {
+     *     return values.get(line) == null ? "" : values.get(line);
+     * }
+     */
+    if constexpr (std::is_same_v<T, std::string>) {
+        return "";
+    } else if constexpr (std::is_same_v<T, int64_t>) {
+        return std::numeric_limits<int64_t>::min();
+    } else if constexpr (std::is_same_v<T, double>) {
+        return std::numeric_limits<double>::lowest();
+    } else {
+        static_assert(details::DependentFalse<T>::value,
+                      "FastADC algorithm supports only int64_t, string, or double as column types. "
+                      "This function should not be called with other types.");
+    }
+}
+
+}  // namespace algos::fastadc