feat: refactor for general data science

.gitignore

@@ -4,6 +4,7 @@
 Pipfile
 public
 release-notes.md
+typescript*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -170,3 +171,4 @@ mlruns/
 # shell script
 *.out
 *.sh
+.aider*

pyproject.toml

@@ -61,6 +61,10 @@ explicit_package_bases = true
 warn_return_any = true
 warn_unused_ignores = true
 
+[[tool.mypy.overrides]]
+ignore_missing_imports = true
+module = "llama"
+
 [tool.pytest.ini_options]
 addopts = "-l -s --durations=0"
 log_cli = true
@@ -77,7 +81,6 @@ src = ["rdagent"]
 [tool.ruff.lint]
 ignore = [
   # https://docs.astral.sh/ruff/rules/#pydocstyle-d
-  "ANN101",
   "ANN401",
   "D",
   "ERA001",
@@ -88,7 +91,7 @@ ignore = [
   "S101",
   "S301",
   "T20",
-  "TCH003",
+  "TC003",
   "TD",
 ]
 select = ["ALL"]

rdagent/app/data_mining/conf.py

@@ -23,7 +23,7 @@ class MedBasePropSetting(BasePropSetting):
     runner: str = "rdagent.scenarios.data_mining.developer.model_runner.DMModelRunner"
     """Runner class"""
 
-    summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelHypothesisExperiment2Feedback"
+    summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10

rdagent/app/data_science/conf.py

@@ -0,0 +1,49 @@
+from rdagent.app.kaggle.conf import KaggleBasePropSetting
+from rdagent.core.conf import ExtendedSettingsConfigDict
+
+
+class DataScienceBasePropSetting(KaggleBasePropSetting):
+    model_config = ExtendedSettingsConfigDict(env_prefix="DS_", protected_namespaces=())
+
+    # Main components
+    ## Scen
+    scen: str = "rdagent.scenarios.data_science.scen.KaggleScen"
+    """Scenario class for data mining model"""
+
+    ## proposal
+    exp_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen"
+    # exp_gen_init_kwargs: dict = {"max_trace_hist": 3}   # TODO: to be configurable
+
+    # the two below should be used in ExpGen
+    # hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
+    # """Hypothesis generation class"""
+    #
+    # hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
+    # """Hypothesis to experiment class"""
+
+    ## dev/coder
+    data_loader_coder: str = "rdagent.components.coder.data_science.raw_data_loader.DataLoaderCoSTEER"
+    """Data Loader CoSTEER"""
+
+    # feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    # """Feature Coder class"""
+
+    # model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
+    # """Model Feature Selection Coder class"""
+
+    # model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    # """Model Coder class"""
+
+    ## dev/runner
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    ## feedback
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGExperiment2Feedback"
+    """Summarizer class"""
+
+
+DS_RD_SETTING = DataScienceBasePropSetting()

rdagent/app/data_science/debug.py

@@ -0,0 +1,6 @@
+import fire
+
+from rdagent.scenarios.data_science.debug.data import create_debug_data
+
+if __name__ == "__main__":
+    fire.Fire(create_debug_data)

rdagent/app/data_science/loop.py

@@ -0,0 +1,163 @@
+from pathlib import Path
+from typing import Any
+
+import fire
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.data_science.ensemble import EnsembleCoSTEER
+from rdagent.components.coder.data_science.feature import FeatureCoSTEER
+from rdagent.components.coder.data_science.model import ModelCoSTEER
+from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER
+from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import CoderError, RunnerError
+from rdagent.core.proposal import ExperimentFeedback, HypothesisFeedback
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback
+from rdagent.scenarios.data_science.dev.runner import DSRunner
+from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+
+
+class DataScienceRDLoop(RDLoop):
+    skip_loop_error = (CoderError, RunnerError)
+
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        logger.log_object(PROP_SETTING.competition, tag="competition")
+        scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+
+        ### shared components in the workflow  # TODO: check if
+        knowledge_base = (
+            import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+            if PROP_SETTING.knowledge_base != ""
+            else None
+        )
+
+        # 1) task generation from scratch
+        # self.scratch_gen: tuple[HypothesisGen, Hypothesis2Experiment] = DummyHypothesisGen(scen),
+
+        # 2) task generation from a complete solution
+        # self.exp_gen: ExpGen = import_class(PROP_SETTING.exp_gen)(scen)
+        self.exp_gen = DSExpGen(scen)
+        self.data_loader_coder = DataLoaderCoSTEER(scen)
+        self.feature_coder = FeatureCoSTEER(scen)
+        self.model_coder = ModelCoSTEER(scen)
+        self.ensemble_coder = EnsembleCoSTEER(scen)
+        self.workflow_coder = WorkflowCoSTEER(scen)
+
+        self.runner = DSRunner(scen)
+        # self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+        # logger.log_object(self.summarizer, tag="summarizer")
+
+        # self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+        self.trace = DSTrace(scen=scen)
+        self.summarizer = DSExperiment2Feedback(scen)
+        super(RDLoop, self).__init__()
+
+    def direct_exp_gen(self, prev_out: dict[str, Any]):
+        exp = self.exp_gen.gen(self.trace)
+        logger.log_object(exp, tag="direct_exp_gen")
+
+        # FIXME: this is for LLM debug webapp, remove this when the debugging is done.
+        logger.log_object(exp, tag="debug_exp_gen")
+        return exp
+
+    def coding(self, prev_out: dict[str, Any]):
+        exp = prev_out["direct_exp_gen"]
+        for tasks in exp.pending_tasks_list:
+            exp.sub_tasks = tasks
+            if exp.hypothesis.component == "DataLoadSpec":
+                exp = self.data_loader_coder.develop(exp)
+            elif exp.hypothesis.component == "FeatureEng":
+                exp = self.feature_coder.develop(exp)
+            elif exp.hypothesis.component == "Model":
+                exp = self.model_coder.develop(exp)
+            elif exp.hypothesis.component == "Ensemble":
+                exp = self.ensemble_coder.develop(exp)
+            elif exp.hypothesis.component == "Workflow":
+                exp = self.workflow_coder.develop(exp)
+            else:
+                raise NotImplementedError(f"Unsupported component in DataScienceRDLoop: {exp.hypothesis.component}")
+            exp.sub_tasks = []
+        logger.log_object(exp, tag="coding")
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        exp: DSExperiment = prev_out["coding"]
+        if exp.next_component_required() is None:
+            new_exp = self.runner.run(exp)
+            logger.log_object(new_exp, tag="running")
+            return new_exp
+        else:
+            return exp
+
+    def feedback(self, prev_out: dict[str, Any]) -> ExperimentFeedback:
+        exp: DSExperiment = prev_out["running"]
+        if exp.next_component_required() is None:
+            feedback = self.summarizer.generate_feedback(exp, self.trace)
+        else:
+            feedback = ExperimentFeedback(
+                reason=f"{exp.hypothesis.component} is completed.",
+                decision=True,
+            )
+        logger.log_object(feedback, tag="feedback")
+        return feedback
+
+    def record(self, prev_out: dict[str, Any]):
+        e = prev_out.get(self.EXCEPTION_KEY, None)
+        if e is None:
+            self.trace.hist.append((prev_out["running"], prev_out["feedback"]))
+        else:
+            self.trace.hist.append(
+                (
+                    prev_out["direct_exp_gen"] if isinstance(e, CoderError) else prev_out["coding"],
+                    ExperimentFeedback.from_exception(e),
+                )
+            )
+        logger.log_object(self.trace, tag="trace")
+        logger.log_object(self.trace.sota_experiment(), tag="SOTA experiment")
+
+
+def main(path=None, step_n=None, competition="bms-molecular-translation"):
+    """
+
+    Parameters
+    ----------
+    path :
+        path like `$LOG_PATH/__session__/1/0_propose`. It indicates that we restore the state that after finish the step 0 in loop1
+    step_n :
+        How many steps to run; if None, it will run forever until error or KeyboardInterrupt
+    competition :
+
+
+    Auto R&D Evolving loop for models in a Kaggle scenario.
+    You can continue running session by
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/data_science/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+    """
+    if competition is not None:
+        DS_RD_SETTING.competition = competition
+
+    if DS_RD_SETTING.competition:
+        if DS_RD_SETTING.scen.endswith("KaggleScen"):
+            download_data(competition=DS_RD_SETTING.competition, settings=DS_RD_SETTING)
+        else:
+            if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists():
+                logger.error(f"Please prepare data for competition {competition} first.")
+                return
+    else:
+        logger.error("Please specify competition name.")
+    if path is None:
+        kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
+    else:
+        kaggle_loop = DataScienceRDLoop.load(path)
+    kaggle_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/kaggle/conf.py

@@ -1,8 +1,7 @@
-from rdagent.components.workflow.conf import BasePropSetting
-from rdagent.core.conf import ExtendedSettingsConfigDict
+from rdagent.core.conf import ExtendedBaseSettings, ExtendedSettingsConfigDict
 
 
-class KaggleBasePropSetting(BasePropSetting):
+class KaggleBasePropSetting(ExtendedBaseSettings):
     model_config = ExtendedSettingsConfigDict(env_prefix="KG_", protected_namespaces=())
 
     # 1) overriding the default
@@ -30,7 +29,7 @@ class KaggleBasePropSetting(BasePropSetting):
     model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
     """Model Runner class"""
 
-    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback"
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10
@@ -45,12 +44,21 @@ class KaggleBasePropSetting(BasePropSetting):
     local_data_path: str = ""
     """Folder storing Kaggle competition data"""
 
+    if_using_mle_data: bool = False
+    auto_submit: bool = False
+    """Automatically upload and submit each experiment result to Kaggle platform"""
+    # Conditionally set the knowledge_base based on the use of graph RAG
+    knowledge_base: str = ""
+    """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""
     if_action_choosing_based_on_UCB: bool = False
     """Enable decision mechanism based on UCB algorithm"""
 
     domain_knowledge_path: str = "/data/userdata/share/kaggle/domain_knowledge"
     """Folder storing domain knowledge files in .case format"""
 
+    knowledge_base_path: str = "kg_graph.pkl"
+    """Advanced version of graph-based RAG"""
+
     rag_path: str = "git_ignore_folder/kaggle_vector_base.pkl"
     """Base version of vector-based RAG"""
 
@@ -60,20 +68,8 @@ class KaggleBasePropSetting(BasePropSetting):
     if_using_graph_rag: bool = False
     """Enable advanced graph-based RAG"""
 
-    # Conditionally set the knowledge_base based on the use of graph RAG
-    knowledge_base: str = ""
-    """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""
-
-    knowledge_base_path: str = "kg_graph.pkl"
-    """Advanced version of graph-based RAG"""
-
-    auto_submit: bool = False
-    """Automatically upload and submit each experiment result to Kaggle platform"""
-
     mini_case: bool = False
     """Enable mini-case study for experiments"""
 
-    if_using_mle_data: bool = False
-
 
 KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()