Initial commit

CL_Benchmark/BoolQA/BoolQA/labels.json

@@ -0,0 +1 @@
+["True", "False"]

CL_Benchmark/COPA/COPA/labels.json

@@ -0,0 +1 @@
+["A", "B"]

CL_Benchmark/MultiRC/MultiRC/labels.json

@@ -0,0 +1 @@
+["False", "True"]

CL_Benchmark/NLI/CB/labels.json

@@ -0,0 +1 @@
+["entailment", "contradiction", "neutral"]

CL_Benchmark/NLI/MNLI/labels.json

@@ -0,0 +1 @@
+["neutral", "entailment", "contradiction"]

CL_Benchmark/NLI/RTE/labels.json

@@ -0,0 +1 @@
+["contradiction", "entailment"]

CL_Benchmark/QQP/QQP/labels.json

@@ -0,0 +1 @@
+["False", "True"]

CL_Benchmark/SC/IMDB/labels.json

@@ -0,0 +1 @@
+["Good", "Bad"]

CL_Benchmark/SC/SST-2/labels.json

@@ -0,0 +1 @@
+["Good", "Bad"]

CL_Benchmark/SC/amazon/labels.json

@@ -0,0 +1 @@
+["very negative", "negative", "neutral", "positive", "very positive"]

CL_Benchmark/SC/yelp/labels.json

@@ -0,0 +1 @@
+["very negative", "negative", "neutral", "positive", "very positive"]

CL_Benchmark/TC/agnews/labels.json

@@ -0,0 +1 @@
+["World", "Sports", "Business", "Science or Technology"]

CL_Benchmark/TC/dbpedia/labels.json

@@ -0,0 +1 @@
+["Company", "Educational Institution", "Artist", "Athlete", "Office Holder", "Mean of Transportation", "Building", "Natural Place", "Village", "Animal", "Plant", "Album", "Film", "Written Work"]

CL_Benchmark/TC/yahoo/labels.json

@@ -0,0 +1 @@
+["Society & Culture", "Science & Mathematics", "Health", "Education & Reference", "Computers & Internet", "Sports", "Business & Finance", "Entertainment & Music", "Family & Relationships", "Politics & Government"]

CL_Benchmark/WiC/WiC/labels.json

@@ -0,0 +1 @@
+["True", "False"]

README.md

@@ -1,7 +1,3 @@
-# Is Parameter Collision Hindering Continual Learning in LLMs?
-Code for the paper "Is Parameter Collision Hindering Continual Learning in LLMs?", exploring novel approaches to address parameter collision issues in large language models for continual learning.
-
-
 # Sn-LoRA
 
 - This repo releases our implementation for the N-LoRA model.

configs/ds_configs/eval.config

@@ -0,0 +1,33 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 1e5,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

configs/ds_configs/stage0.config

@@ -0,0 +1,38 @@
+{
+    "bfloat16": {
+        "enabled": "auto"
+    },
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "zero_optimization": {
+        "stage": 0
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "steps_per_print": 1e5
+}

configs/ds_configs/stage1.config

@@ -0,0 +1,36 @@
+{
+    "bfloat16": {
+        "enabled": "auto"
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "zero_optimization": {
+        "stage": 1,
+        "allgather_partitions": true,
+        "allgather_bucket_size": 2e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 2e8,
+        "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "steps_per_print": 1e5
+}

configs/ds_configs/stage2.config

@@ -0,0 +1,48 @@
+{
+    "bfloat16": {
+        "enabled": "auto"
+    },
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "zero_optimization": {
+        "stage": 2,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "allgather_partitions": true,
+        "allgather_bucket_size": 2e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 2e8,
+        "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "steps_per_print": 1e5
+}

configs/ds_configs/stage2_llama.config

@@ -0,0 +1,48 @@
+{
+    "bfloat16": {
+        "enabled": "true"
+    },
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "zero_optimization": {
+        "stage": 2,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "allgather_partitions": true,
+        "allgather_bucket_size": 2e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 2e8,
+        "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "steps_per_print": 1e5
+}

configs/ds_configs/stage2_without_offload.config

@@ -0,0 +1,44 @@
+{
+    "bfloat16": {
+        "enabled": "auto"
+    },
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "zero_optimization": {
+        "stage": 2,
+        "allgather_partitions": true,
+        "allgather_bucket_size": 2e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 2e8,
+        "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "steps_per_print": 1e5
+}

configs/ds_configs/stage3.config

@@ -0,0 +1,56 @@
+{
+    "bfloat16": {
+        "enabled": false
+    },
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    },
+    "zero_optimization": {
+        "stage": 3,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "offload_param": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto",
+        "stage3_prefetch_bucket_size": "auto",
+        "stage3_param_persistence_threshold": "auto",
+        "stage3_max_live_parameters": 1e9,
+        "stage3_max_reuse_distance": 1e9,
+        "stage3_gather_fp16_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 1e5,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

configs/instruction_config.json

@@ -0,0 +1,23 @@
+{
+  "NLI": [
+    {"instruction_type": "zero-shot", "instruction": "What is the logical relationship between the \"sentence 1\" and the \"sentence 2\"? Choose one from the option.\n"}
+  ],
+  "QQP": [
+    {"instruction_type": "zero-shot", "instruction": "Whether the \"first sentence\" and the \"second sentence\" have the same meaning? Choose one from the option.\n"}
+  ],
+  "SC": [
+    {"instruction_type": "zero-shot", "instruction": "What is the sentiment of the following paragraph? Choose one from the option.\n"}
+  ],
+  "TC": [
+    {"instruction_type": "zero-shot", "instruction": "What is the topic of the following paragraph? Choose one from the option.\n"}
+  ],
+  "BoolQA":[
+    {"instruction_type": "zero-shot", "instruction": "According to the following passage, is the question true or false? Choose one from the option.\n"}
+  ],
+  "MultiRC":[
+    {"instruction_type": "zero-shot", "instruction": "According to the following passage and question, is the candidate answer true or false? Choose one from the option.\n"}
+  ],
+  "WiC":[
+    {"instruction_type": "zero-shot", "instruction": "Given a word and two sentences, whether the word is used with the same sense in both sentence? Choose one from the option.\n"}
+  ]
+}