diff --git a/5.15/0001-NEST.patch b/5.15/0001-NEST.patch index 676830ac..f9a71421 100644 --- a/5.15/0001-NEST.patch +++ b/5.15/0001-NEST.patch @@ -1,6 +1,6 @@ -From 89923bf3230fb3ed867afc0a442eef69252044e9 Mon Sep 17 00:00:00 2001 +From e87fabf12ab7b61d05718076efd69a49433da2b4 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Mon, 19 Sep 2022 18:16:16 +0200 +Date: Wed, 12 Oct 2022 00:05:59 +0200 Subject: [PATCH] NEST Signed-off-by: Peter Jung @@ -9,10 +9,10 @@ Signed-off-by: Peter Jung include/linux/sched.h | 14 ++ include/linux/sched/topology.h | 2 + kernel/sched/core.c | 86 +++++++++- - kernel/sched/fair.c | 297 ++++++++++++++++++++++++++++++++- + kernel/sched/fair.c | 299 ++++++++++++++++++++++++++++++++- kernel/sched/idle.c | 20 +++ kernel/sched/sched.h | 49 ++++++ - 7 files changed, 455 insertions(+), 15 deletions(-) + 7 files changed, 457 insertions(+), 15 deletions(-) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 85f6e242b6b4..8520b84d2046 100644 @@ -73,7 +73,7 @@ index 63a04a65e310..79bf1f6c7b23 100644 struct sched_domain { diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 85be684687b0..0f0ebebafe3b 100644 +index 85be684687b0..fb7840a27cf7 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2495,6 +2495,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 @@ -91,7 +91,7 @@ index 85be684687b0..0f0ebebafe3b 100644 - if (unlikely(!is_cpu_allowed(p, cpu))) + if (unlikely(!is_cpu_allowed(p, cpu))) { + if (p->use_expand_mask) -+ atomic_set(&cpu_rq(cpu)->taken,0); ++ atomic_set(&cpu_rq(p->thread_info.cpu)->taken,0); cpu = select_fallback_rq(task_cpu(p), p); + } @@ -156,7 +156,7 @@ index 85be684687b0..0f0ebebafe3b 100644 post_init_entity_util_avg(p); + if (p->use_expand_mask) -+ atomic_set(&cpu_rq(p->cpu)->taken,0); ++ atomic_set(&cpu_rq(p->thread_info.cpu)->taken,0); activate_task(rq, p, ENQUEUE_NOCLOCK); trace_sched_wakeup_new(p); check_preempt_curr(rq, p, WF_FORK); @@ -269,7 +269,7 @@ index 85be684687b0..0f0ebebafe3b 100644 rq_attach_root(rq, &def_root_domain); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index a853e4e9e3c3..0a61e53fc874 100644 +index a853e4e9e3c3..9857c18956b6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3378,6 +3378,61 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) @@ -446,11 +446,12 @@ index a853e4e9e3c3..0a61e53fc874 100644 /* * On asymmetric system, update task utilization because we will check -@@ -6513,7 +6619,39 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) +@@ -6513,7 +6619,41 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if ((unsigned)i < nr_cpumask_bits) return i; - return target; ++ if (!sd->parent) goto abort; + group0 = group = sd->parent->groups; + do { + struct cpumask *m = sched_group_span(group); @@ -458,7 +459,7 @@ index a853e4e9e3c3..0a61e53fc874 100644 + if (cpumask_test_cpu(otarget, m)) + goto next; + -+ sd1 = rcu_dereference(per_cpu(sd_llc, group_first_cpu(group))); ++ sd1 = rcu_dereference(per_cpu(sd_llc, cpumask_first(sched_group_span(group)))); + if (!sd1 || !READ_ONCE(sd1->shared->has_idle_threads)) + goto next; + @@ -483,11 +484,12 @@ index a853e4e9e3c3..0a61e53fc874 100644 + } while (group != group0); + + ++abort: + return otarget; } /** -@@ -6940,6 +7078,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) +@@ -6940,6 +7080,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) int cpu = smp_processor_id(); int new_cpu = prev_cpu; int want_affine = 0; @@ -495,7 +497,7 @@ index a853e4e9e3c3..0a61e53fc874 100644 /* SD_flags and WF_flags share the first nibble */ int sd_flag = wake_flags & 0xF; -@@ -6961,6 +7100,144 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) +@@ -6961,6 +7102,144 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) } rcu_read_lock(); @@ -640,7 +642,7 @@ index a853e4e9e3c3..0a61e53fc874 100644 for_each_domain(cpu, tmp) { /* * If both 'cpu' and 'prev_cpu' are part of this domain, -@@ -6988,6 +7265,14 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) +@@ -6988,6 +7267,14 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) /* Fast path */ new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); } @@ -656,7 +658,7 @@ index a853e4e9e3c3..0a61e53fc874 100644 return new_cpu; diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index d17b0a5ce6ac..22dda7a58481 100644 +index 499a3e286cd0..cf6a3ceb4c79 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -262,6 +262,26 @@ static void do_idle(void) @@ -754,5 +756,5 @@ index e49902898253..ee9a744377c0 100644 static inline struct task_struct *task_of(struct sched_entity *se) { -- -2.37.3 +2.38.0 diff --git a/6.0/0001-cachy.patch b/6.0/0001-cachy.patch index a601f3ef..3f0e7ef2 100644 --- a/6.0/0001-cachy.patch +++ b/6.0/0001-cachy.patch @@ -1,7 +1,7 @@ -From 64d876548b8197ac91f7b62508d5fc61b8db6ec1 Mon Sep 17 00:00:00 2001 +From a8998dccf3f62a61e162f1278a742e49bf95df8b Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Mon, 10 Oct 2022 18:33:24 +0200 -Subject: [PATCH] cachy +Date: Mon, 10 Oct 2022 13:16:38 +0200 +Subject: [PATCH 01/17] cachy Signed-off-by: Peter Jung --- @@ -30,7 +30,7 @@ Signed-off-by: Peter Jung block/bfq-iosched.c | 6 + block/elevator.c | 7 +- drivers/acpi/cppc_acpi.c | 128 ++- - drivers/cpufreq/amd-pstate.c | 988 +++++++++++++++++++- + drivers/cpufreq/amd-pstate.c | 989 +++++++++++++++++++- drivers/cpufreq/bmips-cpufreq.c | 10 +- drivers/cpufreq/cpufreq-dt-platdev.c | 1 + drivers/cpufreq/cpufreq.c | 2 + @@ -71,7 +71,7 @@ Signed-off-by: Peter Jung mm/vmscan.c | 4 + scripts/Makefile.lib | 13 +- scripts/Makefile.modinst | 7 +- - 66 files changed, 2290 insertions(+), 294 deletions(-) + 66 files changed, 2291 insertions(+), 294 deletions(-) create mode 100644 arch/x86/Makefile.postlink diff --git a/.gitignore b/.gitignore @@ -1150,7 +1150,7 @@ index 1e15a9f25ae9..9c28f47e1b1e 100644 up_write(&pcc_ss_data->pcc_lock); return ret; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c -index 9ac75c1cde9c..bdb44a08771c 100644 +index 9ac75c1cde9c..810aea47645e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -46,8 +46,8 @@ @@ -1164,21 +1164,22 @@ index 9ac75c1cde9c..bdb44a08771c 100644 /* * TODO: We need more time to fine tune processors with shared memory solution -@@ -63,7 +63,12 @@ module_param(shared_mem, bool, 0444); +@@ -63,7 +63,13 @@ module_param(shared_mem, bool, 0444); MODULE_PARM_DESC(shared_mem, "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); -static struct cpufreq_driver amd_pstate_driver; -+static bool epp = false; -+module_param(epp, bool, 0444); -+MODULE_PARM_DESC(epp, "Enable energy performance preference (EPP) control"); ++static bool epp_enabled = false; ++module_param(epp_enabled, bool, 0444); ++MODULE_PARM_DESC(epp_enabled, ++ "Enable energy performance preference (EPP) control"); + +static struct cpufreq_driver *default_pstate_driver; +static struct amd_cpudata **all_cpu_data; /** * struct amd_aperf_mperf -@@ -75,6 +80,7 @@ struct amd_aperf_mperf { +@@ -75,6 +81,7 @@ struct amd_aperf_mperf { u64 aperf; u64 mperf; u64 tsc; @@ -1186,7 +1187,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 }; /** -@@ -97,6 +103,19 @@ struct amd_aperf_mperf { +@@ -97,6 +104,19 @@ struct amd_aperf_mperf { * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value * @boost_supported: check whether the Processor or SBIOS supports boost mode @@ -1206,7 +1207,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. -@@ -120,10 +139,203 @@ struct amd_cpudata { +@@ -120,10 +140,203 @@ struct amd_cpudata { struct amd_aperf_mperf cur; struct amd_aperf_mperf prev; @@ -1411,7 +1412,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 static inline int pstate_enable(bool enable) { return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); -@@ -131,12 +343,25 @@ static inline int pstate_enable(bool enable) +@@ -131,12 +344,25 @@ static inline int pstate_enable(bool enable) static int cppc_enable(bool enable) { @@ -1422,7 +1423,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 ret = cppc_set_enable(cpu, enable); if (ret) return ret; -+ if (epp) { ++ if (epp_enabled) { + /* Enable autonomous mode for EPP */ + ret = cppc_set_auto_epp(cpu, enable); + if (ret) @@ -1437,7 +1438,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 } return ret; -@@ -152,6 +377,7 @@ static inline int amd_pstate_enable(bool enable) +@@ -152,6 +378,7 @@ static inline int amd_pstate_enable(bool enable) static int pstate_init_perf(struct amd_cpudata *cpudata) { u64 cap1; @@ -1445,7 +1446,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); -@@ -163,7 +389,11 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) +@@ -163,7 +390,11 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) * * CPPC entry doesn't indicate the highest performance in some ASICs. */ @@ -1458,7 +1459,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); -@@ -175,12 +405,17 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) +@@ -175,12 +406,17 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) static int cppc_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; @@ -1477,7 +1478,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); WRITE_ONCE(cpudata->lowest_nonlinear_perf, -@@ -269,6 +504,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, +@@ -269,6 +505,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, u64 prev = READ_ONCE(cpudata->cppc_req_cached); u64 value = prev; @@ -1485,7 +1486,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 value &= ~AMD_CPPC_MIN_PERF(~0L); value |= AMD_CPPC_MIN_PERF(min_perf); -@@ -312,7 +548,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, +@@ -312,7 +549,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, return -ENODEV; cap_perf = READ_ONCE(cpudata->highest_perf); @@ -1494,7 +1495,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 max_perf = cap_perf; freqs.old = policy->cur; -@@ -357,8 +593,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, +@@ -357,8 +594,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (max_perf < min_perf) max_perf = min_perf; @@ -1503,7 +1504,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); } -@@ -470,7 +704,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) +@@ -470,7 +705,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) return; cpudata->boost_supported = true; @@ -1512,7 +1513,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 } static int amd_pstate_cpu_init(struct cpufreq_policy *policy) -@@ -555,9 +789,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) +@@ -555,9 +790,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) { @@ -1523,7 +1524,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 freq_qos_remove_request(&cpudata->req[1]); freq_qos_remove_request(&cpudata->req[0]); -@@ -599,9 +831,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, +@@ -599,9 +832,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { int max_freq; @@ -1534,7 +1535,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 max_freq = amd_get_max_freq(cpudata); if (max_freq < 0) -@@ -614,9 +844,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli +@@ -614,9 +845,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli char *buf) { int freq; @@ -1545,7 +1546,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 freq = amd_get_lowest_nonlinear_freq(cpudata); if (freq < 0) -@@ -640,10 +868,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, +@@ -640,10 +869,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sprintf(&buf[0], "%u\n", perf); } @@ -1654,7 +1655,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, -@@ -652,6 +978,552 @@ static struct freq_attr *amd_pstate_attr[] = { +@@ -652,6 +979,552 @@ static struct freq_attr *amd_pstate_attr[] = { NULL, }; @@ -2090,7 +2091,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 + + pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); + -+ if (epp) { ++ if (epp_enabled) { + amd_pstate_epp_reenable(cpudata); + cpudata->suspended = false; + } @@ -2139,7 +2140,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 + if (pstate_exiting) + return 0; + -+ if (epp) ++ if (epp_enabled) + amd_pstate_epp_offline(policy); + + return 0; @@ -2158,7 +2159,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 + int ret; + + /* avoid suspending when EPP is not enabled */ -+ if (!epp) ++ if (!epp_enabled) + return 0; + + /* set this flag to avoid setting core offline*/ @@ -2207,7 +2208,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 static struct cpufreq_driver amd_pstate_driver = { .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, .verify = amd_pstate_verify, -@@ -662,11 +1534,27 @@ static struct cpufreq_driver amd_pstate_driver = { +@@ -662,11 +1535,27 @@ static struct cpufreq_driver amd_pstate_driver = { .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, .name = "amd-pstate", @@ -2236,7 +2237,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 int ret; if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) -@@ -681,10 +1569,24 @@ static int __init amd_pstate_init(void) +@@ -681,10 +1570,24 @@ static int __init amd_pstate_init(void) if (cpufreq_get_current_driver()) return -EEXIST; @@ -2245,7 +2246,7 @@ index 9ac75c1cde9c..bdb44a08771c 100644 + return -ENOMEM; + WRITE_ONCE(all_cpu_data, cpudata); + -+ if (epp) { ++ if (epp_enabled) { + pr_info("AMD CPPC loading with amd_pstate_epp driver instance.\n"); + default_pstate_driver = &amd_pstate_epp_driver; + } else { @@ -2255,14 +2256,14 @@ index 9ac75c1cde9c..bdb44a08771c 100644 + /* capability check */ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!epp) ++ if (!epp_enabled) + default_pstate_driver->adjust_perf = amd_pstate_adjust_perf; pr_debug("AMD CPPC MSR based functionality is supported\n"); - amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; } else if (shared_mem) { static_call_update(amd_pstate_enable, cppc_enable); static_call_update(amd_pstate_init_perf, cppc_init_perf); -@@ -701,19 +1603,57 @@ static int __init amd_pstate_init(void) +@@ -701,19 +1604,57 @@ static int __init amd_pstate_init(void) return ret; } diff --git a/6.0/0001-cachyos-base-all.patch b/6.0/0001-cachyos-base-all.patch new file mode 100644 index 00000000..9805909c --- /dev/null +++ b/6.0/0001-cachyos-base-all.patch @@ -0,0 +1,112360 @@ +From a8998dccf3f62a61e162f1278a742e49bf95df8b Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 10 Oct 2022 13:16:38 +0200 +Subject: [PATCH 01/17] cachy + +Signed-off-by: Peter Jung +--- + .gitignore | 1 + + Documentation/admin-guide/pm/amd-pstate.rst | 19 + + Makefile | 2 + + arch/arc/configs/axs101_defconfig | 1 + + arch/arc/configs/axs103_defconfig | 1 + + arch/arc/configs/axs103_smp_defconfig | 1 + + arch/arc/configs/haps_hs_defconfig | 1 + + arch/arc/configs/haps_hs_smp_defconfig | 1 + + arch/arc/configs/hsdk_defconfig | 1 + + arch/arc/configs/nsim_700_defconfig | 1 + + arch/arc/configs/nsimosci_defconfig | 1 + + arch/arc/configs/nsimosci_hs_defconfig | 1 + + arch/arc/configs/nsimosci_hs_smp_defconfig | 1 + + arch/arc/configs/tb10x_defconfig | 1 + + arch/arc/configs/vdk_hs38_defconfig | 1 + + arch/arc/configs/vdk_hs38_smp_defconfig | 1 + + arch/x86/Kconfig.cpu | 332 ++++++- + arch/x86/Makefile | 43 +- + arch/x86/Makefile.postlink | 41 + + arch/x86/boot/compressed/Makefile | 10 +- + arch/x86/include/asm/msr-index.h | 7 + + arch/x86/include/asm/vermagic.h | 66 ++ + block/bfq-iosched.c | 6 + + block/elevator.c | 7 +- + drivers/acpi/cppc_acpi.c | 128 ++- + drivers/cpufreq/amd-pstate.c | 989 +++++++++++++++++++- + drivers/cpufreq/bmips-cpufreq.c | 10 +- + drivers/cpufreq/cpufreq-dt-platdev.c | 1 + + drivers/cpufreq/cpufreq.c | 2 + + drivers/cpufreq/highbank-cpufreq.c | 2 +- + drivers/cpufreq/intel_pstate.c | 1 + + drivers/cpufreq/qcom-cpufreq-hw.c | 24 +- + drivers/cpufreq/sti-cpufreq.c | 2 +- + drivers/cpufreq/tegra194-cpufreq.c | 35 +- + drivers/cpufreq/ti-cpufreq.c | 2 +- + drivers/md/dm-crypt.c | 5 + + include/acpi/cppc_acpi.h | 17 + + include/linux/pagemap.h | 2 +- + include/linux/user_namespace.h | 4 + + init/Kconfig | 39 + + kernel/Kconfig.hz | 24 + + kernel/fork.c | 14 + + kernel/module/Kconfig | 25 + + kernel/sched/core.c | 21 +- + kernel/sched/cpufreq_schedutil.c | 30 +- + kernel/sched/fair.c | 28 +- + kernel/sched/pelt.c | 60 ++ + kernel/sched/pelt.h | 42 +- + kernel/sched/sched.h | 1 + + kernel/sysctl.c | 22 + + kernel/user_namespace.c | 7 + + lib/Kconfig | 8 +- + lib/string.c | 62 +- + lib/zstd/Makefile | 16 +- + lib/zstd/common/entropy_common.c | 5 +- + lib/zstd/common/zstd_common.c | 10 + + lib/zstd/compress/zstd_double_fast.c | 61 +- + lib/zstd/compress/zstd_fast.c | 69 +- + lib/zstd/compress/zstd_lazy.c | 223 ++--- + mm/compaction.c | 6 +- + mm/page-writeback.c | 8 + + mm/swap.c | 5 + + mm/vmpressure.c | 4 + + mm/vmscan.c | 4 + + scripts/Makefile.lib | 13 +- + scripts/Makefile.modinst | 7 +- + 66 files changed, 2291 insertions(+), 294 deletions(-) + create mode 100644 arch/x86/Makefile.postlink + +diff --git a/.gitignore b/.gitignore +index 265959544978..cd4ef88584ea 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -37,6 +37,7 @@ + *.o + *.o.* + *.patch ++*.relocs + *.s + *.so + *.so.dbg +diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst +index 83b58eb4ab4d..d0f0e115013b 100644 +--- a/Documentation/admin-guide/pm/amd-pstate.rst ++++ b/Documentation/admin-guide/pm/amd-pstate.rst +@@ -261,6 +261,25 @@ lowest non-linear performance in `AMD CPPC Performance Capability + `_.) + This attribute is read-only. + ++``energy_performance_available_preferences`` ++ ++All the supported EPP preference could be selected, List of the strings that ++can be set to the ``energy_performance_preference`` attribute ++those different profiles represent different energy vs efficiency hints provided ++to low-level firmware ++however, the ``default`` represents the epp value is set by platform firmware ++This attribute is read-only. ++ ++``energy_performance_preference`` ++ ++The current energy performance preference can be read from this attribute. ++and user can change current preference according to energy or performance needs ++Please get all support profiles list from ++``energy_performance_available_preferences`` attribute, all the profiles are ++integer values defined between 0 to 255 when EPP feature is enabled by platform ++firmware, if EPP feature is disabled, driver will ignore the written value ++This attribute is read-write. ++ + Other performance and frequency values can be read back from + ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. + +diff --git a/Makefile b/Makefile +index 8478e13e9424..30320363622c 100644 +--- a/Makefile ++++ b/Makefile +@@ -758,6 +758,8 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) + + ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE + KBUILD_CFLAGS += -O2 ++else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 ++KBUILD_CFLAGS += -O3 + else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE + KBUILD_CFLAGS += -Os + endif +diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig +index e31a8ebc3ecc..0016149f9583 100644 +--- a/arch/arc/configs/axs101_defconfig ++++ b/arch/arc/configs/axs101_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig +index e0e8567f0d75..5b031582a1cf 100644 +--- a/arch/arc/configs/axs103_defconfig ++++ b/arch/arc/configs/axs103_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig +index fcbc952bc75b..d4eec39e0112 100644 +--- a/arch/arc/configs/axs103_smp_defconfig ++++ b/arch/arc/configs/axs103_smp_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig +index d87ad7e88d62..7337cdf4ffdd 100644 +--- a/arch/arc/configs/haps_hs_defconfig ++++ b/arch/arc/configs/haps_hs_defconfig +@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EXPERT=y + CONFIG_PERF_EVENTS=y + # CONFIG_COMPAT_BRK is not set +diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig +index 8d82cdb7f86a..bc927221afc0 100644 +--- a/arch/arc/configs/haps_hs_smp_defconfig ++++ b/arch/arc/configs/haps_hs_smp_defconfig +@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig +index f856b03e0fb5..aa000075a575 100644 +--- a/arch/arc/configs/hsdk_defconfig ++++ b/arch/arc/configs/hsdk_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y + CONFIG_BLK_DEV_RAM=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig +index a1ce12bf5b16..326f6cde7826 100644 +--- a/arch/arc/configs/nsim_700_defconfig ++++ b/arch/arc/configs/nsim_700_defconfig +@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y +diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig +index ca10f4a2c823..bf39a0091679 100644 +--- a/arch/arc/configs/nsimosci_defconfig ++++ b/arch/arc/configs/nsimosci_defconfig +@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y +diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig +index 31b6ec3683c6..7121bd71c543 100644 +--- a/arch/arc/configs/nsimosci_hs_defconfig ++++ b/arch/arc/configs/nsimosci_hs_defconfig +@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y +diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig +index 41a0037f48a5..f9863b294a70 100644 +--- a/arch/arc/configs/nsimosci_hs_smp_defconfig ++++ b/arch/arc/configs/nsimosci_hs_smp_defconfig +@@ -8,6 +8,7 @@ CONFIG_IKCONFIG_PROC=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_PERF_EVENTS=y + # CONFIG_COMPAT_BRK is not set + CONFIG_KPROBES=y +diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig +index d93b65008d4a..a12656ec0072 100644 +--- a/arch/arc/configs/tb10x_defconfig ++++ b/arch/arc/configs/tb10x_defconfig +@@ -14,6 +14,7 @@ CONFIG_INITRAMFS_SOURCE="../tb10x-rootfs.cpio" + CONFIG_INITRAMFS_ROOT_UID=2100 + CONFIG_INITRAMFS_ROOT_GID=501 + # CONFIG_RD_GZIP is not set ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + # CONFIG_AIO is not set + CONFIG_EMBEDDED=y +diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig +index 0c3b21416819..d7c858df520c 100644 +--- a/arch/arc/configs/vdk_hs38_defconfig ++++ b/arch/arc/configs/vdk_hs38_defconfig +@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y + CONFIG_IKCONFIG=y + CONFIG_IKCONFIG_PROC=y + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig +index f9ad9d3ee702..015c1d43889e 100644 +--- a/arch/arc/configs/vdk_hs38_smp_defconfig ++++ b/arch/arc/configs/vdk_hs38_smp_defconfig +@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y + CONFIG_IKCONFIG=y + CONFIG_IKCONFIG_PROC=y + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu +index 542377cd419d..22b919cdb6d1 100644 +--- a/arch/x86/Kconfig.cpu ++++ b/arch/x86/Kconfig.cpu +@@ -157,7 +157,7 @@ config MPENTIUM4 + + + config MK6 +- bool "K6/K6-II/K6-III" ++ bool "AMD K6/K6-II/K6-III" + depends on X86_32 + help + Select this for an AMD K6-family processor. Enables use of +@@ -165,7 +165,7 @@ config MK6 + flags to GCC. + + config MK7 +- bool "Athlon/Duron/K7" ++ bool "AMD Athlon/Duron/K7" + depends on X86_32 + help + Select this for an AMD Athlon K7-family processor. Enables use of +@@ -173,12 +173,98 @@ config MK7 + flags to GCC. + + config MK8 +- bool "Opteron/Athlon64/Hammer/K8" ++ bool "AMD Opteron/Athlon64/Hammer/K8" + help + Select this for an AMD Opteron or Athlon64 Hammer-family processor. + Enables use of some extended instructions, and passes appropriate + optimization flags to GCC. + ++config MK8SSE3 ++ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" ++ help ++ Select this for improved AMD Opteron or Athlon64 Hammer-family processors. ++ Enables use of some extended instructions, and passes appropriate ++ optimization flags to GCC. ++ ++config MK10 ++ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" ++ help ++ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, ++ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. ++ Enables use of some extended instructions, and passes appropriate ++ optimization flags to GCC. ++ ++config MBARCELONA ++ bool "AMD Barcelona" ++ help ++ Select this for AMD Family 10h Barcelona processors. ++ ++ Enables -march=barcelona ++ ++config MBOBCAT ++ bool "AMD Bobcat" ++ help ++ Select this for AMD Family 14h Bobcat processors. ++ ++ Enables -march=btver1 ++ ++config MJAGUAR ++ bool "AMD Jaguar" ++ help ++ Select this for AMD Family 16h Jaguar processors. ++ ++ Enables -march=btver2 ++ ++config MBULLDOZER ++ bool "AMD Bulldozer" ++ help ++ Select this for AMD Family 15h Bulldozer processors. ++ ++ Enables -march=bdver1 ++ ++config MPILEDRIVER ++ bool "AMD Piledriver" ++ help ++ Select this for AMD Family 15h Piledriver processors. ++ ++ Enables -march=bdver2 ++ ++config MSTEAMROLLER ++ bool "AMD Steamroller" ++ help ++ Select this for AMD Family 15h Steamroller processors. ++ ++ Enables -march=bdver3 ++ ++config MEXCAVATOR ++ bool "AMD Excavator" ++ help ++ Select this for AMD Family 15h Excavator processors. ++ ++ Enables -march=bdver4 ++ ++config MZEN ++ bool "AMD Zen" ++ help ++ Select this for AMD Family 17h Zen processors. ++ ++ Enables -march=znver1 ++ ++config MZEN2 ++ bool "AMD Zen 2" ++ help ++ Select this for AMD Family 17h Zen 2 processors. ++ ++ Enables -march=znver2 ++ ++config MZEN3 ++ bool "AMD Zen 3" ++ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ help ++ Select this for AMD Family 19h Zen 3 processors. ++ ++ Enables -march=znver3 ++ + config MCRUSOE + bool "Crusoe" + depends on X86_32 +@@ -270,7 +356,7 @@ config MPSC + in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. + + config MCORE2 +- bool "Core 2/newer Xeon" ++ bool "Intel Core 2" + help + + Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and +@@ -278,6 +364,8 @@ config MCORE2 + family in /proc/cpuinfo. Newer ones have 6 and older ones 15 + (not a typo) + ++ Enables -march=core2 ++ + config MATOM + bool "Intel Atom" + help +@@ -287,6 +375,182 @@ config MATOM + accordingly optimized code. Use a recent GCC with specific Atom + support in order to fully benefit from selecting this option. + ++config MNEHALEM ++ bool "Intel Nehalem" ++ select X86_P6_NOP ++ help ++ ++ Select this for 1st Gen Core processors in the Nehalem family. ++ ++ Enables -march=nehalem ++ ++config MWESTMERE ++ bool "Intel Westmere" ++ select X86_P6_NOP ++ help ++ ++ Select this for the Intel Westmere formerly Nehalem-C family. ++ ++ Enables -march=westmere ++ ++config MSILVERMONT ++ bool "Intel Silvermont" ++ select X86_P6_NOP ++ help ++ ++ Select this for the Intel Silvermont platform. ++ ++ Enables -march=silvermont ++ ++config MGOLDMONT ++ bool "Intel Goldmont" ++ select X86_P6_NOP ++ help ++ ++ Select this for the Intel Goldmont platform including Apollo Lake and Denverton. ++ ++ Enables -march=goldmont ++ ++config MGOLDMONTPLUS ++ bool "Intel Goldmont Plus" ++ select X86_P6_NOP ++ help ++ ++ Select this for the Intel Goldmont Plus platform including Gemini Lake. ++ ++ Enables -march=goldmont-plus ++ ++config MSANDYBRIDGE ++ bool "Intel Sandy Bridge" ++ select X86_P6_NOP ++ help ++ ++ Select this for 2nd Gen Core processors in the Sandy Bridge family. ++ ++ Enables -march=sandybridge ++ ++config MIVYBRIDGE ++ bool "Intel Ivy Bridge" ++ select X86_P6_NOP ++ help ++ ++ Select this for 3rd Gen Core processors in the Ivy Bridge family. ++ ++ Enables -march=ivybridge ++ ++config MHASWELL ++ bool "Intel Haswell" ++ select X86_P6_NOP ++ help ++ ++ Select this for 4th Gen Core processors in the Haswell family. ++ ++ Enables -march=haswell ++ ++config MBROADWELL ++ bool "Intel Broadwell" ++ select X86_P6_NOP ++ help ++ ++ Select this for 5th Gen Core processors in the Broadwell family. ++ ++ Enables -march=broadwell ++ ++config MSKYLAKE ++ bool "Intel Skylake" ++ select X86_P6_NOP ++ help ++ ++ Select this for 6th Gen Core processors in the Skylake family. ++ ++ Enables -march=skylake ++ ++config MSKYLAKEX ++ bool "Intel Skylake X" ++ select X86_P6_NOP ++ help ++ ++ Select this for 6th Gen Core processors in the Skylake X family. ++ ++ Enables -march=skylake-avx512 ++ ++config MCANNONLAKE ++ bool "Intel Cannon Lake" ++ select X86_P6_NOP ++ help ++ ++ Select this for 8th Gen Core processors ++ ++ Enables -march=cannonlake ++ ++config MICELAKE ++ bool "Intel Ice Lake" ++ select X86_P6_NOP ++ help ++ ++ Select this for 10th Gen Core processors in the Ice Lake family. ++ ++ Enables -march=icelake-client ++ ++config MCASCADELAKE ++ bool "Intel Cascade Lake" ++ select X86_P6_NOP ++ help ++ ++ Select this for Xeon processors in the Cascade Lake family. ++ ++ Enables -march=cascadelake ++ ++config MCOOPERLAKE ++ bool "Intel Cooper Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) ++ select X86_P6_NOP ++ help ++ ++ Select this for Xeon processors in the Cooper Lake family. ++ ++ Enables -march=cooperlake ++ ++config MTIGERLAKE ++ bool "Intel Tiger Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) ++ select X86_P6_NOP ++ help ++ ++ Select this for third-generation 10 nm process processors in the Tiger Lake family. ++ ++ Enables -march=tigerlake ++ ++config MSAPPHIRERAPIDS ++ bool "Intel Sapphire Rapids" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ select X86_P6_NOP ++ help ++ ++ Select this for third-generation 10 nm process processors in the Sapphire Rapids family. ++ ++ Enables -march=sapphirerapids ++ ++config MROCKETLAKE ++ bool "Intel Rocket Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ select X86_P6_NOP ++ help ++ ++ Select this for eleventh-generation processors in the Rocket Lake family. ++ ++ Enables -march=rocketlake ++ ++config MALDERLAKE ++ bool "Intel Alder Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ select X86_P6_NOP ++ help ++ ++ Select this for twelfth-generation processors in the Alder Lake family. ++ ++ Enables -march=alderlake ++ + config GENERIC_CPU + bool "Generic-x86-64" + depends on X86_64 +@@ -294,6 +558,50 @@ config GENERIC_CPU + Generic x86-64 CPU. + Run equally well on all x86-64 CPUs. + ++config GENERIC_CPU2 ++ bool "Generic-x86-64-v2" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ depends on X86_64 ++ help ++ Generic x86-64 CPU. ++ Run equally well on all x86-64 CPUs with min support of x86-64-v2. ++ ++config GENERIC_CPU3 ++ bool "Generic-x86-64-v3" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ depends on X86_64 ++ help ++ Generic x86-64-v3 CPU with v3 instructions. ++ Run equally well on all x86-64 CPUs with min support of x86-64-v3. ++ ++config GENERIC_CPU4 ++ bool "Generic-x86-64-v4" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ depends on X86_64 ++ help ++ Generic x86-64 CPU with v4 instructions. ++ Run equally well on all x86-64 CPUs with min support of x86-64-v4. ++ ++config MNATIVE_INTEL ++ bool "Intel-Native optimizations autodetected by the compiler" ++ help ++ ++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects ++ the optimum settings to use based on your processor. Do NOT use this ++ for AMD CPUs. Intel Only! ++ ++ Enables -march=native ++ ++config MNATIVE_AMD ++ bool "AMD-Native optimizations autodetected by the compiler" ++ help ++ ++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects ++ the optimum settings to use based on your processor. Do NOT use this ++ for Intel CPUs. AMD Only! ++ ++ Enables -march=native ++ + endchoice + + config X86_GENERIC +@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT + config X86_L1_CACHE_SHIFT + int + default "7" if MPENTIUM4 || MPSC +- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU ++ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 + default "4" if MELAN || M486SX || M486 || MGEODEGX1 + default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX + +@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 + + config X86_INTEL_USERCOPY + def_bool y +- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 ++ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL + + config X86_USE_PPRO_CHECKSUM + def_bool y +- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM ++ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD + + # + # P6_NOPs are a relatively minor optimization that require a family >= +@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM + config X86_P6_NOP + def_bool y + depends on X86_64 +- depends on (MCORE2 || MPENTIUM4 || MPSC) ++ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) + + config X86_TSC + def_bool y +- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 ++ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 + + config X86_CMPXCHG64 + def_bool y +- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 ++ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD + + # this should be set for all -march=.. options where the compiler + # generates cmov. + config X86_CMOV + def_bool y +- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) ++ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) + + config X86_MINIMUM_CPU_FAMILY + int + default "64" if X86_64 +- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) ++ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) + default "5" if X86_32 && X86_CMPXCHG64 + default "4" + +diff --git a/arch/x86/Makefile b/arch/x86/Makefile +index bafbd905e6e7..81142991953b 100644 +--- a/arch/x86/Makefile ++++ b/arch/x86/Makefile +@@ -67,7 +67,8 @@ export BITS + # + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383 + # +-KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx ++KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -mno-avx2 \ ++ -mno-avx512f -O3 + + ifeq ($(CONFIG_X86_KERNEL_IBT),y) + # +@@ -150,8 +151,44 @@ else + # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) + cflags-$(CONFIG_MK8) += -march=k8 + cflags-$(CONFIG_MPSC) += -march=nocona +- cflags-$(CONFIG_MCORE2) += -march=core2 +- cflags-$(CONFIG_MATOM) += -march=atom ++ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3 ++ cflags-$(CONFIG_MK10) += -march=amdfam10 ++ cflags-$(CONFIG_MBARCELONA) += -march=barcelona ++ cflags-$(CONFIG_MBOBCAT) += -march=btver1 ++ cflags-$(CONFIG_MJAGUAR) += -march=btver2 ++ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1 ++ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm ++ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm ++ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm ++ cflags-$(CONFIG_MZEN) += -march=znver1 ++ cflags-$(CONFIG_MZEN2) += -march=znver2 ++ cflags-$(CONFIG_MZEN3) += -march=znver3 ++ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native ++ cflags-$(CONFIG_MNATIVE_AMD) += -march=native ++ cflags-$(CONFIG_MATOM) += -march=bonnell ++ cflags-$(CONFIG_MCORE2) += -march=core2 ++ cflags-$(CONFIG_MNEHALEM) += -march=nehalem ++ cflags-$(CONFIG_MWESTMERE) += -march=westmere ++ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont ++ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont ++ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus ++ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge ++ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge ++ cflags-$(CONFIG_MHASWELL) += -march=haswell ++ cflags-$(CONFIG_MBROADWELL) += -march=broadwell ++ cflags-$(CONFIG_MSKYLAKE) += -march=skylake ++ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512 ++ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake ++ cflags-$(CONFIG_MICELAKE) += -march=icelake-client ++ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake ++ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake ++ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake ++ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids ++ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake ++ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake ++ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2 ++ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3 ++ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4 + cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic + KBUILD_CFLAGS += $(cflags-y) + +diff --git a/arch/x86/Makefile.postlink b/arch/x86/Makefile.postlink +new file mode 100644 +index 000000000000..b38ffa4defb3 +--- /dev/null ++++ b/arch/x86/Makefile.postlink +@@ -0,0 +1,41 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# =========================================================================== ++# Post-link x86 pass ++# =========================================================================== ++# ++# 1. Separate relocations from vmlinux into vmlinux.relocs. ++# 2. Strip relocations from vmlinux. ++ ++PHONY := __archpost ++__archpost: ++ ++-include include/config/auto.conf ++include scripts/Kbuild.include ++ ++CMD_RELOCS = arch/x86/tools/relocs ++quiet_cmd_relocs = RELOCS $@.relocs ++ cmd_relocs = $(CMD_RELOCS) $@ > $@.relocs;$(CMD_RELOCS) --abs-relocs $@ ++ ++quiet_cmd_strip_relocs = RSTRIP $@ ++ cmd_strip_relocs = $(OBJCOPY) --remove-section='.rel.*' --remove-section='.rel__*' --remove-section='.rela.*' --remove-section='.rela__*' $@ ++ ++# `@true` prevents complaint when there is nothing to be done ++ ++vmlinux: FORCE ++ @true ++ifeq ($(CONFIG_X86_NEED_RELOCS),y) ++ $(call cmd,relocs) ++ $(call cmd,strip_relocs) ++endif ++ ++%.ko: FORCE ++ @true ++ ++clean: ++ @rm -f vmlinux.relocs ++ ++PHONY += FORCE clean ++ ++FORCE: ++ ++.PHONY: $(PHONY) +diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile +index 35ce1a64068b..eba7709d75ae 100644 +--- a/arch/x86/boot/compressed/Makefile ++++ b/arch/x86/boot/compressed/Makefile +@@ -120,14 +120,12 @@ $(obj)/vmlinux.bin: vmlinux FORCE + + targets += $(patsubst $(obj)/%,%,$(vmlinux-objs-y)) vmlinux.bin.all vmlinux.relocs + +-CMD_RELOCS = arch/x86/tools/relocs +-quiet_cmd_relocs = RELOCS $@ +- cmd_relocs = $(CMD_RELOCS) $< > $@;$(CMD_RELOCS) --abs-relocs $< +-$(obj)/vmlinux.relocs: vmlinux FORCE +- $(call if_changed,relocs) ++# vmlinux.relocs is created by the vmlinux postlink step. ++vmlinux.relocs: vmlinux ++ @true + + vmlinux.bin.all-y := $(obj)/vmlinux.bin +-vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs ++vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += vmlinux.relocs + + $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE + $(call if_changed,gzip) +diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h +index 6674bdb096f3..53cbdb0c522b 100644 +--- a/arch/x86/include/asm/msr-index.h ++++ b/arch/x86/include/asm/msr-index.h +@@ -569,6 +569,7 @@ + #define MSR_AMD_CPPC_CAP2 0xc00102b2 + #define MSR_AMD_CPPC_REQ 0xc00102b3 + #define MSR_AMD_CPPC_STATUS 0xc00102b4 ++#define MSR_AMD_CPPC_HW_CTL 0xc0010015 + + #define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) + #define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) +@@ -579,12 +580,18 @@ + #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) + #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) + #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) ++#define AMD_CPPC_PRECISION_BOOST_BIT 25 ++#define AMD_CPPC_PRECISION_BOOST_ENABLED BIT_ULL(AMD_CPPC_PRECISION_BOOST_BIT) + + /* AMD Performance Counter Global Status and Control MSRs */ + #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 + #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 + #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 + ++#define AMD_CPPC_EPP_PERFORMANCE 0x00 ++#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 ++#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF ++#define AMD_CPPC_EPP_POWERSAVE 0xFF + /* Fam 17h MSRs */ + #define MSR_F17H_IRPERF 0xc00000e9 + +diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h +index 75884d2cdec3..4e6a08d4c7e5 100644 +--- a/arch/x86/include/asm/vermagic.h ++++ b/arch/x86/include/asm/vermagic.h +@@ -17,6 +17,48 @@ + #define MODULE_PROC_FAMILY "586MMX " + #elif defined CONFIG_MCORE2 + #define MODULE_PROC_FAMILY "CORE2 " ++#elif defined CONFIG_MNATIVE_INTEL ++#define MODULE_PROC_FAMILY "NATIVE_INTEL " ++#elif defined CONFIG_MNATIVE_AMD ++#define MODULE_PROC_FAMILY "NATIVE_AMD " ++#elif defined CONFIG_MNEHALEM ++#define MODULE_PROC_FAMILY "NEHALEM " ++#elif defined CONFIG_MWESTMERE ++#define MODULE_PROC_FAMILY "WESTMERE " ++#elif defined CONFIG_MSILVERMONT ++#define MODULE_PROC_FAMILY "SILVERMONT " ++#elif defined CONFIG_MGOLDMONT ++#define MODULE_PROC_FAMILY "GOLDMONT " ++#elif defined CONFIG_MGOLDMONTPLUS ++#define MODULE_PROC_FAMILY "GOLDMONTPLUS " ++#elif defined CONFIG_MSANDYBRIDGE ++#define MODULE_PROC_FAMILY "SANDYBRIDGE " ++#elif defined CONFIG_MIVYBRIDGE ++#define MODULE_PROC_FAMILY "IVYBRIDGE " ++#elif defined CONFIG_MHASWELL ++#define MODULE_PROC_FAMILY "HASWELL " ++#elif defined CONFIG_MBROADWELL ++#define MODULE_PROC_FAMILY "BROADWELL " ++#elif defined CONFIG_MSKYLAKE ++#define MODULE_PROC_FAMILY "SKYLAKE " ++#elif defined CONFIG_MSKYLAKEX ++#define MODULE_PROC_FAMILY "SKYLAKEX " ++#elif defined CONFIG_MCANNONLAKE ++#define MODULE_PROC_FAMILY "CANNONLAKE " ++#elif defined CONFIG_MICELAKE ++#define MODULE_PROC_FAMILY "ICELAKE " ++#elif defined CONFIG_MCASCADELAKE ++#define MODULE_PROC_FAMILY "CASCADELAKE " ++#elif defined CONFIG_MCOOPERLAKE ++#define MODULE_PROC_FAMILY "COOPERLAKE " ++#elif defined CONFIG_MTIGERLAKE ++#define MODULE_PROC_FAMILY "TIGERLAKE " ++#elif defined CONFIG_MSAPPHIRERAPIDS ++#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " ++#elif defined CONFIG_ROCKETLAKE ++#define MODULE_PROC_FAMILY "ROCKETLAKE " ++#elif defined CONFIG_MALDERLAKE ++#define MODULE_PROC_FAMILY "ALDERLAKE " + #elif defined CONFIG_MATOM + #define MODULE_PROC_FAMILY "ATOM " + #elif defined CONFIG_M686 +@@ -35,6 +77,30 @@ + #define MODULE_PROC_FAMILY "K7 " + #elif defined CONFIG_MK8 + #define MODULE_PROC_FAMILY "K8 " ++#elif defined CONFIG_MK8SSE3 ++#define MODULE_PROC_FAMILY "K8SSE3 " ++#elif defined CONFIG_MK10 ++#define MODULE_PROC_FAMILY "K10 " ++#elif defined CONFIG_MBARCELONA ++#define MODULE_PROC_FAMILY "BARCELONA " ++#elif defined CONFIG_MBOBCAT ++#define MODULE_PROC_FAMILY "BOBCAT " ++#elif defined CONFIG_MBULLDOZER ++#define MODULE_PROC_FAMILY "BULLDOZER " ++#elif defined CONFIG_MPILEDRIVER ++#define MODULE_PROC_FAMILY "PILEDRIVER " ++#elif defined CONFIG_MSTEAMROLLER ++#define MODULE_PROC_FAMILY "STEAMROLLER " ++#elif defined CONFIG_MJAGUAR ++#define MODULE_PROC_FAMILY "JAGUAR " ++#elif defined CONFIG_MEXCAVATOR ++#define MODULE_PROC_FAMILY "EXCAVATOR " ++#elif defined CONFIG_MZEN ++#define MODULE_PROC_FAMILY "ZEN " ++#elif defined CONFIG_MZEN2 ++#define MODULE_PROC_FAMILY "ZEN2 " ++#elif defined CONFIG_MZEN3 ++#define MODULE_PROC_FAMILY "ZEN3 " + #elif defined CONFIG_MELAN + #define MODULE_PROC_FAMILY "ELAN " + #elif defined CONFIG_MCRUSOE +diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c +index c740b41fe0a4..ac24b8222bd1 100644 +--- a/block/bfq-iosched.c ++++ b/block/bfq-iosched.c +@@ -7463,6 +7463,7 @@ MODULE_ALIAS("bfq-iosched"); + static int __init bfq_init(void) + { + int ret; ++ char msg[60] = "BFQ I/O-scheduler: BFQ-CachyOS v6.0"; + + #ifdef CONFIG_BFQ_GROUP_IOSCHED + ret = blkcg_policy_register(&blkcg_policy_bfq); +@@ -7494,6 +7495,11 @@ static int __init bfq_init(void) + if (ret) + goto slab_kill; + ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ strcat(msg, " (with cgroups support)"); ++#endif ++ pr_info("%s", msg); ++ + return 0; + + slab_kill: +diff --git a/block/elevator.c b/block/elevator.c +index c319765892bb..b2687032a0e0 100644 +--- a/block/elevator.c ++++ b/block/elevator.c +@@ -640,8 +640,13 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) + + if (q->nr_hw_queues != 1 && + !blk_mq_is_shared_tags(q->tag_set->flags)) ++#if defined(CONFIG_CACHY) && defined(CONFIG_MQ_IOSCHED_KYBER) ++ return elevator_get(q, "kyber", false); ++#elif defined(CONFIG_CACHY) ++ return elevator_get(q, "mq-deadline", false); ++#else + return NULL; +- ++#endif + return elevator_get(q, "mq-deadline", false); + } + +diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c +index 1e15a9f25ae9..9c28f47e1b1e 100644 +--- a/drivers/acpi/cppc_acpi.c ++++ b/drivers/acpi/cppc_acpi.c +@@ -1320,6 +1320,132 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) + } + EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs); + ++/** ++ * cppc_get_epp_caps - Get the energy preference register value. ++ * @cpunum: CPU from which to get epp preference level. ++ * @perf_caps: Return address. ++ * ++ * Return: 0 for success, -EIO otherwise. ++ */ ++int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) ++{ ++ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); ++ struct cpc_register_resource *energy_perf_reg; ++ u64 energy_perf; ++ ++ if (!cpc_desc) { ++ pr_warn("No CPC descriptor for CPU:%d\n", cpunum); ++ return -ENODEV; ++ } ++ ++ energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; ++ ++ if (!CPC_SUPPORTED(energy_perf_reg)) ++ pr_warn("energy perf reg update is unsupported!\n"); ++ ++ if (CPC_IN_PCC(energy_perf_reg)) { ++ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); ++ struct cppc_pcc_data *pcc_ss_data = NULL; ++ int ret = 0; ++ ++ if (pcc_ss_id < 0) ++ return -EIO; ++ ++ pcc_ss_data = pcc_data[pcc_ss_id]; ++ ++ down_write(&pcc_ss_data->pcc_lock); ++ ++ if (send_pcc_cmd(pcc_ss_id, CMD_READ) >= 0) { ++ cpc_read(cpunum, energy_perf_reg, &energy_perf); ++ perf_caps->energy_perf = energy_perf; ++ } else { ++ ret = -EIO; ++ } ++ ++ up_write(&pcc_ss_data->pcc_lock); ++ ++ return ret; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(cppc_get_epp_caps); ++ ++int cppc_set_auto_epp(int cpu, bool enable) ++{ ++ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); ++ struct cpc_register_resource *auto_sel_reg; ++ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); ++ struct cppc_pcc_data *pcc_ss_data = NULL; ++ int ret = -EINVAL; ++ ++ if (!cpc_desc) { ++ pr_warn("No CPC descriptor for CPU:%d\n", cpu); ++ return -EINVAL; ++ } ++ ++ auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; ++ ++ if (CPC_IN_PCC(auto_sel_reg)) { ++ if (pcc_ss_id < 0) ++ return -EIO; ++ ++ ret = cpc_write(cpu, auto_sel_reg, enable); ++ if (ret) ++ return ret; ++ ++ pcc_ss_data = pcc_data[pcc_ss_id]; ++ ++ down_write(&pcc_ss_data->pcc_lock); ++ /* after writing CPC, transfer the ownership of PCC to platform */ ++ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); ++ up_write(&pcc_ss_data->pcc_lock); ++ return ret; ++ } ++ ++ return cpc_write(cpu, auto_sel_reg, enable); ++} ++EXPORT_SYMBOL_GPL(cppc_set_auto_epp); ++ ++/* ++ * Set Energy Performance Preference Register value through ++ * Performance Controls Interface ++ */ ++int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) ++{ ++ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); ++ struct cpc_register_resource *epp_set_reg; ++ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); ++ struct cppc_pcc_data *pcc_ss_data = NULL; ++ int ret = -EINVAL; ++ ++ if (!cpc_desc) { ++ pr_warn("No CPC descriptor for CPU:%d\n", cpu); ++ return -EINVAL; ++ } ++ ++ epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; ++ ++ if (CPC_IN_PCC(epp_set_reg)) { ++ if (pcc_ss_id < 0) ++ return -EIO; ++ ++ ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); ++ if (ret) ++ return ret; ++ ++ pcc_ss_data = pcc_data[pcc_ss_id]; ++ ++ down_write(&pcc_ss_data->pcc_lock); ++ /* after writing CPC, transfer the ownership of PCC to platform */ ++ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); ++ up_write(&pcc_ss_data->pcc_lock); ++ } ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(cppc_set_epp_perf); ++ + /** + * cppc_set_enable - Set to enable CPPC on the processor by writing the + * Continuous Performance Control package EnableRegister field. +@@ -1355,7 +1481,7 @@ int cppc_set_enable(int cpu, bool enable) + pcc_ss_data = pcc_data[pcc_ss_id]; + + down_write(&pcc_ss_data->pcc_lock); +- /* after writing CPC, transfer the ownership of PCC to platfrom */ ++ /* after writing CPC, transfer the ownership of PCC to platform */ + ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); + up_write(&pcc_ss_data->pcc_lock); + return ret; +diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c +index 9ac75c1cde9c..810aea47645e 100644 +--- a/drivers/cpufreq/amd-pstate.c ++++ b/drivers/cpufreq/amd-pstate.c +@@ -46,8 +46,8 @@ + #include + #include "amd-pstate-trace.h" + +-#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 +-#define AMD_PSTATE_TRANSITION_DELAY 500 ++#define AMD_PSTATE_TRANSITION_LATENCY 20000 ++#define AMD_PSTATE_TRANSITION_DELAY 1000 + + /* + * TODO: We need more time to fine tune processors with shared memory solution +@@ -63,7 +63,13 @@ module_param(shared_mem, bool, 0444); + MODULE_PARM_DESC(shared_mem, + "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); + +-static struct cpufreq_driver amd_pstate_driver; ++static bool epp_enabled = false; ++module_param(epp_enabled, bool, 0444); ++MODULE_PARM_DESC(epp_enabled, ++ "Enable energy performance preference (EPP) control"); ++ ++static struct cpufreq_driver *default_pstate_driver; ++static struct amd_cpudata **all_cpu_data; + + /** + * struct amd_aperf_mperf +@@ -75,6 +81,7 @@ struct amd_aperf_mperf { + u64 aperf; + u64 mperf; + u64 tsc; ++ u64 time; + }; + + /** +@@ -97,6 +104,19 @@ struct amd_aperf_mperf { + * @prev: Last Aperf/Mperf/tsc count value read from register + * @freq: current cpu frequency value + * @boost_supported: check whether the Processor or SBIOS supports boost mode ++ * @epp_powersave: Last saved CPPC energy performance preference ++ when policy switched to performance ++ * @epp_policy: Last saved policy used to set energy-performance preference ++ * @epp_cached: Cached CPPC energy-performance preference value ++ * @policy: Cpufreq policy value ++ * @sched_flags: Store scheduler flags for possible cross CPU update ++ * @update_util_set: CPUFreq utility callback is set ++ * @last_update: Time stamp of the last performance state update ++ * @cppc_boost_min: Last CPPC boosted min performance state ++ * @cppc_cap1_cached: Cached value of the last CPPC Capabilities MSR ++ * @update_util: Cpufreq utility callback information ++ * @sample: the stored performance sample ++ * @suspended: Whether or not the driver has been suspended. + * + * The amd_cpudata is key private data for each CPU thread in AMD P-State, and + * represents all the attributes and goals that AMD P-State requests at runtime. +@@ -120,10 +140,203 @@ struct amd_cpudata { + struct amd_aperf_mperf cur; + struct amd_aperf_mperf prev; + +- u64 freq; ++ u64 freq; + bool boost_supported; ++ u64 cppc_hw_conf_cached; ++ ++ /* EPP feature related attributes*/ ++ s16 epp_powersave; ++ s16 epp_policy; ++ s16 epp_cached; ++ u32 policy; ++ u32 sched_flags; ++ bool update_util_set; ++ u64 last_update; ++ u64 last_io_update; ++ u32 cppc_boost_min; ++ u64 cppc_cap1_cached; ++ struct update_util_data update_util; ++ struct amd_aperf_mperf sample; ++ bool suspended; ++}; ++ ++/** ++ * struct amd_pstate_params - global parameters for the performance control ++ * @ cppc_boost_disabled wheher the core performance boost disabled ++ */ ++struct amd_pstate_params { ++ bool cppc_boost_disabled; ++}; ++ ++/* ++ * AMD Energy Preference Performance (EPP) ++ * The EPP is used in the CCLK DPM controller to drive ++ * the frequency that a core is going to operate during ++ * short periods of activity. EPP values will be utilized for ++ * different OS profiles (balanced, performance, power savings) ++ * display strings corresponding to EPP index in the ++ * energy_perf_strings[] ++ * index String ++ *------------------------------------- ++ * 0 default ++ * 1 performance ++ * 2 balance_performance ++ * 3 balance_power ++ * 4 power ++ */ ++enum energy_perf_value_index { ++ EPP_INDEX_DEFAULT = 0, ++ EPP_INDEX_PERFORMANCE, ++ EPP_INDEX_BALANCE_PERFORMANCE, ++ EPP_INDEX_BALANCE_POWERSAVE, ++ EPP_INDEX_POWERSAVE, ++}; ++ ++static const char * const energy_perf_strings[] = { ++ [EPP_INDEX_DEFAULT] = "default", ++ [EPP_INDEX_PERFORMANCE] = "performance", ++ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", ++ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", ++ [EPP_INDEX_POWERSAVE] = "power", ++ NULL ++}; ++ ++static unsigned int epp_values[] = { ++ [EPP_INDEX_DEFAULT] = 0, ++ [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, ++ [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, ++ [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, ++ [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, + }; + ++static struct amd_pstate_params global_params; ++ ++static DEFINE_MUTEX(amd_pstate_limits_lock); ++static DEFINE_MUTEX(amd_pstate_driver_lock); ++static DEFINE_SPINLOCK(amd_pstate_cpu_lock); ++ ++static bool cppc_boost __read_mostly; ++struct kobject *amd_pstate_kobj; ++ ++/* the flag whether the pstate driver is exiting */ ++static bool pstate_exiting; ++ ++#ifdef CONFIG_ACPI_CPPC_LIB ++static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) ++{ ++ s16 epp; ++ struct cppc_perf_caps perf_caps; ++ int ret; ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ if (!cppc_req_cached) { ++ epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, ++ &cppc_req_cached); ++ if (epp) ++ return epp; ++ } ++ epp = (cppc_req_cached >> 24) & 0xFF; ++ } else { ++ ret = cppc_get_epp_caps(cpudata->cpu, &perf_caps); ++ if (ret < 0) { ++ pr_debug("Could not retrieve energy perf value (%d)\n", ret); ++ return -EIO; ++ } ++ epp = (s16) perf_caps.energy_perf; ++ } ++ ++ return epp; ++} ++#endif ++ ++static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata, int *raw_epp) ++{ ++ s16 epp; ++ int index = -EINVAL; ++ ++ *raw_epp = 0; ++ epp = amd_pstate_get_epp(cpudata, 0); ++ if (epp < 0) ++ return epp; ++ ++ switch (epp) { ++ case AMD_CPPC_EPP_PERFORMANCE: ++ index = EPP_INDEX_PERFORMANCE; ++ break; ++ case AMD_CPPC_EPP_BALANCE_PERFORMANCE: ++ index = EPP_INDEX_BALANCE_PERFORMANCE; ++ break; ++ case AMD_CPPC_EPP_BALANCE_POWERSAVE: ++ index = EPP_INDEX_BALANCE_POWERSAVE; ++ break; ++ case AMD_CPPC_EPP_POWERSAVE: ++ index = EPP_INDEX_POWERSAVE; ++ break; ++ default: ++ *raw_epp = epp; ++ index = 0; ++ } ++ ++ return index; ++} ++ ++#ifdef CONFIG_ACPI_CPPC_LIB ++static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) ++{ ++ int ret; ++ struct cppc_perf_ctrls perf_ctrls; ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ u64 value = READ_ONCE(cpudata->cppc_req_cached); ++ ++ value &= ~GENMASK_ULL(31, 24); ++ value |= (u64)epp << 24; ++ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ++ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ if (!ret) ++ cpudata->epp_cached = epp; ++ } else { ++ perf_ctrls.energy_perf = epp; ++ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls); ++ if (ret) { ++ pr_debug("failed to set energy perf value (%d)\n", ret); ++ return ret; ++ } ++ cpudata->epp_cached = epp; ++ } ++ ++ return ret; ++} ++ ++static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, ++ int pref_index, bool use_raw, ++ u32 raw_epp) ++{ ++ int epp = -EINVAL; ++ int ret; ++ ++ if (!pref_index) { ++ pr_debug("EPP pref_index is invalid\n"); ++ return -EINVAL; ++ } ++ ++ if (use_raw) ++ epp = raw_epp; ++ else if (epp == -EINVAL) ++ epp = epp_values[pref_index]; ++ ++ if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { ++ pr_debug("EPP cannot be set under performance policy\n"); ++ return -EBUSY; ++ } ++ ++ ret = amd_pstate_set_epp(cpudata, epp); ++ ++ return ret; ++} ++#endif ++ + static inline int pstate_enable(bool enable) + { + return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); +@@ -131,12 +344,25 @@ static inline int pstate_enable(bool enable) + + static int cppc_enable(bool enable) + { ++ struct cppc_perf_ctrls perf_ctrls; + int cpu, ret = 0; + + for_each_present_cpu(cpu) { + ret = cppc_set_enable(cpu, enable); + if (ret) + return ret; ++ if (epp_enabled) { ++ /* Enable autonomous mode for EPP */ ++ ret = cppc_set_auto_epp(cpu, enable); ++ if (ret) ++ return ret; ++ ++ /* Set desired perf as zero to allow EPP firmware control */ ++ perf_ctrls.desired_perf = 0; ++ ret = cppc_set_perf(cpu, &perf_ctrls); ++ if (ret) ++ return ret; ++ } + } + + return ret; +@@ -152,6 +378,7 @@ static inline int amd_pstate_enable(bool enable) + static int pstate_init_perf(struct amd_cpudata *cpudata) + { + u64 cap1; ++ u32 highest_perf; + + int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, + &cap1); +@@ -163,7 +390,11 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) + * + * CPPC entry doesn't indicate the highest performance in some ASICs. + */ +- WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf()); ++ highest_perf = amd_get_highest_perf(); ++ if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) ++ highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); ++ ++ WRITE_ONCE(cpudata->highest_perf, highest_perf); + + WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); + WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); +@@ -175,12 +406,17 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) + static int cppc_init_perf(struct amd_cpudata *cpudata) + { + struct cppc_perf_caps cppc_perf; ++ u32 highest_perf; + + int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); + if (ret) + return ret; + +- WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf()); ++ highest_perf = amd_get_highest_perf(); ++ if (highest_perf > cppc_perf.highest_perf) ++ highest_perf = cppc_perf.highest_perf; ++ ++ WRITE_ONCE(cpudata->highest_perf, highest_perf); + + WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); + WRITE_ONCE(cpudata->lowest_nonlinear_perf, +@@ -269,6 +505,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, + u64 prev = READ_ONCE(cpudata->cppc_req_cached); + u64 value = prev; + ++ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); + value &= ~AMD_CPPC_MIN_PERF(~0L); + value |= AMD_CPPC_MIN_PERF(min_perf); + +@@ -312,7 +549,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, + return -ENODEV; + + cap_perf = READ_ONCE(cpudata->highest_perf); +- min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); ++ min_perf = READ_ONCE(cpudata->lowest_perf); + max_perf = cap_perf; + + freqs.old = policy->cur; +@@ -357,8 +594,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, + if (max_perf < min_perf) + max_perf = min_perf; + +- des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); +- + amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); + } + +@@ -470,7 +705,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) + return; + + cpudata->boost_supported = true; +- amd_pstate_driver.boost_enabled = true; ++ default_pstate_driver->boost_enabled = true; + } + + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) +@@ -555,9 +790,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) + + static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) + { +- struct amd_cpudata *cpudata; +- +- cpudata = policy->driver_data; ++ struct amd_cpudata *cpudata = policy->driver_data; + + freq_qos_remove_request(&cpudata->req[1]); + freq_qos_remove_request(&cpudata->req[0]); +@@ -599,9 +832,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, + char *buf) + { + int max_freq; +- struct amd_cpudata *cpudata; +- +- cpudata = policy->driver_data; ++ struct amd_cpudata *cpudata = policy->driver_data; + + max_freq = amd_get_max_freq(cpudata); + if (max_freq < 0) +@@ -614,9 +845,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli + char *buf) + { + int freq; +- struct amd_cpudata *cpudata; +- +- cpudata = policy->driver_data; ++ struct amd_cpudata *cpudata = policy->driver_data; + + freq = amd_get_lowest_nonlinear_freq(cpudata); + if (freq < 0) +@@ -640,10 +869,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, + return sprintf(&buf[0], "%u\n", perf); + } + ++static ssize_t show_energy_performance_available_preferences( ++ struct cpufreq_policy *policy, char *buf) ++{ ++ int i = 0; ++ int ret = 0; ++ ++ while (energy_perf_strings[i] != NULL) ++ ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); ++ ++ ret += sprintf(&buf[ret], "\n"); ++ ++ return ret; ++} ++ ++static ssize_t store_energy_performance_preference( ++ struct cpufreq_policy *policy, const char *buf, size_t count) ++{ ++ struct amd_cpudata *cpudata = policy->driver_data; ++ char str_preference[21]; ++ bool raw = false; ++ ssize_t ret; ++ u32 epp = 0; ++ ++ ret = sscanf(buf, "%20s", str_preference); ++ if (ret != 1) ++ return -EINVAL; ++ ++ ret = match_string(energy_perf_strings, -1, str_preference); ++ if (ret < 0) { ++ ret = kstrtouint(buf, 10, &epp); ++ if (ret) ++ return ret; ++ ++ if ((epp > 255) || (epp < 0)) ++ return -EINVAL; ++ ++ raw = true; ++ } ++ ++ mutex_lock(&amd_pstate_limits_lock); ++ ret = amd_pstate_set_energy_pref_index(cpudata, ret, raw, epp); ++ mutex_unlock(&amd_pstate_limits_lock); ++ ++ return ret ?: count; ++} ++ ++static ssize_t show_energy_performance_preference( ++ struct cpufreq_policy *policy, char *buf) ++{ ++ struct amd_cpudata *cpudata = policy->driver_data; ++ int preference, raw_epp; ++ ++ preference = amd_pstate_get_energy_pref_index(cpudata, &raw_epp); ++ if (preference < 0) ++ return preference; ++ ++ if (raw_epp) ++ return sprintf(buf, "%d\n", raw_epp); ++ else ++ return sprintf(buf, "%s\n", energy_perf_strings[preference]); ++} ++ ++static void amd_pstate_update_policies(void) ++{ ++ int cpu; ++ ++ for_each_possible_cpu(cpu) ++ cpufreq_update_policy(cpu); ++} ++ ++static ssize_t show_pstate_dynamic_boost(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "%u\n", cppc_boost); ++} ++ ++static ssize_t store_pstate_dynamic_boost(struct kobject *a, ++ struct kobj_attribute *b, ++ const char *buf, size_t count) ++{ ++ unsigned int input; ++ int ret; ++ ++ ret = kstrtouint(buf, 10, &input); ++ if (ret) ++ return ret; ++ ++ mutex_lock(&amd_pstate_driver_lock); ++ cppc_boost = !!input; ++ amd_pstate_update_policies(); ++ mutex_unlock(&amd_pstate_driver_lock); ++ ++ return count; ++} ++ + cpufreq_freq_attr_ro(amd_pstate_max_freq); + cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); + + cpufreq_freq_attr_ro(amd_pstate_highest_perf); ++cpufreq_freq_attr_rw(energy_performance_preference); ++cpufreq_freq_attr_ro(energy_performance_available_preferences); ++define_one_global_rw(pstate_dynamic_boost); + + static struct freq_attr *amd_pstate_attr[] = { + &amd_pstate_max_freq, +@@ -652,6 +979,552 @@ static struct freq_attr *amd_pstate_attr[] = { + NULL, + }; + ++static struct freq_attr *amd_pstate_epp_attr[] = { ++ &amd_pstate_max_freq, ++ &amd_pstate_lowest_nonlinear_freq, ++ &amd_pstate_highest_perf, ++ &energy_performance_preference, ++ &energy_performance_available_preferences, ++ NULL, ++}; ++ ++static struct attribute *pstate_global_attributes[] = { ++ &pstate_dynamic_boost.attr, ++ NULL ++}; ++ ++static const struct attribute_group amd_pstate_global_attr_group = { ++ .attrs = pstate_global_attributes, ++}; ++ ++static inline void update_boost_state(void) ++{ ++ u64 misc_en; ++ struct amd_cpudata *cpudata; ++ ++ cpudata = all_cpu_data[0]; ++ rdmsrl(MSR_AMD_CPPC_HW_CTL, misc_en); ++ global_params.cppc_boost_disabled = misc_en & AMD_CPPC_PRECISION_BOOST_ENABLED; ++} ++ ++static int amd_pstate_init_cpu(unsigned int cpunum) ++{ ++ struct amd_cpudata *cpudata; ++ ++ cpudata = all_cpu_data[cpunum]; ++ if (!cpudata) { ++ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); ++ if (!cpudata) ++ return -ENOMEM; ++ WRITE_ONCE(all_cpu_data[cpunum], cpudata); ++ ++ cpudata->cpu = cpunum; ++ } ++ cpudata->epp_powersave = -EINVAL; ++ cpudata->epp_policy = 0; ++ pr_debug("controlling: cpu %d\n", cpunum); ++ return 0; ++} ++ ++static int __amd_pstate_cpu_init(struct cpufreq_policy *policy) ++{ ++ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; ++ struct amd_cpudata *cpudata; ++ struct device *dev; ++ int rc; ++ u64 value; ++ ++ rc = amd_pstate_init_cpu(policy->cpu); ++ if (rc) ++ return rc; ++ ++ cpudata = all_cpu_data[policy->cpu]; ++ ++ dev = get_cpu_device(policy->cpu); ++ if (!dev) ++ goto free_cpudata1; ++ ++ rc = amd_pstate_init_perf(cpudata); ++ if (rc) ++ goto free_cpudata1; ++ ++ min_freq = amd_get_min_freq(cpudata); ++ max_freq = amd_get_max_freq(cpudata); ++ nominal_freq = amd_get_nominal_freq(cpudata); ++ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); ++ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { ++ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", ++ min_freq, max_freq); ++ ret = -EINVAL; ++ goto free_cpudata1; ++ } ++ ++ policy->min = min_freq; ++ policy->max = max_freq; ++ ++ policy->cpuinfo.min_freq = min_freq; ++ policy->cpuinfo.max_freq = max_freq; ++ /* It will be updated by governor */ ++ policy->cur = policy->cpuinfo.min_freq; ++ ++ /* Initial processor data capability frequencies */ ++ cpudata->max_freq = max_freq; ++ cpudata->min_freq = min_freq; ++ cpudata->nominal_freq = nominal_freq; ++ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; ++ ++ policy->driver_data = cpudata; ++ ++ update_boost_state(); ++ cpudata->epp_cached = amd_pstate_get_epp(cpudata, value); ++ ++ policy->min = policy->cpuinfo.min_freq; ++ policy->max = policy->cpuinfo.max_freq; ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) ++ policy->fast_switch_possible = true; ++ ++ if (!shared_mem && boot_cpu_has(X86_FEATURE_CPPC)) { ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); ++ if (ret) ++ return ret; ++ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); ++ if (ret) ++ return ret; ++ WRITE_ONCE(cpudata->cppc_cap1_cached, value); ++ } ++ amd_pstate_boost_init(cpudata); ++ ++ return 0; ++ ++free_cpudata1: ++ kfree(cpudata); ++ return ret; ++} ++ ++static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) ++{ ++ int ret; ++ ++ ret = __amd_pstate_cpu_init(policy); ++ if (ret) ++ return ret; ++ /* ++ * Set the policy to powersave to provide a valid fallback value in case ++ * the default cpufreq governor is neither powersave nor performance. ++ */ ++ policy->policy = CPUFREQ_POLICY_POWERSAVE; ++ ++ return 0; ++} ++ ++static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) ++{ ++ pr_debug("amd-pstate: CPU %d exiting\n", policy->cpu); ++ policy->fast_switch_possible = false; ++ return 0; ++} ++ ++static void amd_pstate_update_max_freq(unsigned int cpu) ++{ ++ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); ++ ++ if (!policy) ++ return; ++ ++ refresh_frequency_limits(policy); ++ cpufreq_cpu_release(policy); ++} ++ ++static void amd_pstate_epp_update_limits(unsigned int cpu) ++{ ++ mutex_lock(&amd_pstate_driver_lock); ++ update_boost_state(); ++ if (global_params.cppc_boost_disabled) { ++ for_each_possible_cpu(cpu) ++ amd_pstate_update_max_freq(cpu); ++ } else { ++ cpufreq_update_policy(cpu); ++ } ++ mutex_unlock(&amd_pstate_driver_lock); ++} ++ ++static int cppc_boost_hold_time_ns = 3 * NSEC_PER_MSEC; ++ ++static inline void amd_pstate_boost_up(struct amd_cpudata *cpudata) ++{ ++ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); ++ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); ++ u32 max_limit = (hwp_req & 0xff); ++ u32 min_limit = (hwp_req & 0xff00) >> 8; ++ u32 boost_level1; ++ ++ /* If max and min are equal or already at max, nothing to boost */ ++ if (max_limit == min_limit) ++ return; ++ ++ /* Set boost max and min to initial value */ ++ if (!cpudata->cppc_boost_min) ++ cpudata->cppc_boost_min = min_limit; ++ ++ boost_level1 = ((AMD_CPPC_NOMINAL_PERF(hwp_cap) + min_limit) >> 1); ++ ++ if (cpudata->cppc_boost_min < boost_level1) ++ cpudata->cppc_boost_min = boost_level1; ++ else if (cpudata->cppc_boost_min < AMD_CPPC_NOMINAL_PERF(hwp_cap)) ++ cpudata->cppc_boost_min = AMD_CPPC_NOMINAL_PERF(hwp_cap); ++ else if (cpudata->cppc_boost_min == AMD_CPPC_NOMINAL_PERF(hwp_cap)) ++ cpudata->cppc_boost_min = max_limit; ++ else ++ return; ++ ++ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); ++ hwp_req |= AMD_CPPC_MIN_PERF(cpudata->cppc_boost_min); ++ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); ++ cpudata->last_update = cpudata->sample.time; ++} ++ ++static inline void amd_pstate_boost_down(struct amd_cpudata *cpudata) ++{ ++ bool expired; ++ ++ if (cpudata->cppc_boost_min) { ++ expired = time_after64(cpudata->sample.time, cpudata->last_update + ++ cppc_boost_hold_time_ns); ++ ++ if (expired) { ++ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, ++ cpudata->cppc_req_cached); ++ cpudata->cppc_boost_min = 0; ++ } ++ } ++ ++ cpudata->last_update = cpudata->sample.time; ++} ++ ++static inline void amd_pstate_boost_update_util(struct amd_cpudata *cpudata, ++ u64 time) ++{ ++ cpudata->sample.time = time; ++ if (smp_processor_id() != cpudata->cpu) ++ return; ++ ++ if (cpudata->sched_flags & SCHED_CPUFREQ_IOWAIT) { ++ bool do_io = false; ++ ++ cpudata->sched_flags = 0; ++ /* ++ * Set iowait_boost flag and update time. Since IO WAIT flag ++ * is set all the time, we can't just conclude that there is ++ * some IO bound activity is scheduled on this CPU with just ++ * one occurrence. If we receive at least two in two ++ * consecutive ticks, then we treat as boost candidate. ++ * This is leveraged from Intel Pstate driver. ++ */ ++ if (time_before64(time, cpudata->last_io_update + 2 * TICK_NSEC)) ++ do_io = true; ++ ++ cpudata->last_io_update = time; ++ ++ if (do_io) ++ amd_pstate_boost_up(cpudata); ++ ++ } else { ++ amd_pstate_boost_down(cpudata); ++ } ++} ++ ++static inline void amd_pstate_cppc_update_hook(struct update_util_data *data, ++ u64 time, unsigned int flags) ++{ ++ struct amd_cpudata *cpudata = container_of(data, ++ struct amd_cpudata, update_util); ++ ++ cpudata->sched_flags |= flags; ++ ++ if (smp_processor_id() == cpudata->cpu) ++ amd_pstate_boost_update_util(cpudata, time); ++} ++ ++static void amd_pstate_clear_update_util_hook(unsigned int cpu) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[cpu]; ++ ++ if (!cpudata->update_util_set) ++ return; ++ ++ cpufreq_remove_update_util_hook(cpu); ++ cpudata->update_util_set = false; ++ synchronize_rcu(); ++} ++ ++static void amd_pstate_set_update_util_hook(unsigned int cpu_num) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[cpu_num]; ++ ++ if (!cppc_boost) { ++ if (cpudata->update_util_set) ++ amd_pstate_clear_update_util_hook(cpudata->cpu); ++ return; ++ } ++ ++ if (cpudata->update_util_set) ++ return; ++ ++ cpudata->sample.time = 0; ++ cpufreq_add_update_util_hook(cpu_num, &cpudata->update_util, ++ amd_pstate_cppc_update_hook); ++ cpudata->update_util_set = true; ++} ++ ++static void amd_pstate_epp_init(unsigned int cpu) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[cpu]; ++ u32 max_perf, min_perf; ++ u64 value; ++ s16 epp; ++ int ret; ++ ++ max_perf = READ_ONCE(cpudata->highest_perf); ++ min_perf = READ_ONCE(cpudata->lowest_perf); ++ ++ value = READ_ONCE(cpudata->cppc_req_cached); ++ ++ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) ++ min_perf = max_perf; ++ ++ /* Initial min/max values for CPPC Performance Controls Register */ ++ value &= ~AMD_CPPC_MIN_PERF(~0L); ++ value |= AMD_CPPC_MIN_PERF(min_perf); ++ ++ value &= ~AMD_CPPC_MAX_PERF(~0L); ++ value |= AMD_CPPC_MAX_PERF(max_perf); ++ ++ /* CPPC EPP feature require to set zero to the desire perf bit */ ++ value &= ~AMD_CPPC_DES_PERF(~0L); ++ value |= AMD_CPPC_DES_PERF(0); ++ ++ if (cpudata->epp_policy == cpudata->policy) ++ goto skip_epp; ++ ++ cpudata->epp_policy = cpudata->policy; ++ ++ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { ++ epp = amd_pstate_get_epp(cpudata, value); ++ cpudata->epp_powersave = epp; ++ if (epp < 0) ++ goto skip_epp; ++ /* force the epp value to be zero for performance policy */ ++ epp = 0; ++ } else { ++ if (cpudata->epp_powersave < 0) ++ goto skip_epp; ++ /* Get BIOS pre-defined epp value */ ++ epp = amd_pstate_get_epp(cpudata, value); ++ if (epp) ++ goto skip_epp; ++ epp = cpudata->epp_powersave; ++ } ++ /* Set initial EPP value */ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ value &= ~GENMASK_ULL(31, 24); ++ value |= (u64)epp << 24; ++ } ++ ++skip_epp: ++ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ if (!ret) ++ cpudata->epp_cached = epp; ++} ++ ++static void amd_pstate_set_max_limits(struct amd_cpudata *cpudata) ++{ ++ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); ++ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); ++ u32 max_limit = (hwp_cap >> 24) & 0xff; ++ ++ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); ++ hwp_req |= AMD_CPPC_MIN_PERF(max_limit); ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); ++} ++ ++static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) ++{ ++ struct amd_cpudata *cpudata; ++ ++ if (!policy->cpuinfo.max_freq) ++ return -ENODEV; ++ ++ pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", ++ policy->cpuinfo.max_freq, policy->max); ++ ++ cpudata = all_cpu_data[policy->cpu]; ++ cpudata->policy = policy->policy; ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ mutex_lock(&amd_pstate_limits_lock); ++ ++ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { ++ amd_pstate_clear_update_util_hook(policy->cpu); ++ amd_pstate_set_max_limits(cpudata); ++ } else { ++ amd_pstate_set_update_util_hook(policy->cpu); ++ } ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) ++ amd_pstate_epp_init(policy->cpu); ++ ++ mutex_unlock(&amd_pstate_limits_lock); ++ } ++ ++ return 0; ++} ++ ++static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) ++{ ++ struct cppc_perf_ctrls perf_ctrls; ++ u64 value, max_perf; ++ int ret; ++ ++ ret = amd_pstate_enable(true); ++ if (ret) ++ pr_err("failed to enable amd pstate during resume, return %d\n", ret); ++ ++ value = READ_ONCE(cpudata->cppc_req_cached); ++ max_perf = READ_ONCE(cpudata->highest_perf); ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ } else { ++ perf_ctrls.max_perf = max_perf; ++ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); ++ cppc_set_perf(cpudata->cpu, &perf_ctrls); ++ } ++} ++ ++static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ ++ pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); ++ ++ if (epp_enabled) { ++ amd_pstate_epp_reenable(cpudata); ++ cpudata->suspended = false; ++ } ++ ++ return 0; ++} ++ ++static void amd_pstate_epp_offline(struct cpufreq_policy *policy) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ struct cppc_perf_ctrls perf_ctrls; ++ int min_perf; ++ u64 value; ++ ++ min_perf = READ_ONCE(cpudata->lowest_perf); ++ value = READ_ONCE(cpudata->cppc_req_cached); ++ ++ mutex_lock(&amd_pstate_limits_lock); ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; ++ ++ /* Set max perf same as min perf */ ++ value &= ~AMD_CPPC_MAX_PERF(~0L); ++ value |= AMD_CPPC_MAX_PERF(min_perf); ++ value &= ~AMD_CPPC_MIN_PERF(~0L); ++ value |= AMD_CPPC_MIN_PERF(min_perf); ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ } else { ++ perf_ctrls.desired_perf = 0; ++ perf_ctrls.max_perf = min_perf; ++ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(AMD_CPPC_EPP_POWERSAVE); ++ cppc_set_perf(cpudata->cpu, &perf_ctrls); ++ } ++ mutex_unlock(&amd_pstate_limits_lock); ++} ++ ++static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ ++ pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); ++ ++ if (cpudata->suspended) ++ return 0; ++ ++ if (pstate_exiting) ++ return 0; ++ ++ if (epp_enabled) ++ amd_pstate_epp_offline(policy); ++ ++ return 0; ++} ++ ++static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) ++{ ++ amd_pstate_clear_update_util_hook(policy->cpu); ++ ++ return amd_pstate_cpu_offline(policy); ++} ++ ++static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ int ret; ++ ++ /* avoid suspending when EPP is not enabled */ ++ if (!epp_enabled) ++ return 0; ++ ++ /* set this flag to avoid setting core offline*/ ++ cpudata->suspended = true; ++ ++ /* disable CPPC in lowlevel firmware */ ++ ret = amd_pstate_enable(false); ++ if (ret) ++ pr_err("failed to suspend, return %d\n", ret); ++ ++ return 0; ++} ++ ++static int amd_pstate_epp_resume(struct cpufreq_policy *policy) ++{ ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ ++ if (cpudata->suspended) { ++ mutex_lock(&amd_pstate_limits_lock); ++ ++ /* enable amd pstate from suspend state*/ ++ amd_pstate_epp_reenable(cpudata); ++ ++ mutex_unlock(&amd_pstate_limits_lock); ++ ++ cpudata->suspended = false; ++ } ++ ++ return 0; ++} ++ ++static void amd_pstate_verify_cpu_policy(struct amd_cpudata *cpudata, ++ struct cpufreq_policy_data *policy) ++{ ++ update_boost_state(); ++ cpufreq_verify_within_cpu_limits(policy); ++} ++ ++static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) ++{ ++ amd_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); ++ pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); ++ return 0; ++} ++ + static struct cpufreq_driver amd_pstate_driver = { + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, + .verify = amd_pstate_verify, +@@ -662,11 +1535,27 @@ static struct cpufreq_driver amd_pstate_driver = { + .resume = amd_pstate_cpu_resume, + .set_boost = amd_pstate_set_boost, + .name = "amd-pstate", +- .attr = amd_pstate_attr, ++ .attr = amd_pstate_attr, ++}; ++ ++static struct cpufreq_driver amd_pstate_epp_driver = { ++ .flags = CPUFREQ_CONST_LOOPS, ++ .verify = amd_pstate_epp_verify_policy, ++ .setpolicy = amd_pstate_epp_set_policy, ++ .init = amd_pstate_epp_cpu_init, ++ .exit = amd_pstate_epp_cpu_exit, ++ .update_limits = amd_pstate_epp_update_limits, ++ .offline = amd_pstate_epp_cpu_offline, ++ .online = amd_pstate_epp_cpu_online, ++ .suspend = amd_pstate_epp_suspend, ++ .resume = amd_pstate_epp_resume, ++ .name = "amd_pstate_epp", ++ .attr = amd_pstate_epp_attr, + }; + + static int __init amd_pstate_init(void) + { ++ static struct amd_cpudata **cpudata; + int ret; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) +@@ -681,10 +1570,24 @@ static int __init amd_pstate_init(void) + if (cpufreq_get_current_driver()) + return -EEXIST; + ++ cpudata = vzalloc(array_size(sizeof(void *), num_possible_cpus())); ++ if (!cpudata) ++ return -ENOMEM; ++ WRITE_ONCE(all_cpu_data, cpudata); ++ ++ if (epp_enabled) { ++ pr_info("AMD CPPC loading with amd_pstate_epp driver instance.\n"); ++ default_pstate_driver = &amd_pstate_epp_driver; ++ } else { ++ pr_info("AMD CPPC loading with amd_pstate driver instance.\n"); ++ default_pstate_driver = &amd_pstate_driver; ++ } ++ + /* capability check */ + if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ if (!epp_enabled) ++ default_pstate_driver->adjust_perf = amd_pstate_adjust_perf; + pr_debug("AMD CPPC MSR based functionality is supported\n"); +- amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; + } else if (shared_mem) { + static_call_update(amd_pstate_enable, cppc_enable); + static_call_update(amd_pstate_init_perf, cppc_init_perf); +@@ -701,19 +1604,57 @@ static int __init amd_pstate_init(void) + return ret; + } + +- ret = cpufreq_register_driver(&amd_pstate_driver); ++ ret = cpufreq_register_driver(default_pstate_driver); + if (ret) +- pr_err("failed to register amd_pstate_driver with return %d\n", ++ pr_err("failed to register amd pstate driver with return %d\n", ++ ret); ++ ++ amd_pstate_kobj = kobject_create_and_add("amd-pstate", &cpu_subsys.dev_root->kobj); ++ if (!amd_pstate_kobj) ++ pr_err("amd-pstate: Global sysfs registration failed.\n"); ++ ++ ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); ++ if (ret) { ++ pr_err("amd-pstate: Sysfs attribute export failed with error %d.\n", + ret); ++ } ++ pstate_exiting = false; + + return ret; + } + ++static inline void amd_pstate_kobj_cleanup(struct kobject *kobj) ++{ ++ kobject_del(kobj); ++ kobject_put(kobj); ++} ++ + static void __exit amd_pstate_exit(void) + { +- cpufreq_unregister_driver(&amd_pstate_driver); ++ unsigned int cpu; ++ ++ pstate_exiting = true; ++ cpufreq_unregister_driver(default_pstate_driver); + + amd_pstate_enable(false); ++ ++ sysfs_remove_group(amd_pstate_kobj, &amd_pstate_global_attr_group); ++ amd_pstate_kobj_cleanup(amd_pstate_kobj); ++ ++ cpus_read_lock(); ++ for_each_online_cpu(cpu) { ++ if (all_cpu_data[cpu]) { ++ if (default_pstate_driver == &amd_pstate_epp_driver) ++ amd_pstate_clear_update_util_hook(cpu); ++ ++ spin_lock(&amd_pstate_cpu_lock); ++ kfree(all_cpu_data[cpu]); ++ WRITE_ONCE(all_cpu_data[cpu], NULL); ++ spin_unlock(&amd_pstate_cpu_lock); ++ } ++ } ++ cpus_read_unlock(); ++ + } + + module_init(amd_pstate_init); +diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c +index f7c23fa468f0..39221a9a187a 100644 +--- a/drivers/cpufreq/bmips-cpufreq.c ++++ b/drivers/cpufreq/bmips-cpufreq.c +@@ -156,7 +156,7 @@ static struct cpufreq_driver bmips_cpufreq_driver = { + .name = BMIPS_CPUFREQ_PREFIX, + }; + +-static int __init bmips_cpufreq_probe(void) ++static int __init bmips_cpufreq_driver_init(void) + { + struct cpufreq_compat *cc; + struct device_node *np; +@@ -176,7 +176,13 @@ static int __init bmips_cpufreq_probe(void) + + return cpufreq_register_driver(&bmips_cpufreq_driver); + } +-device_initcall(bmips_cpufreq_probe); ++module_init(bmips_cpufreq_driver_init); ++ ++static void __exit bmips_cpufreq_driver_exit(void) ++{ ++ cpufreq_unregister_driver(&bmips_cpufreq_driver); ++} ++module_exit(bmips_cpufreq_driver_exit); + + MODULE_AUTHOR("Markus Mayer "); + MODULE_DESCRIPTION("CPUfreq driver for Broadcom BMIPS SoCs"); +diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c +index 2c96de3f2d83..6ac3800db450 100644 +--- a/drivers/cpufreq/cpufreq-dt-platdev.c ++++ b/drivers/cpufreq/cpufreq-dt-platdev.c +@@ -146,6 +146,7 @@ static const struct of_device_id blocklist[] __initconst = { + { .compatible = "qcom,sc8180x", }, + { .compatible = "qcom,sc8280xp", }, + { .compatible = "qcom,sdm845", }, ++ { .compatible = "qcom,sm6115", }, + { .compatible = "qcom,sm6350", }, + { .compatible = "qcom,sm8150", }, + { .compatible = "qcom,sm8250", }, +diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c +index 69b3d61852ac..a491fea4985e 100644 +--- a/drivers/cpufreq/cpufreq.c ++++ b/drivers/cpufreq/cpufreq.c +@@ -262,6 +262,7 @@ void cpufreq_cpu_release(struct cpufreq_policy *policy) + + cpufreq_cpu_put(policy); + } ++EXPORT_SYMBOL_GPL(cpufreq_cpu_release); + + /** + * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it. +@@ -291,6 +292,7 @@ struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu) + + return policy; + } ++EXPORT_SYMBOL_GPL(cpufreq_cpu_acquire); + + /********************************************************************* + * EXTERNALLY AFFECTING FREQUENCY CHANGES * +diff --git a/drivers/cpufreq/highbank-cpufreq.c b/drivers/cpufreq/highbank-cpufreq.c +index ac57cddc5f2f..a45864701143 100644 +--- a/drivers/cpufreq/highbank-cpufreq.c ++++ b/drivers/cpufreq/highbank-cpufreq.c +@@ -55,7 +55,7 @@ static struct notifier_block hb_cpufreq_clk_nb = { + .notifier_call = hb_cpufreq_clk_notify, + }; + +-static int hb_cpufreq_driver_init(void) ++static int __init hb_cpufreq_driver_init(void) + { + struct platform_device_info devinfo = { .name = "cpufreq-dt", }; + struct device *cpu_dev; +diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c +index 57cdb3679885..fc3ebeb0bbe5 100644 +--- a/drivers/cpufreq/intel_pstate.c ++++ b/drivers/cpufreq/intel_pstate.c +@@ -2416,6 +2416,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = { + X86_MATCH(SKYLAKE_X, core_funcs), + X86_MATCH(COMETLAKE, core_funcs), + X86_MATCH(ICELAKE_X, core_funcs), ++ X86_MATCH(TIGERLAKE, core_funcs), + {} + }; + MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); +diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c +index d5ef3c66c762..833589bc95e4 100644 +--- a/drivers/cpufreq/qcom-cpufreq-hw.c ++++ b/drivers/cpufreq/qcom-cpufreq-hw.c +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -56,6 +57,8 @@ struct qcom_cpufreq_data { + struct cpufreq_policy *policy; + + bool per_core_dcvs; ++ ++ struct freq_qos_request throttle_freq_req; + }; + + static unsigned long cpu_hw_rate, xo_rate; +@@ -316,14 +319,16 @@ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) + if (IS_ERR(opp)) { + dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); + } else { +- throttled_freq = freq_hz / HZ_PER_KHZ; +- +- /* Update thermal pressure (the boost frequencies are accepted) */ +- arch_update_thermal_pressure(policy->related_cpus, throttled_freq); +- + dev_pm_opp_put(opp); + } + ++ throttled_freq = freq_hz / HZ_PER_KHZ; ++ ++ freq_qos_update_request(&data->throttle_freq_req, throttled_freq); ++ ++ /* Update thermal pressure (the boost frequencies are accepted) */ ++ arch_update_thermal_pressure(policy->related_cpus, throttled_freq); ++ + /* + * In the unlikely case policy is unregistered do not enable + * polling or h/w interrupt +@@ -413,6 +418,14 @@ static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) + if (data->throttle_irq < 0) + return data->throttle_irq; + ++ ret = freq_qos_add_request(&policy->constraints, ++ &data->throttle_freq_req, FREQ_QOS_MAX, ++ FREQ_QOS_MAX_DEFAULT_VALUE); ++ if (ret < 0) { ++ dev_err(&pdev->dev, "Failed to add freq constraint (%d)\n", ret); ++ return ret; ++ } ++ + data->cancel_throttle = false; + data->policy = policy; + +@@ -479,6 +492,7 @@ static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) + if (data->throttle_irq <= 0) + return; + ++ freq_qos_remove_request(&data->throttle_freq_req); + free_irq(data->throttle_irq, data); + } + +diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c +index a67df90848c2..1a63aeea8711 100644 +--- a/drivers/cpufreq/sti-cpufreq.c ++++ b/drivers/cpufreq/sti-cpufreq.c +@@ -252,7 +252,7 @@ static int sti_cpufreq_fetch_syscon_registers(void) + return 0; + } + +-static int sti_cpufreq_init(void) ++static int __init sti_cpufreq_init(void) + { + int ret; + +diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c +index 1216046cf4c2..c2004cae3f02 100644 +--- a/drivers/cpufreq/tegra194-cpufreq.c ++++ b/drivers/cpufreq/tegra194-cpufreq.c +@@ -38,14 +38,6 @@ + /* cpufreq transisition latency */ + #define TEGRA_CPUFREQ_TRANSITION_LATENCY (300 * 1000) /* unit in nanoseconds */ + +-enum cluster { +- CLUSTER0, +- CLUSTER1, +- CLUSTER2, +- CLUSTER3, +- MAX_CLUSTERS, +-}; +- + struct tegra_cpu_ctr { + u32 cpu; + u32 coreclk_cnt, last_coreclk_cnt; +@@ -67,12 +59,12 @@ struct tegra_cpufreq_ops { + struct tegra_cpufreq_soc { + struct tegra_cpufreq_ops *ops; + int maxcpus_per_cluster; ++ unsigned int num_clusters; + phys_addr_t actmon_cntr_base; + }; + + struct tegra194_cpufreq_data { + void __iomem *regs; +- size_t num_clusters; + struct cpufreq_frequency_table **tables; + const struct tegra_cpufreq_soc *soc; + }; +@@ -166,6 +158,14 @@ static const struct tegra_cpufreq_soc tegra234_cpufreq_soc = { + .ops = &tegra234_cpufreq_ops, + .actmon_cntr_base = 0x9000, + .maxcpus_per_cluster = 4, ++ .num_clusters = 3, ++}; ++ ++static const struct tegra_cpufreq_soc tegra239_cpufreq_soc = { ++ .ops = &tegra234_cpufreq_ops, ++ .actmon_cntr_base = 0x4000, ++ .maxcpus_per_cluster = 8, ++ .num_clusters = 1, + }; + + static void tegra194_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid) +@@ -314,11 +314,7 @@ static void tegra194_get_cpu_ndiv_sysreg(void *ndiv) + + static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv) + { +- int ret; +- +- ret = smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true); +- +- return ret; ++ return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true); + } + + static void tegra194_set_cpu_ndiv_sysreg(void *data) +@@ -382,7 +378,7 @@ static int tegra194_cpufreq_init(struct cpufreq_policy *policy) + + data->soc->ops->get_cpu_cluster_id(policy->cpu, NULL, &clusterid); + +- if (clusterid >= data->num_clusters || !data->tables[clusterid]) ++ if (clusterid >= data->soc->num_clusters || !data->tables[clusterid]) + return -EINVAL; + + start_cpu = rounddown(policy->cpu, maxcpus_per_cluster); +@@ -433,6 +429,7 @@ static struct tegra_cpufreq_ops tegra194_cpufreq_ops = { + static const struct tegra_cpufreq_soc tegra194_cpufreq_soc = { + .ops = &tegra194_cpufreq_ops, + .maxcpus_per_cluster = 2, ++ .num_clusters = 4, + }; + + static void tegra194_cpufreq_free_resources(void) +@@ -525,15 +522,14 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev) + + soc = of_device_get_match_data(&pdev->dev); + +- if (soc->ops && soc->maxcpus_per_cluster) { ++ if (soc->ops && soc->maxcpus_per_cluster && soc->num_clusters) { + data->soc = soc; + } else { + dev_err(&pdev->dev, "soc data missing\n"); + return -EINVAL; + } + +- data->num_clusters = MAX_CLUSTERS; +- data->tables = devm_kcalloc(&pdev->dev, data->num_clusters, ++ data->tables = devm_kcalloc(&pdev->dev, data->soc->num_clusters, + sizeof(*data->tables), GFP_KERNEL); + if (!data->tables) + return -ENOMEM; +@@ -558,7 +554,7 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev) + goto put_bpmp; + } + +- for (i = 0; i < data->num_clusters; i++) { ++ for (i = 0; i < data->soc->num_clusters; i++) { + data->tables[i] = init_freq_table(pdev, bpmp, i); + if (IS_ERR(data->tables[i])) { + err = PTR_ERR(data->tables[i]); +@@ -590,6 +586,7 @@ static int tegra194_cpufreq_remove(struct platform_device *pdev) + static const struct of_device_id tegra194_cpufreq_of_match[] = { + { .compatible = "nvidia,tegra194-ccplex", .data = &tegra194_cpufreq_soc }, + { .compatible = "nvidia,tegra234-ccplex-cluster", .data = &tegra234_cpufreq_soc }, ++ { .compatible = "nvidia,tegra239-ccplex-cluster", .data = &tegra239_cpufreq_soc }, + { /* sentinel */ } + }; + +diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c +index df85a77d476b..f64180dd2005 100644 +--- a/drivers/cpufreq/ti-cpufreq.c ++++ b/drivers/cpufreq/ti-cpufreq.c +@@ -398,7 +398,7 @@ static int ti_cpufreq_probe(struct platform_device *pdev) + return ret; + } + +-static int ti_cpufreq_init(void) ++static int __init ti_cpufreq_init(void) + { + const struct of_device_id *match; + +diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c +index 159c6806c19b..a201f1c2fce5 100644 +--- a/drivers/md/dm-crypt.c ++++ b/drivers/md/dm-crypt.c +@@ -3137,6 +3137,11 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar + } + } + ++#ifdef CONFIG_CACHY ++ set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); ++ set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); ++#endif ++ + return 0; + } + +diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h +index f73d357ecdf5..41d151a6d52e 100644 +--- a/include/acpi/cppc_acpi.h ++++ b/include/acpi/cppc_acpi.h +@@ -108,12 +108,14 @@ struct cppc_perf_caps { + u32 lowest_nonlinear_perf; + u32 lowest_freq; + u32 nominal_freq; ++ u32 energy_perf; + }; + + struct cppc_perf_ctrls { + u32 max_perf; + u32 min_perf; + u32 desired_perf; ++ u32 energy_perf; + }; + + struct cppc_perf_fb_ctrs { +@@ -148,6 +150,9 @@ extern bool cpc_ffh_supported(void); + extern bool cpc_supported_by_cpu(void); + extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); + extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); ++extern int cppc_set_auto_epp(int cpu, bool enable); ++extern int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps); ++extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); + #else /* !CONFIG_ACPI_CPPC_LIB */ + static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) + { +@@ -197,6 +202,18 @@ static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) + { + return -ENOTSUPP; + } ++static inline int cppc_set_auto_epp(int cpu, bool enable) ++{ ++ return -ENOTSUPP; ++} ++static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) ++{ ++ return -ENOTSUPP; ++} ++static inline int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) ++{ ++ return -ENOTSUPP; ++} + #endif /* !CONFIG_ACPI_CPPC_LIB */ + + #endif /* _CPPC_ACPI_H*/ +diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h +index 0178b2040ea3..39f05e00dac4 100644 +--- a/include/linux/pagemap.h ++++ b/include/linux/pagemap.h +@@ -1183,7 +1183,7 @@ struct readahead_control { + ._index = i, \ + } + +-#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) ++#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) + + void page_cache_ra_unbounded(struct readahead_control *, + unsigned long nr_to_read, unsigned long lookahead_count); +diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h +index 33a4240e6a6f..82213f9c4c17 100644 +--- a/include/linux/user_namespace.h ++++ b/include/linux/user_namespace.h +@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns, + + #ifdef CONFIG_USER_NS + ++extern int unprivileged_userns_clone; ++ + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) + { + if (ns) +@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); + struct ns_common *ns_get_owner(struct ns_common *ns); + #else + ++#define unprivileged_userns_clone 0 ++ + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) + { + return &init_user_ns; +diff --git a/init/Kconfig b/init/Kconfig +index 532362fcfe31..f5bd72b39352 100644 +--- a/init/Kconfig ++++ b/init/Kconfig +@@ -112,6 +112,10 @@ config THREAD_INFO_IN_TASK + + menu "General setup" + ++config CACHY ++ bool "Some kernel tweaks by CachyOS" ++ default y ++ + config BROKEN + bool + +@@ -334,6 +338,19 @@ config KERNEL_UNCOMPRESSED + + endchoice + ++menu "ZSTD compression options" ++ depends on KERNEL_ZSTD ++ ++config ZSTD_COMP_VAL ++ int "Compression level (1-22)" ++ range 1 22 ++ default "22" ++ help ++ Choose a compression level for zstd kernel compression. ++ Default is 22, which is the maximum. ++ ++endmenu ++ + config DEFAULT_INIT + string "Default init path" + default "" +@@ -1241,6 +1258,22 @@ config USER_NS + + If unsure, say N. + ++config USER_NS_UNPRIVILEGED ++ bool "Allow unprivileged users to create namespaces" ++ default y ++ depends on USER_NS ++ help ++ When disabled, unprivileged users will not be able to create ++ new namespaces. Allowing users to create their own namespaces ++ has been part of several recent local privilege escalation ++ exploits, so if you need user namespaces but are ++ paranoid^Wsecurity-conscious you want to disable this. ++ ++ This setting can be overridden at runtime via the ++ kernel.unprivileged_userns_clone sysctl. ++ ++ If unsure, say Y. ++ + config PID_NS + bool "PID Namespaces" + default y +@@ -1407,6 +1440,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE + with the "-O2" compiler flag for best performance and most + helpful compile-time warnings. + ++config CC_OPTIMIZE_FOR_PERFORMANCE_O3 ++ bool "Optimize more for performance (-O3)" ++ help ++ Choosing this option will pass "-O3" to your compiler to optimize ++ the kernel yet more for performance. ++ + config CC_OPTIMIZE_FOR_SIZE + bool "Optimize for size (-Os)" + help +diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz +index 38ef6d06888e..0f78364efd4f 100644 +--- a/kernel/Kconfig.hz ++++ b/kernel/Kconfig.hz +@@ -40,6 +40,27 @@ choice + on SMP and NUMA systems and exactly dividing by both PAL and + NTSC frame rates for video and multimedia work. + ++ config HZ_500 ++ bool "500 HZ" ++ help ++ 500 Hz is a balanced timer frequency. Provides fast interactivity ++ on desktops with good smoothness without increasing CPU power ++ consumption and sacrificing the battery life on laptops. ++ ++ config HZ_600 ++ bool "600 HZ" ++ help ++ 600 Hz is a balanced timer frequency. Provides fast interactivity ++ on desktops with good smoothness without increasing CPU power ++ consumption and sacrificing the battery life on laptops. ++ ++ config HZ_750 ++ bool "750 HZ" ++ help ++ 750 Hz is a balanced timer frequency. Provides fast interactivity ++ on desktops with good smoothness without increasing CPU power ++ consumption and sacrificing the battery life on laptops. ++ + config HZ_1000 + bool "1000 HZ" + help +@@ -53,6 +74,9 @@ config HZ + default 100 if HZ_100 + default 250 if HZ_250 + default 300 if HZ_300 ++ default 500 if HZ_500 ++ default 600 if HZ_600 ++ default 750 if HZ_750 + default 1000 if HZ_1000 + + config SCHED_HRTICK +diff --git a/kernel/fork.c b/kernel/fork.c +index 2b6bd511c6ed..704fe6bc9cb4 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -99,6 +99,10 @@ + #include + #include + ++#ifdef CONFIG_USER_NS ++#include ++#endif ++ + #include + #include + #include +@@ -2009,6 +2013,10 @@ static __latent_entropy struct task_struct *copy_process( + if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) + return ERR_PTR(-EINVAL); + ++ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) ++ if (!capable(CAP_SYS_ADMIN)) ++ return ERR_PTR(-EPERM); ++ + /* + * Thread groups must share signals as well, and detached threads + * can only be started up within the thread group. +@@ -3159,6 +3167,12 @@ int ksys_unshare(unsigned long unshare_flags) + if (unshare_flags & CLONE_NEWNS) + unshare_flags |= CLONE_FS; + ++ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { ++ err = -EPERM; ++ if (!capable(CAP_SYS_ADMIN)) ++ goto bad_unshare_out; ++ } ++ + err = check_unshare_flags(unshare_flags); + if (err) + goto bad_unshare_out; +diff --git a/kernel/module/Kconfig b/kernel/module/Kconfig +index 26ea5d04f56c..e5311101b93d 100644 +--- a/kernel/module/Kconfig ++++ b/kernel/module/Kconfig +@@ -219,6 +219,31 @@ config MODULE_COMPRESS_ZSTD + + endchoice + ++menu "ZSTD module compression options" ++ depends on MODULE_COMPRESS_ZSTD ++ ++config MODULE_COMPRESS_ZSTD_LEVEL ++ int "Compression level (1-19)" ++ range 1 19 ++ default 9 ++ help ++ Compression level used by zstd for compressing modules. ++ ++config MODULE_COMPRESS_ZSTD_ULTRA ++ bool "Enable ZSTD ultra compression" ++ help ++ Compress modules with ZSTD using the highest possible compression. ++ ++config MODULE_COMPRESS_ZSTD_LEVEL_ULTRA ++ int "Compression level (20-22)" ++ depends on MODULE_COMPRESS_ZSTD_ULTRA ++ range 20 22 ++ default 20 ++ help ++ Ultra compression level used by zstd for compressing modules. ++ ++endmenu ++ + config MODULE_DECOMPRESS + bool "Support in-kernel module decompression" + depends on MODULE_COMPRESS_GZIP || MODULE_COMPRESS_XZ +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index ee28253c9ac0..29f1fff78059 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -138,6 +138,18 @@ __read_mostly int sysctl_resched_latency_warn_ms = 100; + __read_mostly int sysctl_resched_latency_warn_once = 1; + #endif /* CONFIG_SCHED_DEBUG */ + ++/* ++ * Choose the yield level that will perform. ++ * 0: No yield. ++ * 1: Yield only to better priority/deadline tasks. ++ * 2: Re-queue current tasks. (default CFS) ++ */ ++#ifdef CONFIG_CACHY ++__read_mostly int sysctl_sched_yield_type = 1; ++#else ++__read_mostly int sysctl_sched_yield_type = 0; ++#endif ++ + /* + * Number of tasks to iterate in a single balance run. + * Limited because this is done with IRQs disabled. +@@ -728,7 +740,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) + if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) + update_irq_load_avg(rq, irq_delta + steal); + #endif +- update_rq_clock_pelt(rq, delta); ++ update_rq_clock_task_mult(rq, delta); + } + + void update_rq_clock(struct rq *rq) +@@ -8266,10 +8278,15 @@ static void do_sched_yield(void) + struct rq_flags rf; + struct rq *rq; + ++ if (!sysctl_sched_yield_type) ++ return; ++ + rq = this_rq_lock_irq(&rf); + + schedstat_inc(rq->yld_count); +- current->sched_class->yield_task(rq); ++ ++ if (sysctl_sched_yield_type > 1) ++ current->sched_class->yield_task(rq); + + preempt_disable(); + rq_unlock_irq(rq, &rf); +diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c +index 1207c78f85c1..9161d1136d01 100644 +--- a/kernel/sched/cpufreq_schedutil.c ++++ b/kernel/sched/cpufreq_schedutil.c +@@ -25,6 +25,9 @@ struct sugov_policy { + unsigned int next_freq; + unsigned int cached_raw_freq; + ++ /* max CPU capacity, which is equal for all CPUs in freq. domain */ ++ unsigned long max; ++ + /* The next fields are only needed if fast switch cannot be used: */ + struct irq_work irq_work; + struct kthread_work work; +@@ -48,7 +51,6 @@ struct sugov_cpu { + + unsigned long util; + unsigned long bw_dl; +- unsigned long max; + + /* The field below is for single-CPU policies only: */ + #ifdef CONFIG_NO_HZ_COMMON +@@ -158,7 +160,6 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu) + { + struct rq *rq = cpu_rq(sg_cpu->cpu); + +- sg_cpu->max = arch_scale_cpu_capacity(sg_cpu->cpu); + sg_cpu->bw_dl = cpu_bw_dl(rq); + sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(sg_cpu->cpu), + FREQUENCY_UTIL, NULL); +@@ -253,6 +254,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, + */ + static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time) + { ++ struct sugov_policy *sg_policy = sg_cpu->sg_policy; + unsigned long boost; + + /* No boost currently required */ +@@ -280,7 +282,8 @@ static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time) + * sg_cpu->util is already in capacity scale; convert iowait_boost + * into the same scale so we can compare. + */ +- boost = (sg_cpu->iowait_boost * sg_cpu->max) >> SCHED_CAPACITY_SHIFT; ++ boost = sg_cpu->iowait_boost * sg_policy->max; ++ boost >>= SCHED_CAPACITY_SHIFT; + boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL); + if (sg_cpu->util < boost) + sg_cpu->util = boost; +@@ -337,7 +340,7 @@ static void sugov_update_single_freq(struct update_util_data *hook, u64 time, + if (!sugov_update_single_common(sg_cpu, time, flags)) + return; + +- next_f = get_next_freq(sg_policy, sg_cpu->util, sg_cpu->max); ++ next_f = get_next_freq(sg_policy, sg_cpu->util, sg_policy->max); + /* + * Do not reduce the frequency if the CPU has not been idle + * recently, as the reduction is likely to be premature then. +@@ -373,6 +376,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, + unsigned int flags) + { + struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); ++ struct sugov_policy *sg_policy = sg_cpu->sg_policy; + unsigned long prev_util = sg_cpu->util; + + /* +@@ -399,7 +403,8 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, + sg_cpu->util = prev_util; + + cpufreq_driver_adjust_perf(sg_cpu->cpu, map_util_perf(sg_cpu->bw_dl), +- map_util_perf(sg_cpu->util), sg_cpu->max); ++ map_util_perf(sg_cpu->util), ++ sg_policy->max); + + sg_cpu->sg_policy->last_freq_update_time = time; + } +@@ -408,25 +413,19 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) + { + struct sugov_policy *sg_policy = sg_cpu->sg_policy; + struct cpufreq_policy *policy = sg_policy->policy; +- unsigned long util = 0, max = 1; ++ unsigned long util = 0; + unsigned int j; + + for_each_cpu(j, policy->cpus) { + struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); +- unsigned long j_util, j_max; + + sugov_get_util(j_sg_cpu); + sugov_iowait_apply(j_sg_cpu, time); +- j_util = j_sg_cpu->util; +- j_max = j_sg_cpu->max; + +- if (j_util * max > j_max * util) { +- util = j_util; +- max = j_max; +- } ++ util = max(j_sg_cpu->util, util); + } + +- return get_next_freq(sg_policy, util, max); ++ return get_next_freq(sg_policy, util, sg_policy->max); + } + + static void +@@ -752,7 +751,7 @@ static int sugov_start(struct cpufreq_policy *policy) + { + struct sugov_policy *sg_policy = policy->governor_data; + void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); +- unsigned int cpu; ++ unsigned int cpu = cpumask_first(policy->cpus); + + sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; + sg_policy->last_freq_update_time = 0; +@@ -760,6 +759,7 @@ static int sugov_start(struct cpufreq_policy *policy) + sg_policy->work_in_progress = false; + sg_policy->limits_changed = false; + sg_policy->cached_raw_freq = 0; ++ sg_policy->max = arch_scale_cpu_capacity(cpu); + + sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); + +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index 914096c5b1ae..c4fd77e7e8f3 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -68,9 +68,13 @@ + * + * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) + */ ++#ifdef CONFIG_CACHY ++unsigned int sysctl_sched_latency = 3000000ULL; ++static unsigned int normalized_sysctl_sched_latency = 3000000ULL; ++#else + unsigned int sysctl_sched_latency = 6000000ULL; + static unsigned int normalized_sysctl_sched_latency = 6000000ULL; +- ++#endif + /* + * The initial- and re-scaling of tunables is configurable + * +@@ -89,8 +93,13 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; + * + * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) + */ ++#ifdef CONFIG_CACHY ++unsigned int sysctl_sched_min_granularity = 400000ULL; ++static unsigned int normalized_sysctl_sched_min_granularity = 400000ULL; ++#else + unsigned int sysctl_sched_min_granularity = 750000ULL; + static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; ++#endif + + /* + * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. +@@ -120,8 +129,13 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; + * + * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) + */ ++#ifdef CONFIG_CACHY ++unsigned int sysctl_sched_wakeup_granularity = 500000UL; ++static unsigned int normalized_sysctl_sched_wakeup_granularity = 500000UL; ++#else + unsigned int sysctl_sched_wakeup_granularity = 1000000UL; + static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; ++#endif + + const_debug unsigned int sysctl_sched_migration_cost = 500000UL; + +@@ -174,8 +188,12 @@ int __weak arch_asym_cpu_priority(int cpu) + * + * (default: 5 msec, units: microseconds) + */ ++#ifdef CONFIG_CACHY ++static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; ++#else + static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; + #endif ++#endif + + #ifdef CONFIG_SYSCTL + static struct ctl_table sched_fair_sysctls[] = { +@@ -4573,7 +4591,13 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) + struct sched_entity *se; + s64 delta; + +- ideal_runtime = sched_slice(cfs_rq, curr); ++ /* ++ * When many tasks blow up the sched_period; it is possible that ++ * sched_slice() reports unusually large results (when many tasks are ++ * very light for example). Therefore impose a maximum. ++ */ ++ ideal_runtime = min_t(u64, sched_slice(cfs_rq, curr), sysctl_sched_latency); ++ + delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + if (delta_exec > ideal_runtime) { + resched_curr(rq_of(cfs_rq)); +diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c +index 0f310768260c..036b0e2cd2b4 100644 +--- a/kernel/sched/pelt.c ++++ b/kernel/sched/pelt.c +@@ -467,3 +467,63 @@ int update_irq_load_avg(struct rq *rq, u64 running) + return ret; + } + #endif ++ ++__read_mostly unsigned int sched_pelt_lshift; ++ ++#ifdef CONFIG_SYSCTL ++static unsigned int sysctl_sched_pelt_multiplier = 1; ++ ++int sched_pelt_multiplier(struct ctl_table *table, int write, void *buffer, ++ size_t *lenp, loff_t *ppos) ++{ ++ static DEFINE_MUTEX(mutex); ++ unsigned int old; ++ int ret; ++ ++ mutex_lock(&mutex); ++ old = sysctl_sched_pelt_multiplier; ++ ret = proc_dointvec(table, write, buffer, lenp, ppos); ++ if (ret) ++ goto undo; ++ if (!write) ++ goto done; ++ ++ switch (sysctl_sched_pelt_multiplier) { ++ case 1: ++ fallthrough; ++ case 2: ++ fallthrough; ++ case 4: ++ WRITE_ONCE(sched_pelt_lshift, ++ sysctl_sched_pelt_multiplier >> 1); ++ goto done; ++ default: ++ ret = -EINVAL; ++ } ++ ++undo: ++ sysctl_sched_pelt_multiplier = old; ++done: ++ mutex_unlock(&mutex); ++ ++ return ret; ++} ++ ++static struct ctl_table sched_pelt_sysctls[] = { ++ { ++ .procname = "sched_pelt_multiplier", ++ .data = &sysctl_sched_pelt_multiplier, ++ .maxlen = sizeof(unsigned int), ++ .mode = 0644, ++ .proc_handler = sched_pelt_multiplier, ++ }, ++ {} ++}; ++ ++static int __init sched_pelt_sysctl_init(void) ++{ ++ register_sysctl_init("kernel", sched_pelt_sysctls); ++ return 0; ++} ++late_initcall(sched_pelt_sysctl_init); ++#endif +diff --git a/kernel/sched/pelt.h b/kernel/sched/pelt.h +index 3a0e0dc28721..9b35b5072bae 100644 +--- a/kernel/sched/pelt.h ++++ b/kernel/sched/pelt.h +@@ -61,6 +61,14 @@ static inline void cfs_se_util_change(struct sched_avg *avg) + WRITE_ONCE(avg->util_est.enqueued, enqueued); + } + ++static inline u64 rq_clock_task_mult(struct rq *rq) ++{ ++ lockdep_assert_rq_held(rq); ++ assert_clock_updated(rq); ++ ++ return rq->clock_task_mult; ++} ++ + static inline u64 rq_clock_pelt(struct rq *rq) + { + lockdep_assert_rq_held(rq); +@@ -72,7 +80,7 @@ static inline u64 rq_clock_pelt(struct rq *rq) + /* The rq is idle, we can sync to clock_task */ + static inline void _update_idle_rq_clock_pelt(struct rq *rq) + { +- rq->clock_pelt = rq_clock_task(rq); ++ rq->clock_pelt = rq_clock_task_mult(rq); + + u64_u32_store(rq->clock_idle, rq_clock(rq)); + /* Paired with smp_rmb in migrate_se_pelt_lag() */ +@@ -121,6 +129,27 @@ static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) + rq->clock_pelt += delta; + } + ++extern unsigned int sched_pelt_lshift; ++ ++/* ++ * absolute time |1 |2 |3 |4 |5 |6 | ++ * @ mult = 1 --------****************--------****************- ++ * @ mult = 2 --------********----------------********--------- ++ * @ mult = 4 --------****--------------------****------------- ++ * clock task mult ++ * @ mult = 2 | | |2 |3 | | | | |5 |6 | | | ++ * @ mult = 4 | | | | |2|3| | | | | | | | | | |5|6| | | | | | | ++ * ++ */ ++static inline void update_rq_clock_task_mult(struct rq *rq, s64 delta) ++{ ++ delta <<= READ_ONCE(sched_pelt_lshift); ++ ++ rq->clock_task_mult += delta; ++ ++ update_rq_clock_pelt(rq, delta); ++} ++ + /* + * When rq becomes idle, we have to check if it has lost idle time + * because it was fully busy. A rq is fully used when the /Sum util_sum +@@ -147,7 +176,7 @@ static inline void update_idle_rq_clock_pelt(struct rq *rq) + * rq's clock_task. + */ + if (util_sum >= divider) +- rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; ++ rq->lost_idle_time += rq_clock_task_mult(rq) - rq->clock_pelt; + + _update_idle_rq_clock_pelt(rq); + } +@@ -218,13 +247,18 @@ update_irq_load_avg(struct rq *rq, u64 running) + return 0; + } + +-static inline u64 rq_clock_pelt(struct rq *rq) ++static inline u64 rq_clock_task_mult(struct rq *rq) + { + return rq_clock_task(rq); + } + ++static inline u64 rq_clock_pelt(struct rq *rq) ++{ ++ return rq_clock_task_mult(rq); ++} ++ + static inline void +-update_rq_clock_pelt(struct rq *rq, s64 delta) { } ++update_rq_clock_task_mult(struct rq *rq, s64 delta) { } + + static inline void + update_idle_rq_clock_pelt(struct rq *rq) { } +diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h +index e26688d387ae..aa9f4d4df5bc 100644 +--- a/kernel/sched/sched.h ++++ b/kernel/sched/sched.h +@@ -1024,6 +1024,7 @@ struct rq { + u64 clock; + /* Ensure that all clocks are in the same cache line */ + u64 clock_task ____cacheline_aligned; ++ u64 clock_task_mult; + u64 clock_pelt; + unsigned long lost_idle_time; + u64 clock_pelt_idle; +diff --git a/kernel/sysctl.c b/kernel/sysctl.c +index 205d605cacc5..4432b6f15890 100644 +--- a/kernel/sysctl.c ++++ b/kernel/sysctl.c +@@ -89,8 +89,12 @@ + #ifdef CONFIG_PERF_EVENTS + static const int six_hundred_forty_kb = 640 * 1024; + #endif ++#ifdef CONFIG_USER_NS ++#include ++#endif + + ++extern int sysctl_sched_yield_type; + static const int ngroups_max = NGROUPS_MAX; + static const int cap_last_cap = CAP_LAST_CAP; + +@@ -1649,6 +1653,24 @@ static struct ctl_table kern_table[] = { + .mode = 0644, + .proc_handler = proc_dointvec, + }, ++ { ++ .procname = "yield_type", ++ .data = &sysctl_sched_yield_type, ++ .maxlen = sizeof (int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_TWO, ++ }, ++#ifdef CONFIG_USER_NS ++ { ++ .procname = "unprivileged_userns_clone", ++ .data = &unprivileged_userns_clone, ++ .maxlen = sizeof(int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec, ++ }, ++#endif + #ifdef CONFIG_PROC_SYSCTL + { + .procname = "tainted", +diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c +index 5481ba44a8d6..423ab2563ad7 100644 +--- a/kernel/user_namespace.c ++++ b/kernel/user_namespace.c +@@ -21,6 +21,13 @@ + #include + #include + ++/* sysctl */ ++#ifdef CONFIG_USER_NS_UNPRIVILEGED ++int unprivileged_userns_clone = 1; ++#else ++int unprivileged_userns_clone; ++#endif ++ + static struct kmem_cache *user_ns_cachep __read_mostly; + static DEFINE_MUTEX(userns_state_mutex); + +diff --git a/lib/Kconfig b/lib/Kconfig +index dc1ab2ed1dc6..3ea8941ab18d 100644 +--- a/lib/Kconfig ++++ b/lib/Kconfig +@@ -343,12 +343,16 @@ config LZ4HC_COMPRESS + config LZ4_DECOMPRESS + tristate + +-config ZSTD_COMPRESS ++config ZSTD_COMMON + select XXHASH + tristate + ++config ZSTD_COMPRESS ++ select ZSTD_COMMON ++ tristate ++ + config ZSTD_DECOMPRESS +- select XXHASH ++ select ZSTD_COMMON + tristate + + source "lib/xz/Kconfig" +diff --git a/lib/string.c b/lib/string.c +index 6f334420f687..2e60e9e40535 100644 +--- a/lib/string.c ++++ b/lib/string.c +@@ -866,24 +866,61 @@ char *strnstr(const char *s1, const char *s2, size_t len) + EXPORT_SYMBOL(strnstr); + #endif + ++#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 ++ ++#define MEMCHR_MASK_GEN(mask) (mask *= 0x0101010101010101ULL) ++ ++#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) ++ ++#define MEMCHR_MASK_GEN(mask) \ ++ do { \ ++ mask *= 0x01010101; \ ++ mask |= mask << 32; \ ++ } while (0) ++ ++#else ++ ++#define MEMCHR_MASK_GEN(mask) \ ++ do { \ ++ mask |= mask << 8; \ ++ mask |= mask << 16; \ ++ mask |= mask << 32; \ ++ } while (0) ++ ++#endif ++ + #ifndef __HAVE_ARCH_MEMCHR + /** + * memchr - Find a character in an area of memory. +- * @s: The memory area ++ * @p: The memory area + * @c: The byte to search for +- * @n: The size of the area. ++ * @length: The size of the area. + * + * returns the address of the first occurrence of @c, or %NULL + * if @c is not found + */ +-void *memchr(const void *s, int c, size_t n) ++void *memchr(const void *p, int c, unsigned long length) + { +- const unsigned char *p = s; +- while (n-- != 0) { +- if ((unsigned char)c == *p++) { +- return (void *)(p - 1); ++ u64 mask, val; ++ const void *end = p + length; ++ ++ c &= 0xff; ++ if (p <= end - 8) { ++ mask = c; ++ MEMCHR_MASK_GEN(mask); ++ ++ for (; p <= end - 8; p += 8) { ++ val = *(u64 *)p ^ mask; ++ if ((val + 0xfefefefefefefeffu) & ++ (~val & 0x8080808080808080u)) ++ break; + } + } ++ ++ for (; p < end; p++) ++ if (*(unsigned char *)p == c) ++ return (void *)p; ++ + return NULL; + } + EXPORT_SYMBOL(memchr); +@@ -919,16 +956,7 @@ void *memchr_inv(const void *start, int c, size_t bytes) + return check_bytes8(start, value, bytes); + + value64 = value; +-#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 +- value64 *= 0x0101010101010101ULL; +-#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) +- value64 *= 0x01010101; +- value64 |= value64 << 32; +-#else +- value64 |= value64 << 8; +- value64 |= value64 << 16; +- value64 |= value64 << 32; +-#endif ++ MEMCHR_MASK_GEN(value64); + + prefix = (unsigned long)start % 8; + if (prefix) { +diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile +index fc45339fc3a3..440bd0007ae2 100644 +--- a/lib/zstd/Makefile ++++ b/lib/zstd/Makefile +@@ -10,14 +10,10 @@ + # ################################################################ + obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o + obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o ++obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o + + zstd_compress-y := \ + zstd_compress_module.o \ +- common/debug.o \ +- common/entropy_common.o \ +- common/error_private.o \ +- common/fse_decompress.o \ +- common/zstd_common.o \ + compress/fse_compress.o \ + compress/hist.o \ + compress/huf_compress.o \ +@@ -33,12 +29,14 @@ zstd_compress-y := \ + + zstd_decompress-y := \ + zstd_decompress_module.o \ ++ decompress/huf_decompress.o \ ++ decompress/zstd_ddict.o \ ++ decompress/zstd_decompress.o \ ++ decompress/zstd_decompress_block.o \ ++ ++zstd_common-y := \ + common/debug.o \ + common/entropy_common.o \ + common/error_private.o \ + common/fse_decompress.o \ + common/zstd_common.o \ +- decompress/huf_decompress.o \ +- decompress/zstd_ddict.o \ +- decompress/zstd_decompress.o \ +- decompress/zstd_decompress_block.o \ +diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c +index 53b47a2b52ff..a311808c0d56 100644 +--- a/lib/zstd/common/entropy_common.c ++++ b/lib/zstd/common/entropy_common.c +@@ -15,6 +15,7 @@ + /* ************************************* + * Dependencies + ***************************************/ ++#include + #include "mem.h" + #include "error_private.h" /* ERR_*, ERROR */ + #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +@@ -239,7 +240,7 @@ size_t FSE_readNCount( + { + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); + } +- ++EXPORT_SYMBOL_GPL(FSE_readNCount); + + /*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). +@@ -255,6 +256,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + } ++EXPORT_SYMBOL_GPL(HUF_readStats); + + FORCE_INLINE_TEMPLATE size_t + HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, +@@ -355,3 +357,4 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + (void)bmi2; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } ++EXPORT_SYMBOL_GPL(HUF_readStats_wksp); +diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c +index 3d7e35b309b5..0f1f63be25d9 100644 +--- a/lib/zstd/common/zstd_common.c ++++ b/lib/zstd/common/zstd_common.c +@@ -13,6 +13,7 @@ + /*-************************************* + * Dependencies + ***************************************/ ++#include + #define ZSTD_DEPS_NEED_MALLOC + #include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + #include "error_private.h" +@@ -35,14 +36,17 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } + * tells if a return value is an error code + * symbol is required for external callers */ + unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } ++EXPORT_SYMBOL_GPL(ZSTD_isError); + + /*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ + const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } ++EXPORT_SYMBOL_GPL(ZSTD_getErrorName); + + /*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ + ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } ++EXPORT_SYMBOL_GPL(ZSTD_getErrorCode); + + /*! ZSTD_getErrorString() : + * provides error code string from enum */ +@@ -59,6 +63,7 @@ void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); + } ++EXPORT_SYMBOL_GPL(ZSTD_customMalloc); + + void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) + { +@@ -71,6 +76,7 @@ void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) + } + return ZSTD_calloc(1, size); + } ++EXPORT_SYMBOL_GPL(ZSTD_customCalloc); + + void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) + { +@@ -81,3 +87,7 @@ void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) + ZSTD_free(ptr); + } + } ++EXPORT_SYMBOL_GPL(ZSTD_customFree); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("Zstd Common"); +diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c +index b0424d23ac57..fb941a5b70f5 100644 +--- a/lib/zstd/compress/zstd_double_fast.c ++++ b/lib/zstd/compress/zstd_double_fast.c +@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(dictMode, mls) \ ++ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \ ++ } ++ ++ZSTD_GEN_FN(noDict, 4) ++ZSTD_GEN_FN(noDict, 5) ++ZSTD_GEN_FN(noDict, 6) ++ZSTD_GEN_FN(noDict, 7) ++ ++ZSTD_GEN_FN(dictMatchState, 4) ++ZSTD_GEN_FN(dictMatchState, 5) ++ZSTD_GEN_FN(dictMatchState, 6) ++ZSTD_GEN_FN(dictMatchState, 7) ++ ++#undef ZSTD_GEN_FN ++ + + size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ +@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + } + + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN ++ + size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); + } + } +diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c +index 96b7d48e2868..e0652e31d421 100644 +--- a/lib/zstd/compress/zstd_fast.c ++++ b/lib/zstd/compress/zstd_fast.c +@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_fast_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN + + size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN ++ + size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); ++ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ +@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_fast_extDict_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN + + size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize); + } + } +diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c +index fb54d4e28a2b..1db22db5b1e9 100644 +--- a/lib/zstd/compress/zstd_lazy.c ++++ b/lib/zstd/compress/zstd_lazy.c +@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + } + + +-static size_t +-ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); +- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); +- case 7 : +- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); +- } +-} +- +- +-static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); +- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); +- case 7 : +- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); +- } +-} +- +- +-static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); +- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); +- case 7 : +- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); +- } +-} +- +- +- + /* ********************************* + * Hash Chain + ***********************************/ +@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B + + /* inlining is important to hardwire a hot branch (template emulation) */ + FORCE_INLINE_TEMPLATE +-size_t ZSTD_HcFindBestMatch_generic ( ++size_t ZSTD_HcFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, +@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic ( + return ml; + } + ++typedef size_t (*searchMax_f)( ++ ZSTD_matchState_t* ms, ++ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + +-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); +- } +-} ++/* ++ * This struct contains the functions necessary for lazy to search. ++ * Currently, that is only searchMax. However, it is still valuable to have the ++ * VTable because this makes it easier to add more functions to the VTable later. ++ */ ++typedef struct { ++ searchMax_f searchMax; ++} ZSTD_LazyVTable; ++ ++#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \ ++ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ ++ ZSTD_matchState_t* ms, \ ++ const BYTE* ip, const BYTE* const iLimit, \ ++ size_t* offsetPtr) \ ++ { \ ++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ ++ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ ++ } \ ++ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ ++ ZSTD_BtFindBestMatch_##dictMode##_##mls \ ++ }; + ++#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \ ++ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ ++ ZSTD_matchState_t* ms, \ ++ const BYTE* ip, const BYTE* const iLimit, \ ++ size_t* offsetPtr) \ ++ { \ ++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ ++ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ ++ } \ ++ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ ++ ZSTD_HcFindBestMatch_##dictMode##_##mls \ ++ }; + +-static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); ++#define ZSTD_FOR_EACH_MLS(X, dictMode) \ ++ X(dictMode, 4) \ ++ X(dictMode, 5) \ ++ X(dictMode, 6) ++ ++#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ ++ X(__VA_ARGS__, noDict) \ ++ X(__VA_ARGS__, extDict) \ ++ X(__VA_ARGS__, dictMatchState) \ ++ X(__VA_ARGS__, dedicatedDictSearch) ++ ++/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ ++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) ++/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ ++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) ++ ++#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ ++ { \ ++ &ZSTD_BtVTable_##dictMode##_4, \ ++ &ZSTD_BtVTable_##dictMode##_5, \ ++ &ZSTD_BtVTable_##dictMode##_6 \ + } +-} +- + +-static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); ++#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ ++ { \ ++ &ZSTD_HcVTable_##dictMode##_4, \ ++ &ZSTD_HcVTable_##dictMode##_5, \ ++ &ZSTD_HcVTable_##dictMode##_6 \ + } +-} + +- +-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); ++#define GEN_ZSTD_VTABLE_ARRAY(X) \ ++ { \ ++ X(noDict), \ ++ X(extDict), \ ++ X(dictMatchState), \ ++ X(dedicatedDictSearch) \ + } +-} +- + + /* ******************************* + * Common parser - lazy strategy + *********************************/ + typedef enum { search_hashChain, search_binaryTree } searchMethod_e; + ++static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) ++{ ++ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ ++ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); ++ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); ++ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ ++ ++ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); ++ switch (searchMethod) { ++ case search_hashChain: ++ return hcVTables[dictMode][mls - 4]; ++ case search_binaryTree: ++ return btVTables[dictMode][mls - 4]; ++ default: ++ return NULL; ++ } ++} ++ + FORCE_INLINE_TEMPLATE size_t + ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, +@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic( + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + +- typedef size_t (*searchMax_f)( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); +- + /* + * This table is indexed first by the four ZSTD_dictMode_e values, and then + * by the two searchMethod_e values. NULLs are placed for configurations + * that should never occur (extDict modes go to the other implementation + * below and there is no DDSS for binary tree search yet). + */ +- const searchMax_f searchFuncs[4][2] = { +- { +- ZSTD_HcFindBestMatch_selectMLS, +- ZSTD_BtFindBestMatch_selectMLS +- }, +- { +- NULL, +- NULL +- }, +- { +- ZSTD_HcFindBestMatch_dictMatchState_selectMLS, +- ZSTD_BtFindBestMatch_dictMatchState_selectMLS +- }, +- { +- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, +- NULL +- } +- }; +- +- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; ++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; + U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + + const int isDMS = dictMode == ZSTD_dictMatchState; +@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + +- typedef size_t (*searchMax_f)( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); +- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; ++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; + + U32 offset_1 = rep[0], offset_2 = rep[1]; + +diff --git a/mm/compaction.c b/mm/compaction.c +index 640fa76228dd..01a661f5ecac 100644 +--- a/mm/compaction.c ++++ b/mm/compaction.c +@@ -1727,7 +1727,7 @@ typedef enum { + * Allow userspace to control policy on scanning the unevictable LRU for + * compactable pages. + */ +-#ifdef CONFIG_PREEMPT_RT ++#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) + int sysctl_compact_unevictable_allowed __read_mostly = 0; + #else + int sysctl_compact_unevictable_allowed __read_mostly = 1; +@@ -2719,7 +2719,11 @@ static void compact_nodes(void) + * aggressively the kernel should compact memory in the + * background. It takes values in the range [0, 100]. + */ ++#ifdef CONFIG_CACHY ++unsigned int __read_mostly sysctl_compaction_proactiveness; ++#else + unsigned int __read_mostly sysctl_compaction_proactiveness = 20; ++#endif + + int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, + void *buffer, size_t *length, loff_t *ppos) +diff --git a/mm/page-writeback.c b/mm/page-writeback.c +index 032a7bf8d259..cbcecd565c16 100644 +--- a/mm/page-writeback.c ++++ b/mm/page-writeback.c +@@ -70,7 +70,11 @@ static long ratelimit_pages = 32; + /* + * Start background writeback (via writeback threads) at this percentage + */ ++#ifdef CONFIG_CACHY ++static int dirty_background_ratio = 5; ++#else + static int dirty_background_ratio = 10; ++#endif + + /* + * dirty_background_bytes starts at 0 (disabled) so that it is a function of +@@ -98,7 +102,11 @@ static unsigned long vm_dirty_bytes; + /* + * The interval between `kupdate'-style writebacks + */ ++#ifdef CONFIG_CACHY ++unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */ ++#else + unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ ++#endif + + EXPORT_SYMBOL_GPL(dirty_writeback_interval); + +diff --git a/mm/swap.c b/mm/swap.c +index 9cee7f6a3809..186b4e5dcecf 100644 +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -1070,6 +1070,10 @@ EXPORT_SYMBOL(pagevec_lookup_range_tag); + */ + void __init swap_setup(void) + { ++#ifdef CONFIG_CACHY ++ /* Only swap-in pages requested, avoid readahead */ ++ page_cluster = 0; ++#else + unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); + + /* Use a smaller cluster for small-memory machines */ +@@ -1081,4 +1085,5 @@ void __init swap_setup(void) + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ ++#endif + } +diff --git a/mm/vmpressure.c b/mm/vmpressure.c +index b52644771cc4..11a4b0e3b583 100644 +--- a/mm/vmpressure.c ++++ b/mm/vmpressure.c +@@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; + * essence, they are percents: the higher the value, the more number + * unsuccessful reclaims there were. + */ ++#ifdef CONFIG_CACHY ++static const unsigned int vmpressure_level_med = 65; ++#else + static const unsigned int vmpressure_level_med = 60; ++#endif + static const unsigned int vmpressure_level_critical = 95; + + /* +diff --git a/mm/vmscan.c b/mm/vmscan.c +index 382dbe97329f..fbc8c8f4fe60 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -178,7 +178,11 @@ struct scan_control { + /* + * From 0 .. 200. Higher means more swappy. + */ ++#ifdef CONFIG_CACHY ++int vm_swappiness = 20; ++#else + int vm_swappiness = 60; ++#endif + + static void set_task_reclaim_state(struct task_struct *task, + struct reclaim_state *rs) +diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib +index 3fb6a99e78c4..f62770a0a84f 100644 +--- a/scripts/Makefile.lib ++++ b/scripts/Makefile.lib +@@ -504,14 +504,21 @@ quiet_cmd_xzmisc = XZMISC $@ + # decompression is used, like initramfs decompression, zstd22 should likely not + # be used because it would require zstd to allocate a 128 MB buffer. + ++ifdef CONFIG_ZSTD_COMP_VAL ++zstd_comp_val := $(CONFIG_ZSTD_COMP_VAL) ++ifeq ($(shell test $(zstd_comp_val) -gt 19; echo $$?),0) ++zstd_comp_val += --ultra ++endif ++endif ++ + quiet_cmd_zstd = ZSTD $@ +- cmd_zstd = cat $(real-prereqs) | $(ZSTD) -19 > $@ ++ cmd_zstd = cat $(real-prereqs) | $(ZSTD) -T0 -19 > $@ + + quiet_cmd_zstd22 = ZSTD22 $@ +- cmd_zstd22 = cat $(real-prereqs) | $(ZSTD) -22 --ultra > $@ ++ cmd_zstd22 = cat $(real-prereqs) | $(ZSTD) -T0 -22 --ultra > $@ + + quiet_cmd_zstd22_with_size = ZSTD22 $@ +- cmd_zstd22_with_size = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@ ++ cmd_zstd22_with_size = { cat $(real-prereqs) | $(ZSTD) -T0 -$(zstd_comp_val); $(size_append); } > $@ + + # ASM offsets + # --------------------------------------------------------------------------- +diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst +index a4c987c23750..132863cf3183 100644 +--- a/scripts/Makefile.modinst ++++ b/scripts/Makefile.modinst +@@ -96,8 +96,13 @@ quiet_cmd_gzip = GZIP $@ + cmd_gzip = $(KGZIP) -n -f $< + quiet_cmd_xz = XZ $@ + cmd_xz = $(XZ) --lzma2=dict=2MiB -f $< ++ifdef CONFIG_MODULE_COMPRESS_ZSTD_ULTRA + quiet_cmd_zstd = ZSTD $@ +- cmd_zstd = $(ZSTD) -T0 --rm -f -q $< ++ cmd_zstd = $(ZSTD) -$(CONFIG_MODULE_COMPRESS_ZSTD_LEVEL_ULTRA) --ultra --zstd=wlog=21 -T0 --rm -f -q $< ++else ++quiet_cmd_zstd = ZSTD $@ ++ cmd_zstd = $(ZSTD) -$(CONFIG_MODULE_COMPRESS_ZSTD_LEVEL) --zstd=wlog=21 -T0 --rm -f -q $< ++endif + + $(dst)/%.ko.gz: $(dst)/%.ko FORCE + $(call cmd,gzip) +-- +2.38.0 + +From c4125b4903e46e792158e16439f728813dfe3f44 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 5 Sep 2022 08:34:43 +0200 +Subject: [PATCH 02/17] bbr2 + +Signed-off-by: Peter Jung +--- + Documentation/networking/ip-sysctl.rst | 58 + + include/linux/tcp.h | 6 +- + include/net/inet_connection_sock.h | 3 +- + include/net/netns/ipv4.h | 5 + + include/net/tcp.h | 58 +- + include/uapi/linux/inet_diag.h | 33 + + include/uapi/linux/snmp.h | 1 + + net/ipv4/Kconfig | 22 + + net/ipv4/Makefile | 3 +- + net/ipv4/bpf_tcp_ca.c | 12 + + net/ipv4/proc.c | 1 + + net/ipv4/sysctl_net_ipv4.c | 43 + + net/ipv4/tcp.c | 1 + + net/ipv4/tcp_bbr.c | 38 +- + net/ipv4/tcp_bbr2.c | 2692 ++++++++++++++++++++++++ + net/ipv4/tcp_cong.c | 1 + + net/ipv4/tcp_input.c | 27 +- + net/ipv4/tcp_ipv4.c | 7 + + net/ipv4/tcp_output.c | 26 +- + net/ipv4/tcp_plb.c | 100 + + net/ipv4/tcp_rate.c | 30 +- + net/ipv4/tcp_timer.c | 1 + + 22 files changed, 3133 insertions(+), 35 deletions(-) + create mode 100644 net/ipv4/tcp_bbr2.c + create mode 100644 net/ipv4/tcp_plb.c + +diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst +index a759872a2883..f2372f8f860b 100644 +--- a/Documentation/networking/ip-sysctl.rst ++++ b/Documentation/networking/ip-sysctl.rst +@@ -1040,6 +1040,64 @@ tcp_challenge_ack_limit - INTEGER + TCP stack implements per TCP socket limits anyway. + Default: INT_MAX (unlimited) + ++tcp_plb_enabled - BOOLEAN ++ If set, TCP PLB (Protective Load Balancing) is enabled. PLB is ++ described in the following paper: ++ https://doi.org/10.1145/3544216.3544226. Based on PLB parameters, ++ upon sensing sustained congestion, TCP triggers a change in ++ flow label field for outgoing IPv6 packets. A change in flow label ++ field potentially changes the path of outgoing packets for switches ++ that use ECMP/WCMP for routing. ++ ++ Default: 0 ++ ++tcp_plb_cong_thresh - INTEGER ++ Fraction of packets marked with congestion over a round (RTT) to ++ tag that round as congested. This is referred to as K in the PLB paper: ++ https://doi.org/10.1145/3544216.3544226. ++ ++ The 0-1 fraction range is mapped to 0-256 range to avoid floating ++ point operations. For example, 128 means that if at least 50% of ++ the packets in a round were marked as congested then the round ++ will be tagged as congested. ++ ++ Possible Values: 0 - 256 ++ ++ Default: 128 ++ ++tcp_plb_idle_rehash_rounds - INTEGER ++ Number of consecutive congested rounds (RTT) seen after which ++ a rehash can be performed, given there are no packets in flight. ++ This is referred to as M in PLB paper: ++ https://doi.org/10.1145/3544216.3544226. ++ ++ Possible Values: 0 - 31 ++ ++ Default: 3 ++ ++tcp_plb_rehash_rounds - INTEGER ++ Number of consecutive congested rounds (RTT) seen after which ++ a forced rehash can be performed. Be careful when setting this ++ parameter, as a small value increases the risk of retransmissions. ++ This is referred to as N in PLB paper: ++ https://doi.org/10.1145/3544216.3544226. ++ ++ Possible Values: 0 - 31 ++ ++ Default: 12 ++ ++tcp_plb_suspend_rto_sec - INTEGER ++ Time, in seconds, to suspend PLB in event of an RTO. In order to avoid ++ having PLB repath onto a connectivity "black hole", after an RTO a TCP ++ connection suspends PLB repathing for a random duration between 1x and ++ 2x of this parameter. Randomness is added to avoid concurrent rehashing ++ of multiple TCP connections. This should be set corresponding to the ++ amount of time it takes to repair a failed link. ++ ++ Possible Values: 0 - 255 ++ ++ Default: 60 ++ + UDP variables + ============= + +diff --git a/include/linux/tcp.h b/include/linux/tcp.h +index a9fbe22732c3..4034e46093b4 100644 +--- a/include/linux/tcp.h ++++ b/include/linux/tcp.h +@@ -255,7 +255,8 @@ struct tcp_sock { + u8 compressed_ack; + u8 dup_ack_counter:2, + tlp_retrans:1, /* TLP is a retransmission */ +- unused:5; ++ fast_ack_mode:2, /* which fast ack mode ? */ ++ unused:3; + u32 chrono_start; /* Start time in jiffies of a TCP chrono */ + u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ + u8 chrono_type:2, /* current chronograph type */ +@@ -443,6 +444,9 @@ struct tcp_sock { + */ + struct request_sock __rcu *fastopen_rsk; + struct saved_syn *saved_syn; ++ ++/* Rerouting information */ ++ u16 ecn_rehash; /* PLB triggered rehash attempts */ + }; + + enum tsq_enum { +diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h +index ee88f0f1350f..e3075b3f1ece 100644 +--- a/include/net/inet_connection_sock.h ++++ b/include/net/inet_connection_sock.h +@@ -132,7 +132,8 @@ struct inet_connection_sock { + u32 icsk_probes_tstamp; + u32 icsk_user_timeout; + +- u64 icsk_ca_priv[104 / sizeof(u64)]; ++/* XXX inflated by temporary internal debugging info */ ++ u64 icsk_ca_priv[224 / sizeof(u64)]; + #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) + }; + +diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h +index 6320a76cefdc..2e39e07ed41f 100644 +--- a/include/net/netns/ipv4.h ++++ b/include/net/netns/ipv4.h +@@ -181,6 +181,11 @@ struct netns_ipv4 { + unsigned long tfo_active_disable_stamp; + u32 tcp_challenge_timestamp; + u32 tcp_challenge_count; ++ u8 sysctl_tcp_plb_enabled; ++ int sysctl_tcp_plb_cong_thresh; ++ u8 sysctl_tcp_plb_idle_rehash_rounds; ++ u8 sysctl_tcp_plb_rehash_rounds; ++ u8 sysctl_tcp_plb_suspend_rto_sec; + + int sysctl_udp_wmem_min; + int sysctl_udp_rmem_min; +diff --git a/include/net/tcp.h b/include/net/tcp.h +index d10962b9f0d0..d3947345feaa 100644 +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -372,6 +372,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk, + #define TCP_ECN_QUEUE_CWR 2 + #define TCP_ECN_DEMAND_CWR 4 + #define TCP_ECN_SEEN 8 ++#define TCP_ECN_ECT_PERMANENT 16 + + enum tcp_tw_status { + TCP_TW_SUCCESS = 0, +@@ -816,6 +817,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) + return max_t(s64, t1 - t0, 0); + } + ++static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) ++{ ++ return max_t(s32, t1 - t0, 0); ++} ++ + static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) + { + return tcp_ns_to_ts(skb->skb_mstamp_ns); +@@ -891,9 +897,14 @@ struct tcp_skb_cb { + /* pkts S/ACKed so far upon tx of skb, incl retrans: */ + __u32 delivered; + /* start of send pipeline phase */ +- u64 first_tx_mstamp; ++ u32 first_tx_mstamp; + /* when we reached the "delivered" count */ +- u64 delivered_mstamp; ++ u32 delivered_mstamp; ++#define TCPCB_IN_FLIGHT_BITS 20 ++#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) ++ u32 in_flight:20, /* packets in flight at transmit */ ++ unused2:12; ++ u32 lost; /* packets lost so far upon tx of skb */ + } tx; /* only used for outgoing skbs */ + union { + struct inet_skb_parm h4; +@@ -1019,7 +1030,11 @@ enum tcp_ca_ack_event_flags { + #define TCP_CONG_NON_RESTRICTED 0x1 + /* Requires ECN/ECT set on all packets */ + #define TCP_CONG_NEEDS_ECN 0x2 +-#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) ++/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ ++#define TCP_CONG_WANTS_CE_EVENTS 0x4 ++#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ ++ TCP_CONG_NEEDS_ECN | \ ++ TCP_CONG_WANTS_CE_EVENTS) + + union tcp_cc_info; + +@@ -1039,8 +1054,11 @@ struct ack_sample { + */ + struct rate_sample { + u64 prior_mstamp; /* starting timestamp for interval */ ++ u32 prior_lost; /* tp->lost at "prior_mstamp" */ + u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ + u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ ++ u32 tx_in_flight; /* packets in flight at starting timestamp */ ++ s32 lost; /* number of packets lost over interval */ + s32 delivered; /* number of packets delivered over interval */ + s32 delivered_ce; /* number of packets delivered w/ CE marks*/ + long interval_us; /* time for tp->delivered to incr "delivered" */ +@@ -1054,6 +1072,7 @@ struct rate_sample { + bool is_app_limited; /* is sample from packet with bubble in pipe? */ + bool is_retrans; /* is sample from retransmission? */ + bool is_ack_delayed; /* is this (likely) a delayed ACK? */ ++ bool is_ece; /* did this ACK have ECN marked? */ + }; + + struct tcp_congestion_ops { +@@ -1077,8 +1096,11 @@ struct tcp_congestion_ops { + /* hook for packet ack accounting (optional) */ + void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); + +- /* override sysctl_tcp_min_tso_segs */ +- u32 (*min_tso_segs)(struct sock *sk); ++ /* pick target number of segments per TSO/GSO skb (optional): */ ++ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); ++ ++ /* react to a specific lost skb (optional) */ ++ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); + + /* call when packets are delivered to update cwnd and pacing rate, + * after all the ca_state processing. (optional) +@@ -1141,6 +1163,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) + } + #endif + ++static inline bool tcp_ca_wants_ce_events(const struct sock *sk) ++{ ++ const struct inet_connection_sock *icsk = inet_csk(sk); ++ ++ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | ++ TCP_CONG_WANTS_CE_EVENTS); ++} ++ + static inline bool tcp_ca_needs_ecn(const struct sock *sk) + { + const struct inet_connection_sock *icsk = inet_csk(sk); +@@ -1160,6 +1190,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) + void tcp_set_ca_state(struct sock *sk, const u8 ca_state); + + /* From tcp_rate.c */ ++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); + void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); + void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, + struct rate_sample *rs); +@@ -2130,6 +2161,23 @@ extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, + extern void tcp_rack_reo_timeout(struct sock *sk); + extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); + ++/* tcp_plb.c */ ++ ++#define TCP_PLB_SCALE 8 /* scaling factor for fractions in PLB (e.g. ce_ratio) */ ++ ++/* State for PLB (Protective Load Balancing) for a single TCP connection. */ ++struct tcp_plb_state { ++ u8 consec_cong_rounds:5, /* consecutive congested rounds */ ++ enabled:1, /* Check if PLB is enabled */ ++ unused:2; ++ u32 pause_until; /* jiffies32 when PLB can resume repathing */ ++}; ++ ++void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, ++ const int cong_ratio); ++void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); ++void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); ++ + /* At how many usecs into the future should the RTO fire? */ + static inline s64 tcp_rto_delta_us(const struct sock *sk) + { +diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h +index 50655de04c9b..0e24f11627d5 100644 +--- a/include/uapi/linux/inet_diag.h ++++ b/include/uapi/linux/inet_diag.h +@@ -231,9 +231,42 @@ struct tcp_bbr_info { + __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + }; + ++/* Phase as reported in netlink/ss stats. */ ++enum tcp_bbr2_phase { ++ BBR2_PHASE_INVALID = 0, ++ BBR2_PHASE_STARTUP = 1, ++ BBR2_PHASE_DRAIN = 2, ++ BBR2_PHASE_PROBE_RTT = 3, ++ BBR2_PHASE_PROBE_BW_UP = 4, ++ BBR2_PHASE_PROBE_BW_DOWN = 5, ++ BBR2_PHASE_PROBE_BW_CRUISE = 6, ++ BBR2_PHASE_PROBE_BW_REFILL = 7 ++}; ++ ++struct tcp_bbr2_info { ++ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ ++ __u32 bbr_bw_lsb; /* lower 32 bits of bw */ ++ __u32 bbr_bw_msb; /* upper 32 bits of bw */ ++ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ ++ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ ++ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ ++ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ ++ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ ++ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ ++ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ ++ __u8 bbr_mode; /* current bbr_mode in state machine */ ++ __u8 bbr_phase; /* current state machine phase */ ++ __u8 unused1; /* alignment padding; not used yet */ ++ __u8 bbr_version; /* MUST be at this offset in struct */ ++ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ ++ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ ++ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ ++}; ++ + union tcp_cc_info { + struct tcpvegas_info vegas; + struct tcp_dctcp_info dctcp; + struct tcp_bbr_info bbr; ++ struct tcp_bbr2_info bbr2; + }; + #endif /* _UAPI_INET_DIAG_H_ */ +diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h +index 4d7470036a8b..8ce035f1c874 100644 +--- a/include/uapi/linux/snmp.h ++++ b/include/uapi/linux/snmp.h +@@ -292,6 +292,7 @@ enum + LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */ + LINUX_MIB_TCPMIGRATEREQSUCCESS, /* TCPMigrateReqSuccess */ + LINUX_MIB_TCPMIGRATEREQFAILURE, /* TCPMigrateReqFailure */ ++ LINUX_MIB_TCPECNREHASH, /* TCPECNRehash */ + __LINUX_MIB_MAX + }; + +diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig +index e983bb0c5012..d172ffbdf27f 100644 +--- a/net/ipv4/Kconfig ++++ b/net/ipv4/Kconfig +@@ -668,6 +668,24 @@ config TCP_CONG_BBR + AQM schemes that do not provide a delay signal. It requires the fq + ("Fair Queue") pacing packet scheduler. + ++config TCP_CONG_BBR2 ++ tristate "BBR2 TCP" ++ default n ++ help ++ ++ BBR2 TCP congestion control is a model-based congestion control ++ algorithm that aims to maximize network utilization, keep queues and ++ retransmit rates low, and to be able to coexist with Reno/CUBIC in ++ common scenarios. It builds an explicit model of the network path. It ++ tolerates a targeted degree of random packet loss and delay that are ++ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, ++ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can ++ coexist with flows that use loss-based congestion control, and can ++ operate with shallow buffers, deep buffers, bufferbloat, policers, or ++ AQM schemes that do not provide a delay signal. It requires pacing, ++ using either TCP internal pacing or the fq ("Fair Queue") pacing packet ++ scheduler. ++ + choice + prompt "Default TCP congestion control" + default DEFAULT_CUBIC +@@ -705,6 +723,9 @@ choice + config DEFAULT_BBR + bool "BBR" if TCP_CONG_BBR=y + ++ config DEFAULT_BBR2 ++ bool "BBR2" if TCP_CONG_BBR2=y ++ + config DEFAULT_RENO + bool "Reno" + endchoice +@@ -729,6 +750,7 @@ config DEFAULT_TCP_CONG + default "dctcp" if DEFAULT_DCTCP + default "cdg" if DEFAULT_CDG + default "bbr" if DEFAULT_BBR ++ default "bbr2" if DEFAULT_BBR2 + default "cubic" + + config TCP_MD5SIG +diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile +index bbdd9c44f14e..e7a86a50838a 100644 +--- a/net/ipv4/Makefile ++++ b/net/ipv4/Makefile +@@ -10,7 +10,7 @@ obj-y := route.o inetpeer.o protocol.o \ + tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \ + tcp_minisocks.o tcp_cong.o tcp_metrics.o tcp_fastopen.o \ + tcp_rate.o tcp_recovery.o tcp_ulp.o \ +- tcp_offload.o datagram.o raw.o udp.o udplite.o \ ++ tcp_offload.o tcp_plb.o datagram.o raw.o udp.o udplite.o \ + udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \ + fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \ + inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \ +@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o + obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o + obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o + obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o ++obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o + obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o + obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o + obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o +diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c +index 85a9e500c42d..24fcbac984fe 100644 +--- a/net/ipv4/bpf_tcp_ca.c ++++ b/net/ipv4/bpf_tcp_ca.c +@@ -14,6 +14,18 @@ + /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ + extern struct bpf_struct_ops bpf_tcp_congestion_ops; + ++static u32 optional_ops[] = { ++ offsetof(struct tcp_congestion_ops, init), ++ offsetof(struct tcp_congestion_ops, release), ++ offsetof(struct tcp_congestion_ops, set_state), ++ offsetof(struct tcp_congestion_ops, cwnd_event), ++ offsetof(struct tcp_congestion_ops, in_ack_event), ++ offsetof(struct tcp_congestion_ops, pkts_acked), ++ offsetof(struct tcp_congestion_ops, tso_segs), ++ offsetof(struct tcp_congestion_ops, sndbuf_expand), ++ offsetof(struct tcp_congestion_ops, cong_control), ++}; ++ + static u32 unsupported_ops[] = { + offsetof(struct tcp_congestion_ops, get_info), + }; +diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c +index 0088a4c64d77..e0a664b467e0 100644 +--- a/net/ipv4/proc.c ++++ b/net/ipv4/proc.c +@@ -297,6 +297,7 @@ static const struct snmp_mib snmp4_net_list[] = { + SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS), + SNMP_MIB_ITEM("TCPMigrateReqSuccess", LINUX_MIB_TCPMIGRATEREQSUCCESS), + SNMP_MIB_ITEM("TCPMigrateReqFailure", LINUX_MIB_TCPMIGRATEREQFAILURE), ++ SNMP_MIB_ITEM("TCPECNRehash", LINUX_MIB_TCPECNREHASH), + SNMP_MIB_SENTINEL + }; + +diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c +index 5490c285668b..ed35a9485e71 100644 +--- a/net/ipv4/sysctl_net_ipv4.c ++++ b/net/ipv4/sysctl_net_ipv4.c +@@ -39,6 +39,8 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ; + static int one_day_secs = 24 * 3600; + static u32 fib_multipath_hash_fields_all_mask __maybe_unused = + FIB_MULTIPATH_HASH_FIELD_ALL_MASK; ++static int tcp_plb_max_rounds = 31; ++static int tcp_plb_max_cong_thresh = 256; + + /* obsolete */ + static int sysctl_tcp_low_latency __read_mostly; +@@ -1346,6 +1348,47 @@ static struct ctl_table ipv4_net_table[] = { + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, ++ { ++ .procname = "tcp_plb_enabled", ++ .data = &init_net.ipv4.sysctl_tcp_plb_enabled, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "tcp_plb_cong_thresh", ++ .data = &init_net.ipv4.sysctl_tcp_plb_cong_thresh, ++ .maxlen = sizeof(int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &tcp_plb_max_cong_thresh, ++ }, ++ { ++ .procname = "tcp_plb_idle_rehash_rounds", ++ .data = &init_net.ipv4.sysctl_tcp_plb_idle_rehash_rounds, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra2 = &tcp_plb_max_rounds, ++ }, ++ { ++ .procname = "tcp_plb_rehash_rounds", ++ .data = &init_net.ipv4.sysctl_tcp_plb_rehash_rounds, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra2 = &tcp_plb_max_rounds, ++ }, ++ { ++ .procname = "tcp_plb_suspend_rto_sec", ++ .data = &init_net.ipv4.sysctl_tcp_plb_suspend_rto_sec, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ }, + { } + }; + +diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c +index e373dde1f46f..b735776c0225 100644 +--- a/net/ipv4/tcp.c ++++ b/net/ipv4/tcp.c +@@ -3187,6 +3187,7 @@ int tcp_disconnect(struct sock *sk, int flags) + tp->rx_opt.dsack = 0; + tp->rx_opt.num_sacks = 0; + tp->rcv_ooopack = 0; ++ tp->fast_ack_mode = 0; + + + /* Clean up fastopen related fields */ +diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c +index 54eec33c6e1c..bfbf158c71f4 100644 +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) + sk->sk_pacing_rate = rate; + } + +-/* override sysctl_tcp_min_tso_segs */ + static u32 bbr_min_tso_segs(struct sock *sk) + { + return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; + } + ++/* Return the number of segments BBR would like in a TSO/GSO skb, given ++ * a particular max gso size as a constraint. ++ */ ++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, ++ u32 gso_max_size) ++{ ++ u32 segs; ++ u64 bytes; ++ ++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ ++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; ++ ++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); ++ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); ++ return segs; ++} ++ ++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ ++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) ++{ ++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); ++} ++ ++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ + static u32 bbr_tso_segs_goal(struct sock *sk) + { + struct tcp_sock *tp = tcp_sk(sk); +- u32 segs, bytes; +- +- /* Sort of tcp_tso_autosize() but ignoring +- * driver provided sk_gso_max_size. +- */ +- bytes = min_t(unsigned long, +- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), +- GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); +- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); + +- return min(segs, 0x7FU); ++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); + } + + /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { + .undo_cwnd = bbr_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .ssthresh = bbr_ssthresh, +- .min_tso_segs = bbr_min_tso_segs, ++ .tso_segs = bbr_tso_segs, + .get_info = bbr_get_info, + .set_state = bbr_set_state, + }; +diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c +new file mode 100644 +index 000000000000..2e39f7a353be +--- /dev/null ++++ b/net/ipv4/tcp_bbr2.c +@@ -0,0 +1,2692 @@ ++/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 ++ * ++ * BBRv2 is a model-based congestion control algorithm that aims for low ++ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model ++ * of the network path, it uses measurements of bandwidth and RTT, as well as ++ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that ++ * although it can use ECN or loss signals explicitly, it does not require ++ * either; it can bound its in-flight data based on its estimate of the BDP. ++ * ++ * The model has both higher and lower bounds for the operating range: ++ * lo: bw_lo, inflight_lo: conservative short-term lower bound ++ * hi: bw_hi, inflight_hi: robust long-term upper bound ++ * The bandwidth-probing time scale is (a) extended dynamically based on ++ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by ++ * an interactive wall-clock time-scale to be more scalable and responsive ++ * than Reno and CUBIC. ++ * ++ * Here is a state transition diagram for BBR: ++ * ++ * | ++ * V ++ * +---> STARTUP ----+ ++ * | | | ++ * | V | ++ * | DRAIN ----+ ++ * | | | ++ * | V | ++ * +---> PROBE_BW ----+ ++ * | ^ | | ++ * | | | | ++ * | +----+ | ++ * | | ++ * +---- PROBE_RTT <--+ ++ * ++ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. ++ * When it estimates the pipe is full, it enters DRAIN to drain the queue. ++ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. ++ * A long-lived BBR flow spends the vast majority of its time remaining ++ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth ++ * in a fair manner, with a small, bounded queue. *If* a flow has been ++ * continuously sending for the entire min_rtt window, and hasn't seen an RTT ++ * sample that matches or decreases its min_rtt estimate for 10 seconds, then ++ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe ++ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if ++ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; ++ * otherwise we enter STARTUP to try to fill the pipe. ++ * ++ * BBR is described in detail in: ++ * "BBR: Congestion-Based Congestion Control", ++ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, ++ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. ++ * ++ * There is a public e-mail list for discussing BBR development and testing: ++ * https://groups.google.com/forum/#!forum/bbr-dev ++ * ++ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, ++ * otherwise TCP stack falls back to an internal pacing using one high ++ * resolution timer per TCP socket and may use more resources. ++ */ ++#include ++#include ++#include ++#include ++#include ++ ++#include "tcp_dctcp.h" ++ ++/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth ++ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. ++ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. ++ * Since the minimum window is >=4 packets, the lower bound isn't ++ * an issue. The upper bound isn't an issue with existing technologies. ++ */ ++#define BW_SCALE 24 ++#define BW_UNIT (1 << BW_SCALE) ++ ++#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ ++#define BBR_UNIT (1 << BBR_SCALE) ++ ++#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ ++#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ ++ ++#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ ++ ++/* BBR has the following modes for deciding how fast to send: */ ++enum bbr_mode { ++ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ ++ BBR_DRAIN, /* drain any queue created during startup */ ++ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ ++ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ ++}; ++ ++/* How does the incoming ACK stream relate to our bandwidth probing? */ ++enum bbr_ack_phase { ++ BBR_ACKS_INIT, /* not probing; not getting probe feedback */ ++ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ ++ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ ++ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ ++ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ ++}; ++ ++/* BBR congestion control block */ ++struct bbr { ++ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ ++ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ ++ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ ++ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ ++ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ ++ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ ++ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ ++ u64 cycle_mstamp; /* time of this cycle phase start */ ++ u32 mode:3, /* current bbr_mode in state machine */ ++ prev_ca_state:3, /* CA state on previous ACK */ ++ packet_conservation:1, /* use packet conservation? */ ++ round_start:1, /* start of packet-timed tx->ack round? */ ++ ce_state:1, /* If most recent data has CE bit set */ ++ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ ++ try_fast_path:1, /* can we take fast path? */ ++ unused2:11, ++ idle_restart:1, /* restarting after idle? */ ++ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ ++ cycle_idx:3, /* current index in pacing_gain cycle array */ ++ has_seen_rtt:1; /* have we seen an RTT sample yet? */ ++ u32 pacing_gain:11, /* current gain for setting pacing rate */ ++ cwnd_gain:11, /* current gain for setting cwnd */ ++ full_bw_reached:1, /* reached full bw in Startup? */ ++ full_bw_cnt:2, /* number of rounds without large bw gains */ ++ init_cwnd:7; /* initial cwnd */ ++ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ ++ u32 full_bw; /* recent bw, to estimate if pipe is full */ ++ ++ /* For tracking ACK aggregation: */ ++ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ ++ u16 extra_acked[2]; /* max excess data ACKed in epoch */ ++ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ ++ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ ++ extra_acked_win_idx:1, /* current index in extra_acked array */ ++ /* BBR v2 state: */ ++ unused1:2, ++ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ ++ loss_in_cycle:1, /* packet loss in this cycle? */ ++ ecn_in_cycle:1; /* ECN in this cycle? */ ++ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ ++ u32 undo_bw_lo; /* bw_lo before latest losses */ ++ u32 undo_inflight_lo; /* inflight_lo before latest losses */ ++ u32 undo_inflight_hi; /* inflight_hi before latest losses */ ++ u32 bw_latest; /* max delivered bw in last round trip */ ++ u32 bw_lo; /* lower bound on sending bandwidth */ ++ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ ++ u32 inflight_latest; /* max delivered data in last round trip */ ++ u32 inflight_lo; /* lower bound of inflight data range */ ++ u32 inflight_hi; /* upper bound of inflight data range */ ++ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ ++ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ ++ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ ++ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ ++ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ ++ bw_probe_samples:1, /* rate samples reflect bw probing? */ ++ prev_probe_too_high:1, /* did last PROBE_UP go too high? */ ++ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ ++ rounds_since_probe:8, /* packet-timed rounds since probed bw */ ++ loss_round_start:1, /* loss_round_delivered round trip? */ ++ loss_in_round:1, /* loss marked in this round trip? */ ++ ecn_in_round:1, /* ECN marked in this round trip? */ ++ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ ++ loss_events_in_round:4,/* losses in STARTUP round */ ++ initialized:1; /* has bbr_init() been called? */ ++ u32 alpha_last_delivered; /* tp->delivered at alpha update */ ++ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ ++ struct tcp_plb_state plb; ++ ++ /* Params configurable using setsockopt. Refer to correspoding ++ * module param for detailed description of params. ++ */ ++ struct bbr_params { ++ u32 high_gain:11, /* max allowed value: 2047 */ ++ drain_gain:10, /* max allowed value: 1023 */ ++ cwnd_gain:11; /* max allowed value: 2047 */ ++ u32 cwnd_min_target:4, /* max allowed value: 15 */ ++ min_rtt_win_sec:5, /* max allowed value: 31 */ ++ probe_rtt_mode_ms:9, /* max allowed value: 511 */ ++ full_bw_cnt:3, /* max allowed value: 7 */ ++ cwnd_tso_budget:1, /* allowed values: {0, 1} */ ++ unused3:6, ++ drain_to_target:1, /* boolean */ ++ precise_ece_ack:1, /* boolean */ ++ extra_acked_in_startup:1, /* allowed values: {0, 1} */ ++ fast_path:1; /* boolean */ ++ u32 full_bw_thresh:10, /* max allowed value: 1023 */ ++ startup_cwnd_gain:11, /* max allowed value: 2047 */ ++ bw_probe_pif_gain:9, /* max allowed value: 511 */ ++ usage_based_cwnd:1, /* boolean */ ++ unused2:1; ++ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ ++ refill_add_inc:2; /* max allowed value: 3 */ ++ u16 extra_acked_gain:11, /* max allowed value: 2047 */ ++ extra_acked_win_rtts:5; /* max allowed value: 31*/ ++ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ ++ /* Mostly BBR v2 parameters below here: */ ++ u32 ecn_alpha_gain:8, /* max allowed value: 255 */ ++ ecn_factor:8, /* max allowed value: 255 */ ++ ecn_thresh:8, /* max allowed value: 255 */ ++ beta:8; /* max allowed value: 255 */ ++ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ ++ bw_probe_reno_gain:9, /* max allowed value: 511 */ ++ full_loss_cnt:4; /* max allowed value: 15 */ ++ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ ++ inflight_headroom:8, /* max allowed value: 255 */ ++ loss_thresh:8, /* max allowed value: 255 */ ++ bw_probe_max_rounds:8; /* max allowed value: 255 */ ++ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ ++ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ ++ full_ecn_cnt:2; /* max allowed value: 3 */ ++ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ ++ undo:1, /* boolean */ ++ tso_rtt_shift:4, /* max allowed value: 15 */ ++ unused5:1; ++ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ ++ unused1:14, ++ ecn_alpha_init:9; /* max allowed value: 256 */ ++ } params; ++ ++ struct { ++ u32 snd_isn; /* Initial sequence number */ ++ u32 rs_bw; /* last valid rate sample bw */ ++ u32 target_cwnd; /* target cwnd, based on BDP */ ++ u8 undo:1, /* Undo even happened but not yet logged */ ++ unused:7; ++ char event; /* single-letter event debug codes */ ++ u16 unused2; ++ } debug; ++}; ++ ++struct bbr_context { ++ u32 sample_bw; ++ u32 target_cwnd; ++ u32 log:1; ++}; ++ ++/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ ++static u32 bbr_min_rtt_win_sec = 10; ++/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. ++ * Max allowed value is 511 (0x1FF). ++ */ ++static u32 bbr_probe_rtt_mode_ms = 200; ++/* Window length of probe_rtt_min_us filter (in ms), and consequently the ++ * typical interval between PROBE_RTT mode entries. ++ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC ++ */ ++static u32 bbr_probe_rtt_win_ms = 5000; ++/* Skip TSO below the following bandwidth (bits/sec): */ ++static int bbr_min_tso_rate = 1200000; ++ ++/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting ++ * in bigger TSO bursts. By default we cut the RTT-based allowance in half ++ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance ++ * is below 1500 bytes after 6 * ~500 usec = 3ms. ++ */ ++static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ ++ ++/* Select cwnd TSO budget approach: ++ * 0: padding ++ * 1: flooring ++ */ ++static uint bbr_cwnd_tso_budget = 1; ++ ++/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. ++ * In order to help drive the network toward lower queues and low latency while ++ * maintaining high utilization, the average pacing rate aims to be slightly ++ * lower than the estimated bandwidth. This is an important aspect of the ++ * design. ++ */ ++static const int bbr_pacing_margin_percent = 1; ++ ++/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain ++ * that will allow a smoothly increasing pacing rate that will double each RTT ++ * and send the same number of packets per RTT that an un-paced, slow-starting ++ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). ++ */ ++static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; ++/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ ++static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; ++/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain ++ * the queue created in BBR_STARTUP in a single round. Max allowed value ++ * is 1023 (0x3FF). ++ */ ++static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; ++/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. ++ * Max allowed value is 2047 (0x7FF). ++ */ ++static int bbr_cwnd_gain = BBR_UNIT * 2; ++/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. ++ * Max allowed value for each element is 1023 (0x3FF). ++ */ ++enum bbr_pacing_gain_phase { ++ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ ++ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ ++ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ ++ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ ++}; ++static int bbr_pacing_gain[] = { ++ BBR_UNIT * 5 / 4, /* probe for more available bw */ ++ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ ++ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ ++ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ ++}; ++ ++/* Try to keep at least this many packets in flight, if things go smoothly. For ++ * smooth functioning, a sliding window protocol ACKing every other packet ++ * needs at least 4 packets in flight. Max allowed value is 15 (0xF). ++ */ ++static u32 bbr_cwnd_min_target = 4; ++ ++/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. ++ * Use 0 to disable. Max allowed value is 255. ++ */ ++static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; ++ ++/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ ++/* If bw has increased significantly (1.25x), there may be more bw available. ++ * Max allowed value is 1023 (0x3FF). ++ */ ++static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; ++/* But after 3 rounds w/o significant bw growth, estimate pipe is full. ++ * Max allowed value is 7 (0x7). ++ */ ++static u32 bbr_full_bw_cnt = 3; ++ ++static u32 bbr_flags; /* Debugging related stuff */ ++ ++/* Whether to debug using printk. ++ */ ++static bool bbr_debug_with_printk; ++ ++/* Whether to debug using ftrace event tcp:tcp_bbr_event. ++ * Ignored when bbr_debug_with_printk is set. ++ */ ++static bool bbr_debug_ftrace; ++ ++/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ ++static bool bbr_drain_to_target = true; /* default: enabled */ ++ ++/* Experiment: Flags to control BBR with ECN behavior. ++ */ ++static bool bbr_precise_ece_ack = true; /* default: enabled */ ++ ++/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is ++ * (2^(16+14) B)/(1024 B/packet) = 1M packets. ++ */ ++static u32 bbr_cwnd_warn_val = 1U << 20; ++ ++static u16 bbr_debug_port_mask; ++ ++/* BBR module parameters. These are module parameters only in Google prod. ++ * Upstream these are intentionally not module parameters. ++ */ ++static int bbr_pacing_gain_size = CYCLE_LEN; ++ ++/* Gain factor for adding extra_acked to target cwnd: */ ++static int bbr_extra_acked_gain = 256; ++ ++/* Window length of extra_acked window. Max allowed val is 31. */ ++static u32 bbr_extra_acked_win_rtts = 5; ++ ++/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ ++static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; ++ ++/* Time period for clamping cwnd increment due to ack aggregation */ ++static u32 bbr_extra_acked_max_us = 100 * 1000; ++ ++/* Use extra acked in startup ? ++ * 0: disabled ++ * 1: use latest extra_acked value from 1-2 rtt in startup ++ */ ++static int bbr_extra_acked_in_startup = 1; /* default: enabled */ ++ ++/* Experiment: don't grow cwnd beyond twice of what we just probed. */ ++static bool bbr_usage_based_cwnd; /* default: disabled */ ++ ++/* For lab testing, researchers can enable BBRv2 ECN support with this flag, ++ * when they know that any ECN marks that the connections experience will be ++ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. ++ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on ++ * negotiation or configuration that is outside the scope of the BBRv2 ++ * alpha release. ++ */ ++static bool bbr_ecn_enable = false; ++ ++module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); ++module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); ++module_param_named(high_gain, bbr_high_gain, int, 0644); ++module_param_named(drain_gain, bbr_drain_gain, int, 0644); ++module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); ++module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); ++module_param_array_named(pacing_gain, bbr_pacing_gain, int, ++ &bbr_pacing_gain_size, 0644); ++module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); ++module_param_named(probe_rtt_cwnd_gain, ++ bbr_probe_rtt_cwnd_gain, uint, 0664); ++module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); ++module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); ++module_param_named(flags, bbr_flags, uint, 0644); ++module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); ++module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); ++module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); ++module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); ++module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); ++module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); ++module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); ++module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); ++module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); ++module_param_named(extra_acked_win_rtts, ++ bbr_extra_acked_win_rtts, uint, 0664); ++module_param_named(extra_acked_max_us, ++ bbr_extra_acked_max_us, uint, 0664); ++module_param_named(ack_epoch_acked_reset_thresh, ++ bbr_ack_epoch_acked_reset_thresh, uint, 0664); ++module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); ++module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); ++module_param_named(extra_acked_in_startup, ++ bbr_extra_acked_in_startup, int, 0664); ++module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); ++module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); ++ ++static void bbr2_exit_probe_rtt(struct sock *sk); ++static void bbr2_reset_congestion_signals(struct sock *sk); ++ ++static void bbr_check_probe_rtt_done(struct sock *sk); ++ ++/* Do we estimate that STARTUP filled the pipe? */ ++static bool bbr_full_bw_reached(const struct sock *sk) ++{ ++ const struct bbr *bbr = inet_csk_ca(sk); ++ ++ return bbr->full_bw_reached; ++} ++ ++/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ ++static u32 bbr_max_bw(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return max(bbr->bw_hi[0], bbr->bw_hi[1]); ++} ++ ++/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ ++static u32 bbr_bw(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return min(bbr_max_bw(sk), bbr->bw_lo); ++} ++ ++/* Return maximum extra acked in past k-2k round trips, ++ * where k = bbr_extra_acked_win_rtts. ++ */ ++static u16 bbr_extra_acked(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return max(bbr->extra_acked[0], bbr->extra_acked[1]); ++} ++ ++/* Return rate in bytes per second, optionally with a gain. ++ * The order here is chosen carefully to avoid overflow of u64. This should ++ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. ++ */ ++static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, ++ int margin) ++{ ++ unsigned int mss = tcp_sk(sk)->mss_cache; ++ ++ rate *= mss; ++ rate *= gain; ++ rate >>= BBR_SCALE; ++ rate *= USEC_PER_SEC / 100 * (100 - margin); ++ rate >>= BW_SCALE; ++ rate = max(rate, 1ULL); ++ return rate; ++} ++ ++static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) ++{ ++ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); ++} ++ ++static u64 bbr_rate_kbps(struct sock *sk, u64 rate) ++{ ++ rate = bbr_bw_bytes_per_sec(sk, rate); ++ rate *= 8; ++ do_div(rate, 1000); ++ return rate; ++} ++ ++static u32 bbr_tso_segs_goal(struct sock *sk); ++static void bbr_debug(struct sock *sk, u32 acked, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ static const char ca_states[] = { ++ [TCP_CA_Open] = 'O', ++ [TCP_CA_Disorder] = 'D', ++ [TCP_CA_CWR] = 'C', ++ [TCP_CA_Recovery] = 'R', ++ [TCP_CA_Loss] = 'L', ++ }; ++ static const char mode[] = { ++ 'G', /* Growing - BBR_STARTUP */ ++ 'D', /* Drain - BBR_DRAIN */ ++ 'W', /* Window - BBR_PROBE_BW */ ++ 'M', /* Min RTT - BBR_PROBE_RTT */ ++ }; ++ static const char ack_phase[] = { /* bbr_ack_phase strings */ ++ 'I', /* BBR_ACKS_INIT - 'Init' */ ++ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ ++ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ ++ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ ++ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ ++ }; ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ const u32 una = tp->snd_una - bbr->debug.snd_isn; ++ const u32 fack = tcp_highest_sack_seq(tp); ++ const u16 dport = ntohs(inet_sk(sk)->inet_dport); ++ bool is_port_match = (bbr_debug_port_mask && ++ ((dport & bbr_debug_port_mask) == 0)); ++ char debugmsg[320]; ++ ++ if (sk->sk_state == TCP_SYN_SENT) ++ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ ++ ++ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { ++ char addr[INET6_ADDRSTRLEN + 10] = { 0 }; ++ ++ if (sk->sk_family == AF_INET) ++ snprintf(addr, sizeof(addr), "%pI4:%u", ++ &inet_sk(sk)->inet_daddr, dport); ++ else if (sk->sk_family == AF_INET6) ++ snprintf(addr, sizeof(addr), "%pI6:%u", ++ &sk->sk_v6_daddr, dport); ++ ++ WARN_ONCE(1, ++ "BBR %s cwnd alert: %u " ++ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " ++ "bw: %u rtt: %u min_rtt: %u " ++ "acked: %u tso_segs: %u " ++ "bw: %d %ld %d pif: %u\n", ++ addr, tp->snd_cwnd, ++ una, inet_csk(sk)->icsk_ca_state, ++ bbr->pacing_gain, bbr->cwnd_gain, ++ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, ++ acked, bbr_tso_segs_goal(sk), ++ rs->delivered, rs->interval_us, rs->is_retrans, ++ tcp_packets_in_flight(tp)); ++ } ++ ++ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) ++ return; ++ ++ if (!sock_flag(sk, SOCK_DBG) && !is_port_match) ++ return; ++ ++ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) ++ return; ++ ++ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && ++ !(bbr_flags & FLAG_DEBUG_LOOPBACK)) ++ return; ++ ++ snprintf(debugmsg, sizeof(debugmsg) - 1, ++ "BBR %pI4:%-5u %5u,%03u:%-7u %c " ++ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " ++ "bw %llu lb %llu ib %llu qb %llu " ++ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " ++ "lr %d er %d ea %d bwl %lld il %d ih %d c %d " ++ "v %d %c %u %c %s\n", ++ &inet_sk(sk)->inet_daddr, dport, ++ una / 1000, una % 1000, fack - tp->snd_una, ++ ca_states[inet_csk(sk)->icsk_ca_state], ++ bbr->debug.undo ? '@' : mode[bbr->mode], ++ tp->snd_cwnd, ++ bbr_extra_acked(sk), /* br (legacy): extra_acked */ ++ rs->tx_in_flight, /* cr (legacy): tx_inflight */ ++ rs->rtt_us, ++ rs->delivered, ++ rs->interval_us, ++ bbr->min_rtt_us, ++ rs->is_app_limited ? '_' : 'l', ++ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ ++ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ ++ 0ULL, /* lb: [obsolete] */ ++ 0ULL, /* ib: [obsolete] */ ++ div_u64((u64)sk->sk_pacing_rate * 8, 1000), ++ acked, ++ tcp_packets_in_flight(tp), ++ rs->is_ack_delayed ? 'd' : '.', ++ bbr->round_start ? '*' : '.', ++ tp->delivered, tp->lost, ++ tp->app_limited, ++ 0, /* #: [obsolete] */ ++ ctx->target_cwnd, ++ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ ++ ca_states[bbr->prev_ca_state], ++ (rs->lost + rs->delivered) > 0 ? ++ (1000 * rs->lost / ++ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ ++ (rs->delivered) > 0 ? ++ (1000 * rs->delivered_ce / ++ (rs->delivered)) : 0, /* er: ECN rate x1000 */ ++ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ ++ bbr->bw_lo == ~0U ? ++ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ ++ bbr->inflight_lo, /* il */ ++ bbr->inflight_hi, /* ih */ ++ bbr->bw_probe_up_cnt, /* c */ ++ 2, /* v: version */ ++ bbr->debug.event, ++ bbr->cycle_idx, ++ ack_phase[bbr->ack_phase], ++ bbr->bw_probe_samples ? "Y" : "N"); ++ debugmsg[sizeof(debugmsg) - 1] = 0; ++ ++ /* printk takes a higher precedence. */ ++ if (bbr_debug_with_printk) ++ printk(KERN_DEBUG "%s", debugmsg); ++ ++ if (unlikely(bbr->debug.undo)) ++ bbr->debug.undo = 0; ++} ++ ++/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ ++static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) ++{ ++ u64 rate = bw; ++ ++ rate = bbr_rate_bytes_per_sec(sk, rate, gain, ++ bbr_pacing_margin_percent); ++ rate = min_t(u64, rate, sk->sk_max_pacing_rate); ++ return rate; ++} ++ ++/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ ++static void bbr_init_pacing_rate_from_rtt(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw; ++ u32 rtt_us; ++ ++ if (tp->srtt_us) { /* any RTT sample yet? */ ++ rtt_us = max(tp->srtt_us >> 3, 1U); ++ bbr->has_seen_rtt = 1; ++ } else { /* no RTT sample yet */ ++ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ ++ } ++ bw = (u64)tp->snd_cwnd * BW_UNIT; ++ do_div(bw, rtt_us); ++ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); ++} ++ ++/* Pace using current bw estimate and a gain factor. */ ++static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); ++ ++ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) ++ bbr_init_pacing_rate_from_rtt(sk); ++ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) ++ sk->sk_pacing_rate = rate; ++} ++ ++static u32 bbr_min_tso_segs(struct sock *sk) ++{ ++ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; ++} ++ ++/* Return the number of segments BBR would like in a TSO/GSO skb, given ++ * a particular max gso size as a constraint. ++ */ ++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, ++ u32 gso_max_size) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 segs, r; ++ u64 bytes; ++ ++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ ++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; ++ ++ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every ++ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. ++ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) ++ */ ++ if (bbr->params.tso_rtt_shift) { ++ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; ++ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ ++ bytes += GSO_MAX_SIZE >> r; ++ } ++ ++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); ++ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); ++ return segs; ++} ++ ++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ ++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) ++{ ++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); ++} ++ ++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ ++static u32 bbr_tso_segs_goal(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ ++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); ++} ++ ++/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ ++static void bbr_save_cwnd(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) ++ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ ++ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ ++ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); ++} ++ ++static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (event == CA_EVENT_TX_START) { ++ tcp_plb_check_rehash(sk, &bbr->plb); ++ ++ if (!tp->app_limited) ++ return; ++ bbr->idle_restart = 1; ++ bbr->ack_epoch_mstamp = tp->tcp_mstamp; ++ bbr->ack_epoch_acked = 0; ++ /* Avoid pointless buffer overflows: pace at est. bw if we don't ++ * need more speed (we're restarting from idle and app-limited). ++ */ ++ if (bbr->mode == BBR_PROBE_BW) ++ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); ++ else if (bbr->mode == BBR_PROBE_RTT) ++ bbr_check_probe_rtt_done(sk); ++ } else if ((event == CA_EVENT_ECN_IS_CE || ++ event == CA_EVENT_ECN_NO_CE) && ++ bbr_ecn_enable && ++ bbr->params.precise_ece_ack) { ++ u32 state = bbr->ce_state; ++ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); ++ bbr->ce_state = state; ++ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) ++ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); ++ } ++} ++ ++/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: ++ * ++ * bdp = ceil(bw * min_rtt * gain) ++ * ++ * The key factor, gain, controls the amount of queue. While a small gain ++ * builds a smaller queue, it becomes more vulnerable to noise in RTT ++ * measurements (e.g., delayed ACKs or other ACK compression effects). This ++ * noise may cause BBR to under-estimate the rate. ++ */ ++static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 bdp; ++ u64 w; ++ ++ /* If we've never had a valid RTT sample, cap cwnd at the initial ++ * default. This should only happen when the connection is not using TCP ++ * timestamps and has retransmitted all of the SYN/SYNACK/data packets ++ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which ++ * case we need to slow-start up toward something safe: initial cwnd. ++ */ ++ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ ++ return bbr->init_cwnd; /* be safe: cap at initial cwnd */ ++ ++ w = (u64)bw * bbr->min_rtt_us; ++ ++ /* Apply a gain to the given value, remove the BW_SCALE shift, and ++ * round the value up to avoid a negative feedback loop. ++ */ ++ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ ++ return bdp; ++} ++ ++/* To achieve full performance in high-speed paths, we budget enough cwnd to ++ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: ++ * - one skb in sending host Qdisc, ++ * - one skb in sending host TSO/GSO engine ++ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine ++ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because ++ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, ++ * which allows 2 outstanding 2-packet sequences, to try to keep pipe ++ * full even with ACK-every-other-packet delayed ACKs. ++ */ ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 tso_segs_goal; ++ ++ tso_segs_goal = 3 * bbr_tso_segs_goal(sk); ++ ++ /* Allow enough full-sized skbs in flight to utilize end systems. */ ++ if (bbr->params.cwnd_tso_budget == 1) { ++ cwnd = max_t(u32, cwnd, tso_segs_goal); ++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); ++ } else { ++ cwnd += tso_segs_goal; ++ cwnd = (cwnd + 1) & ~1U; ++ } ++ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ ++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) ++ cwnd += 2; ++ ++ return cwnd; ++} ++ ++/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ ++static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) ++{ ++ u32 inflight; ++ ++ inflight = bbr_bdp(sk, bw, gain); ++ inflight = bbr_quantization_budget(sk, inflight); ++ ++ return inflight; ++} ++ ++/* With pacing at lower layers, there's often less data "in the network" than ++ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), ++ * we often have several skbs queued in the pacing layer with a pre-scheduled ++ * earliest departure time (EDT). BBR adapts its pacing rate based on the ++ * inflight level that it estimates has already been "baked in" by previous ++ * departure time decisions. We calculate a rough estimate of the number of our ++ * packets that might be in the network at the earliest departure time for the ++ * next skb scheduled: ++ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw ++ * If we're increasing inflight, then we want to know if the transmit of the ++ * EDT skb will push inflight above the target, so inflight_at_edt includes ++ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, ++ * then estimate if inflight will sink too low just before the EDT transmit. ++ */ ++static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 now_ns, edt_ns, interval_us; ++ u32 interval_delivered, inflight_at_edt; ++ ++ now_ns = tp->tcp_clock_cache; ++ edt_ns = max(tp->tcp_wstamp_ns, now_ns); ++ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); ++ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; ++ inflight_at_edt = inflight_now; ++ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ ++ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ ++ if (interval_delivered >= inflight_at_edt) ++ return 0; ++ return inflight_at_edt - interval_delivered; ++} ++ ++/* Find the cwnd increment based on estimate of ack aggregation */ ++static u32 bbr_ack_aggregation_cwnd(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 max_aggr_cwnd, aggr_cwnd = 0; ++ ++ if (bbr->params.extra_acked_gain && ++ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { ++ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) ++ / BW_UNIT; ++ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) ++ >> BBR_SCALE; ++ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); ++ } ++ ++ return aggr_cwnd; ++} ++ ++/* Returns the cwnd for PROBE_RTT mode. */ ++static u32 bbr_probe_rtt_cwnd(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->params.probe_rtt_cwnd_gain == 0) ++ return bbr->params.cwnd_min_target; ++ return max_t(u32, bbr->params.cwnd_min_target, ++ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); ++} ++ ++/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss ++ * has drawn us down below target), or snap down to target if we're above it. ++ */ ++static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, ++ u32 acked, u32 bw, int gain, u32 cwnd, ++ struct bbr_context *ctx) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; ++ ++ if (!acked) ++ goto done; /* no packet fully ACKed; just apply caps */ ++ ++ target_cwnd = bbr_bdp(sk, bw, gain); ++ ++ /* Increment the cwnd to account for excess ACKed data that seems ++ * due to aggregation (of data and/or ACKs) visible in the ACK stream. ++ */ ++ target_cwnd += bbr_ack_aggregation_cwnd(sk); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd); ++ ++ /* If we're below target cwnd, slow start cwnd toward target cwnd. */ ++ bbr->debug.target_cwnd = target_cwnd; ++ ++ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ ++ bbr->try_fast_path = 0; ++ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ ++ cwnd += acked; ++ if (cwnd >= target_cwnd) { ++ cwnd = target_cwnd; ++ bbr->try_fast_path = 1; ++ } ++ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { ++ cwnd += acked; ++ } else { ++ bbr->try_fast_path = 1; ++ } ++ ++ /* When growing cwnd, don't grow beyond twice what we just probed. */ ++ if (bbr->params.usage_based_cwnd) { ++ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); ++ cwnd = min(cwnd, max_probe); ++ } ++ ++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); ++done: ++ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ ++ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ ++ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); ++ ++ ctx->target_cwnd = target_cwnd; ++ ctx->log = (tp->snd_cwnd != prev_cwnd); ++} ++ ++/* See if we have reached next round trip */ ++static void bbr_update_round_start(struct sock *sk, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->round_start = 0; ++ ++ /* See if we've reached the next RTT */ ++ if (rs->interval_us > 0 && ++ !before(rs->prior_delivered, bbr->next_rtt_delivered)) { ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr->round_start = 1; ++ } ++} ++ ++/* Calculate the bandwidth based on how fast packets are delivered */ ++static void bbr_calculate_bw_sample(struct sock *sk, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw = 0; ++ ++ /* Divide delivered by the interval to find a (lower bound) bottleneck ++ * bandwidth sample. Delivered is in packets and interval_us in uS and ++ * ratio will be <<1 for most connections. So delivered is first scaled. ++ * Round up to allow growth at low rates, even with integer division. ++ */ ++ if (rs->interval_us > 0) { ++ if (WARN_ONCE(rs->delivered < 0, ++ "negative delivered: %d interval_us: %ld\n", ++ rs->delivered, rs->interval_us)) ++ return; ++ ++ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); ++ } ++ ++ ctx->sample_bw = bw; ++ bbr->debug.rs_bw = bw; ++} ++ ++/* Estimates the windowed max degree of ack aggregation. ++ * This is used to provision extra in-flight data to keep sending during ++ * inter-ACK silences. ++ * ++ * Degree of ack aggregation is estimated as extra data acked beyond expected. ++ * ++ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" ++ * cwnd += max_extra_acked ++ * ++ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). ++ * Max filter is an approximate sliding window of 5-10 (packet timed) round ++ * trips for non-startup phase, and 1-2 round trips for startup. ++ */ ++static void bbr_update_ack_aggregation(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ u32 epoch_us, expected_acked, extra_acked; ++ struct bbr *bbr = inet_csk_ca(sk); ++ struct tcp_sock *tp = tcp_sk(sk); ++ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; ++ ++ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || ++ rs->delivered < 0 || rs->interval_us <= 0) ++ return; ++ ++ if (bbr->round_start) { ++ bbr->extra_acked_win_rtts = min(0x1F, ++ bbr->extra_acked_win_rtts + 1); ++ if (bbr->params.extra_acked_in_startup && ++ !bbr_full_bw_reached(sk)) ++ extra_acked_win_rtts_thresh = 1; ++ if (bbr->extra_acked_win_rtts >= ++ extra_acked_win_rtts_thresh) { ++ bbr->extra_acked_win_rtts = 0; ++ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? ++ 0 : 1; ++ bbr->extra_acked[bbr->extra_acked_win_idx] = 0; ++ } ++ } ++ ++ /* Compute how many packets we expected to be delivered over epoch. */ ++ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, ++ bbr->ack_epoch_mstamp); ++ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; ++ ++ /* Reset the aggregation epoch if ACK rate is below expected rate or ++ * significantly large no. of ack received since epoch (potentially ++ * quite old epoch). ++ */ ++ if (bbr->ack_epoch_acked <= expected_acked || ++ (bbr->ack_epoch_acked + rs->acked_sacked >= ++ bbr_ack_epoch_acked_reset_thresh)) { ++ bbr->ack_epoch_acked = 0; ++ bbr->ack_epoch_mstamp = tp->delivered_mstamp; ++ expected_acked = 0; ++ } ++ ++ /* Compute excess data delivered, beyond what was expected. */ ++ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, ++ bbr->ack_epoch_acked + rs->acked_sacked); ++ extra_acked = bbr->ack_epoch_acked - expected_acked; ++ extra_acked = min(extra_acked, tp->snd_cwnd); ++ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) ++ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; ++} ++ ++/* Estimate when the pipe is full, using the change in delivery rate: BBR ++ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by ++ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited ++ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the ++ * higher rwin, 3: we get higher delivery rate samples. Or transient ++ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar ++ * design goal, but uses delay and inter-ACK spacing instead of bandwidth. ++ */ ++static void bbr_check_full_bw_reached(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 bw_thresh; ++ ++ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) ++ return; ++ ++ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; ++ if (bbr_max_bw(sk) >= bw_thresh) { ++ bbr->full_bw = bbr_max_bw(sk); ++ bbr->full_bw_cnt = 0; ++ return; ++ } ++ ++bbr->full_bw_cnt; ++ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; ++} ++ ++/* If pipe is probably full, drain the queue and then enter steady-state. */ ++static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, ++ struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { ++ bbr->mode = BBR_DRAIN; /* drain queue we created */ ++ tcp_sk(sk)->snd_ssthresh = ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); ++ bbr2_reset_congestion_signals(sk); ++ } /* fall through to check if in-flight is already small: */ ++ if (bbr->mode == BBR_DRAIN && ++ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) ++ return true; /* exiting DRAIN now */ ++ return false; ++} ++ ++static void bbr_check_probe_rtt_done(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (!(bbr->probe_rtt_done_stamp && ++ after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) ++ return; ++ ++ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ ++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); ++ bbr2_exit_probe_rtt(sk); ++} ++ ++/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and ++ * periodically drain the bottleneck queue, to converge to measure the true ++ * min_rtt (unloaded propagation delay). This allows the flows to keep queues ++ * small (reducing queuing delay and packet loss) and achieve fairness among ++ * BBR flows. ++ * ++ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, ++ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. ++ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed ++ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and ++ * re-enter the previous mode. BBR uses 200ms to approximately bound the ++ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). ++ * ++ * Note that flows need only pay 2% if they are busy sending over the last 10 ++ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have ++ * natural silences or low-rate periods within 10 seconds where the rate is low ++ * enough for long enough to drain its queue in the bottleneck. We pick up ++ * these min RTT measurements opportunistically with our min_rtt filter. :-) ++ */ ++static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ bool probe_rtt_expired, min_rtt_expired; ++ u32 expire; ++ ++ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ ++ expire = bbr->probe_rtt_min_stamp + ++ msecs_to_jiffies(bbr->params.probe_rtt_win_ms); ++ probe_rtt_expired = after(tcp_jiffies32, expire); ++ if (rs->rtt_us >= 0 && ++ (rs->rtt_us <= bbr->probe_rtt_min_us || ++ (probe_rtt_expired && !rs->is_ack_delayed))) { ++ bbr->probe_rtt_min_us = rs->rtt_us; ++ bbr->probe_rtt_min_stamp = tcp_jiffies32; ++ } ++ /* Track min RTT seen in the min_rtt_win_sec filter window: */ ++ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; ++ min_rtt_expired = after(tcp_jiffies32, expire); ++ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || ++ min_rtt_expired) { ++ bbr->min_rtt_us = bbr->probe_rtt_min_us; ++ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; ++ } ++ ++ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && ++ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { ++ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ ++ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ ++ bbr->probe_rtt_done_stamp = 0; ++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; ++ bbr->next_rtt_delivered = tp->delivered; ++ } ++ ++ if (bbr->mode == BBR_PROBE_RTT) { ++ /* Ignore low rate samples during this mode. */ ++ tp->app_limited = ++ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; ++ /* Maintain min packets in flight for max(200 ms, 1 round). */ ++ if (!bbr->probe_rtt_done_stamp && ++ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { ++ bbr->probe_rtt_done_stamp = tcp_jiffies32 + ++ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); ++ bbr->probe_rtt_round_done = 0; ++ bbr->next_rtt_delivered = tp->delivered; ++ } else if (bbr->probe_rtt_done_stamp) { ++ if (bbr->round_start) ++ bbr->probe_rtt_round_done = 1; ++ if (bbr->probe_rtt_round_done) ++ bbr_check_probe_rtt_done(sk); ++ } ++ } ++ /* Restart after idle ends only once we process a new S/ACK for data */ ++ if (rs->delivered > 0) ++ bbr->idle_restart = 0; ++} ++ ++static void bbr_update_gains(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ switch (bbr->mode) { ++ case BBR_STARTUP: ++ bbr->pacing_gain = bbr->params.high_gain; ++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; ++ break; ++ case BBR_DRAIN: ++ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ ++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ ++ break; ++ case BBR_PROBE_BW: ++ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; ++ bbr->cwnd_gain = bbr->params.cwnd_gain; ++ break; ++ case BBR_PROBE_RTT: ++ bbr->pacing_gain = BBR_UNIT; ++ bbr->cwnd_gain = BBR_UNIT; ++ break; ++ default: ++ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); ++ break; ++ } ++} ++ ++static void bbr_init(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ int i; ++ ++ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); ++ ++ bbr->initialized = 1; ++ bbr->params.high_gain = min(0x7FF, bbr_high_gain); ++ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); ++ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); ++ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); ++ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); ++ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); ++ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); ++ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); ++ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); ++ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); ++ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); ++ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); ++ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; ++ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; ++ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; ++ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); ++ bbr->params.probe_rtt_win_ms = ++ min(0x3FFFU, ++ min_t(u32, bbr_probe_rtt_win_ms, ++ bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); ++ for (i = 0; i < CYCLE_LEN; i++) ++ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); ++ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; ++ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); ++ ++ bbr->debug.snd_isn = tp->snd_una; ++ bbr->debug.target_cwnd = 0; ++ bbr->debug.undo = 0; ++ ++ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); ++ bbr->prior_cwnd = tp->prior_cwnd; ++ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; ++ bbr->next_rtt_delivered = 0; ++ bbr->prev_ca_state = TCP_CA_Open; ++ bbr->packet_conservation = 0; ++ ++ bbr->probe_rtt_done_stamp = 0; ++ bbr->probe_rtt_round_done = 0; ++ bbr->probe_rtt_min_us = tcp_min_rtt(tp); ++ bbr->probe_rtt_min_stamp = tcp_jiffies32; ++ bbr->min_rtt_us = tcp_min_rtt(tp); ++ bbr->min_rtt_stamp = tcp_jiffies32; ++ ++ bbr->has_seen_rtt = 0; ++ bbr_init_pacing_rate_from_rtt(sk); ++ ++ bbr->round_start = 0; ++ bbr->idle_restart = 0; ++ bbr->full_bw_reached = 0; ++ bbr->full_bw = 0; ++ bbr->full_bw_cnt = 0; ++ bbr->cycle_mstamp = 0; ++ bbr->cycle_idx = 0; ++ bbr->mode = BBR_STARTUP; ++ bbr->debug.rs_bw = 0; ++ ++ bbr->ack_epoch_mstamp = tp->tcp_mstamp; ++ bbr->ack_epoch_acked = 0; ++ bbr->extra_acked_win_rtts = 0; ++ bbr->extra_acked_win_idx = 0; ++ bbr->extra_acked[0] = 0; ++ bbr->extra_acked[1] = 0; ++ ++ bbr->ce_state = 0; ++ bbr->prior_rcv_nxt = tp->rcv_nxt; ++ bbr->try_fast_path = 0; ++ ++ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); ++} ++ ++static u32 bbr_sndbuf_expand(struct sock *sk) ++{ ++ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ ++ return 3; ++} ++ ++/* __________________________________________________________________________ ++ * ++ * Functions new to BBR v2 ("bbr") congestion control are below here. ++ * __________________________________________________________________________ ++ */ ++ ++/* Incorporate a new bw sample into the current window of our max filter. */ ++static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); ++} ++ ++/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ ++static void bbr2_advance_bw_hi_filter(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (!bbr->bw_hi[1]) ++ return; /* no samples in this window; remember old window */ ++ bbr->bw_hi[0] = bbr->bw_hi[1]; ++ bbr->bw_hi[1] = 0; ++} ++ ++/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ ++static u32 bbr2_target_inflight(struct sock *sk) ++{ ++ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); ++ ++ return min(bdp, tcp_sk(sk)->snd_cwnd); ++} ++ ++static bool bbr2_is_probing_bandwidth(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return (bbr->mode == BBR_STARTUP) || ++ (bbr->mode == BBR_PROBE_BW && ++ (bbr->cycle_idx == BBR_BW_PROBE_REFILL || ++ bbr->cycle_idx == BBR_BW_PROBE_UP)); ++} ++ ++/* Has the given amount of time elapsed since we marked the phase start? */ ++static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) ++{ ++ const struct tcp_sock *tp = tcp_sk(sk); ++ const struct bbr *bbr = inet_csk_ca(sk); ++ ++ return tcp_stamp_us_delta(tp->tcp_mstamp, ++ bbr->cycle_mstamp + interval_us) > 0; ++} ++ ++static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->full_bw_reached = 1; ++ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); ++} ++ ++/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ ++static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || ++ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) ++ return; ++ ++ if (ce_ratio >= bbr->params.ecn_thresh) ++ bbr->startup_ecn_rounds++; ++ else ++ bbr->startup_ecn_rounds = 0; ++ ++ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { ++ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ ++ bbr2_handle_queue_too_high_in_startup(sk); ++ return; ++ } ++} ++ ++static int bbr2_update_ecn_alpha(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ s32 delivered, delivered_ce; ++ u64 alpha, ce_ratio; ++ u32 gain; ++ ++ if (bbr->params.ecn_factor == 0) ++ return -1; ++ ++ delivered = tp->delivered - bbr->alpha_last_delivered; ++ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; ++ ++ if (delivered == 0 || /* avoid divide by zero */ ++ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ ++ return -1; ++ ++ /* See if we should use ECN sender logic for this connection. */ ++ if (!bbr->ecn_eligible && bbr_ecn_enable && ++ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || ++ !bbr->params.ecn_max_rtt_us)) ++ bbr->ecn_eligible = 1; ++ ++ ce_ratio = (u64)delivered_ce << BBR_SCALE; ++ do_div(ce_ratio, delivered); ++ gain = bbr->params.ecn_alpha_gain; ++ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; ++ alpha += (gain * ce_ratio) >> BBR_SCALE; ++ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); ++ ++ bbr->alpha_last_delivered = tp->delivered; ++ bbr->alpha_last_delivered_ce = tp->delivered_ce; ++ ++ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); ++ return (int)ce_ratio; ++} ++ ++/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ ++static void bbr2_raise_inflight_hi_slope(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 growth_this_round, cnt; ++ ++ /* Calculate "slope": packets S/Acked per inflight_hi increment. */ ++ growth_this_round = 1 << bbr->bw_probe_up_rounds; ++ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); ++ cnt = tp->snd_cwnd / growth_this_round; ++ cnt = max(cnt, 1U); ++ bbr->bw_probe_up_cnt = cnt; ++ bbr->debug.event = 'G'; /* Grow inflight_hi slope */ ++} ++ ++/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ ++static void bbr2_probe_inflight_hi_upward(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 delta; ++ ++ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { ++ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ ++ return; /* not fully using inflight_hi, so don't grow it */ ++ } ++ ++ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ ++ bbr->bw_probe_up_acks += rs->acked_sacked; ++ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { ++ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; ++ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; ++ bbr->inflight_hi += delta; ++ bbr->debug.event = 'I'; /* Increment inflight_hi */ ++ } ++ ++ if (bbr->round_start) ++ bbr2_raise_inflight_hi_slope(sk); ++} ++ ++/* Does loss/ECN rate for this sample say inflight is "too high"? ++ * This is used by both the bbr_check_loss_too_high_in_startup() function, ++ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which ++ * uses it to notice when loss/ECN rates suggest inflight is too high. ++ */ ++static bool bbr2_is_inflight_too_high(const struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ const struct bbr *bbr = inet_csk_ca(sk); ++ u32 loss_thresh, ecn_thresh; ++ ++ if (rs->lost > 0 && rs->tx_in_flight) { ++ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> ++ BBR_SCALE; ++ if (rs->lost > loss_thresh) ++ return true; ++ } ++ ++ if (rs->delivered_ce > 0 && rs->delivered > 0 && ++ bbr->ecn_eligible && bbr->params.ecn_thresh) { ++ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> ++ BBR_SCALE; ++ if (rs->delivered_ce >= ecn_thresh) ++ return true; ++ } ++ ++ return false; ++} ++ ++/* Calculate the tx_in_flight level that corresponded to excessive loss. ++ * We find "lost_prefix" segs of the skb where loss rate went too high, ++ * by solving for "lost_prefix" in the following equation: ++ * lost / inflight >= loss_thresh ++ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh ++ * Then we take that equation, convert it to fixed point, and ++ * round up to the nearest packet. ++ */ ++static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, ++ const struct rate_sample *rs, ++ const struct sk_buff *skb) ++{ ++ const struct bbr *bbr = inet_csk_ca(sk); ++ u32 loss_thresh = bbr->params.loss_thresh; ++ u32 pcount, divisor, inflight_hi; ++ s32 inflight_prev, lost_prev; ++ u64 loss_budget, lost_prefix; ++ ++ pcount = tcp_skb_pcount(skb); ++ ++ /* How much data was in flight before this skb? */ ++ inflight_prev = rs->tx_in_flight - pcount; ++ if (WARN_ONCE(inflight_prev < 0, ++ "tx_in_flight: %u pcount: %u reneg: %u", ++ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) ++ return ~0U; ++ ++ /* How much inflight data was marked lost before this skb? */ ++ lost_prev = rs->lost - pcount; ++ if (WARN_ON_ONCE(lost_prev < 0)) ++ return ~0U; ++ ++ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ ++ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; ++ loss_budget >>= BBR_SCALE; ++ if (lost_prev >= loss_budget) { ++ lost_prefix = 0; /* previous losses crossed loss_thresh */ ++ } else { ++ lost_prefix = loss_budget - lost_prev; ++ lost_prefix <<= BBR_SCALE; ++ divisor = BBR_UNIT - loss_thresh; ++ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ ++ return ~0U; ++ do_div(lost_prefix, divisor); ++ } ++ ++ inflight_hi = inflight_prev + lost_prefix; ++ return inflight_hi; ++} ++ ++/* If loss/ECN rates during probing indicated we may have overfilled a ++ * buffer, return an operating point that tries to leave unutilized headroom in ++ * the path for other flows, for fairness convergence and lower RTTs and loss. ++ */ ++static u32 bbr2_inflight_with_headroom(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 headroom, headroom_fraction; ++ ++ if (bbr->inflight_hi == ~0U) ++ return ~0U; ++ ++ headroom_fraction = bbr->params.inflight_headroom; ++ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; ++ headroom = max(headroom, 1U); ++ return max_t(s32, bbr->inflight_hi - headroom, ++ bbr->params.cwnd_min_target); ++} ++ ++/* Bound cwnd to a sensible level, based on our current probing state ++ * machine phase and model of a good inflight level (inflight_lo, inflight_hi). ++ */ ++static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 cap; ++ ++ /* tcp_rcv_synsent_state_process() currently calls tcp_ack() ++ * and thus cong_control() without first initializing us(!). ++ */ ++ if (!bbr->initialized) ++ return; ++ ++ cap = ~0U; ++ if (bbr->mode == BBR_PROBE_BW && ++ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { ++ /* Probe to see if more packets fit in the path. */ ++ cap = bbr->inflight_hi; ++ } else { ++ if (bbr->mode == BBR_PROBE_RTT || ++ (bbr->mode == BBR_PROBE_BW && ++ bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) ++ cap = bbr2_inflight_with_headroom(sk); ++ } ++ /* Adapt to any loss/ECN since our last bw probe. */ ++ cap = min(cap, bbr->inflight_lo); ++ ++ cap = max_t(u32, cap, bbr->params.cwnd_min_target); ++ tp->snd_cwnd = min(cap, tp->snd_cwnd); ++} ++ ++/* Estimate a short-term lower bound on the capacity available now, based ++ * on measurements of the current delivery process and recent history. When we ++ * are seeing loss/ECN at times when we are not probing bw, then conservatively ++ * move toward flow balance by multiplicatively cutting our short-term ++ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a ++ * multiplicative decrease in order to converge to a lower capacity in time ++ * logarithmic in the magnitude of the decrease. ++ * ++ * However, we do not cut our short-term estimates lower than the current rate ++ * and volume of delivered data from this round trip, since from the current ++ * delivery process we can estimate the measured capacity available now. ++ * ++ * Anything faster than that approach would knowingly risk high loss, which can ++ * cause low bw for Reno/CUBIC and high loss recovery latency for ++ * request/response flows using any congestion control. ++ */ ++static void bbr2_adapt_lower_bounds(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 ecn_cut, ecn_inflight_lo, beta; ++ ++ /* We only use lower-bound estimates when not probing bw. ++ * When probing we need to push inflight higher to probe bw. ++ */ ++ if (bbr2_is_probing_bandwidth(sk)) ++ return; ++ ++ /* ECN response. */ ++ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { ++ /* Reduce inflight to (1 - alpha*ecn_factor). */ ++ ecn_cut = (BBR_UNIT - ++ ((bbr->ecn_alpha * bbr->params.ecn_factor) >> ++ BBR_SCALE)); ++ if (bbr->inflight_lo == ~0U) ++ bbr->inflight_lo = tp->snd_cwnd; ++ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; ++ } else { ++ ecn_inflight_lo = ~0U; ++ } ++ ++ /* Loss response. */ ++ if (bbr->loss_in_round) { ++ /* Reduce bw and inflight to (1 - beta). */ ++ if (bbr->bw_lo == ~0U) ++ bbr->bw_lo = bbr_max_bw(sk); ++ if (bbr->inflight_lo == ~0U) ++ bbr->inflight_lo = tp->snd_cwnd; ++ beta = bbr->params.beta; ++ bbr->bw_lo = ++ max_t(u32, bbr->bw_latest, ++ (u64)bbr->bw_lo * ++ (BBR_UNIT - beta) >> BBR_SCALE); ++ bbr->inflight_lo = ++ max_t(u32, bbr->inflight_latest, ++ (u64)bbr->inflight_lo * ++ (BBR_UNIT - beta) >> BBR_SCALE); ++ } ++ ++ /* Adjust to the lower of the levels implied by loss or ECN. */ ++ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); ++} ++ ++/* Reset any short-term lower-bound adaptation to congestion, so that we can ++ * push our inflight up. ++ */ ++static void bbr2_reset_lower_bounds(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->bw_lo = ~0U; ++ bbr->inflight_lo = ~0U; ++} ++ ++/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state ++ * machine phase where we adapt our lower bound based on congestion signals. ++ */ ++static void bbr2_reset_congestion_signals(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->loss_in_round = 0; ++ bbr->ecn_in_round = 0; ++ bbr->loss_in_cycle = 0; ++ bbr->ecn_in_cycle = 0; ++ bbr->bw_latest = 0; ++ bbr->inflight_latest = 0; ++} ++ ++/* Update (most of) our congestion signals: track the recent rate and volume of ++ * delivered data, presence of loss, and EWMA degree of ECN marking. ++ */ ++static void bbr2_update_congestion_signals( ++ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw; ++ ++ bbr->loss_round_start = 0; ++ if (rs->interval_us <= 0 || !rs->acked_sacked) ++ return; /* Not a valid observation */ ++ bw = ctx->sample_bw; ++ ++ if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) ++ bbr2_take_bw_hi_sample(sk, bw); ++ ++ bbr->loss_in_round |= (rs->losses > 0); ++ ++ /* Update rate and volume of delivered data from latest round trip: */ ++ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); ++ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); ++ ++ if (before(rs->prior_delivered, bbr->loss_round_delivered)) ++ return; /* skip the per-round-trip updates */ ++ /* Now do per-round-trip updates. */ ++ bbr->loss_round_delivered = tp->delivered; /* mark round trip */ ++ bbr->loss_round_start = 1; ++ bbr2_adapt_lower_bounds(sk); ++ ++ /* Update windowed "latest" (single-round-trip) filters. */ ++ bbr->loss_in_round = 0; ++ bbr->ecn_in_round = 0; ++ bbr->bw_latest = ctx->sample_bw; ++ bbr->inflight_latest = rs->delivered; ++} ++ ++/* Bandwidth probing can cause loss. To help coexistence with loss-based ++ * congestion control we spread out our probing in a Reno-conscious way. Due to ++ * the shape of the Reno sawtooth, the time required between loss epochs for an ++ * idealized Reno flow is a number of round trips that is the BDP of that ++ * flow. We count packet-timed round trips directly, since measured RTT can ++ * vary widely, and Reno is driven by packet-timed round trips. ++ */ ++static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 inflight, rounds, reno_gain, reno_rounds; ++ ++ /* Random loss can shave some small percentage off of our inflight ++ * in each round. To survive this, flows need robust periodic probes. ++ */ ++ rounds = bbr->params.bw_probe_max_rounds; ++ ++ reno_gain = bbr->params.bw_probe_reno_gain; ++ if (reno_gain) { ++ inflight = bbr2_target_inflight(sk); ++ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; ++ rounds = min(rounds, reno_rounds); ++ } ++ return bbr->rounds_since_probe >= rounds; ++} ++ ++/* How long do we want to wait before probing for bandwidth (and risking ++ * loss)? We randomize the wait, for better mixing and fairness convergence. ++ * ++ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. ++ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, ++ * (eg 4K video to a broadband user): ++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets ++ * ++ * We bound the BBR-native inter-bw-probe wall clock time to be: ++ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time ++ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must ++ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs ++ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable ++ * amount of time to discover unutilized bw on human-scale interactive ++ * time-scales (e.g. perhaps traffic from a web page download that we ++ * were competing with is now complete). ++ */ ++static void bbr2_pick_probe_wait(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ /* Decide the random round-trip bound for wait until probe: */ ++ bbr->rounds_since_probe = ++ prandom_u32_max(bbr->params.bw_probe_rand_rounds); ++ /* Decide the random wall clock bound for wait until probe: */ ++ bbr->probe_wait_us = bbr->params.bw_probe_base_us + ++ prandom_u32_max(bbr->params.bw_probe_rand_us); ++} ++ ++static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->cycle_idx = cycle_idx; ++ /* New phase, so need to update cwnd and pacing rate. */ ++ bbr->try_fast_path = 0; ++} ++ ++/* Send at estimated bw to fill the pipe, but not queue. We need this phase ++ * before PROBE_UP, because as soon as we send faster than the available bw ++ * we will start building a queue, and if the buffer is shallow we can cause ++ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and ++ * inflight_hi estimates will underestimate. ++ */ ++static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr2_reset_lower_bounds(sk); ++ if (bbr->inflight_hi != ~0U) ++ bbr->inflight_hi += bbr->params.refill_add_inc; ++ bbr->bw_probe_up_rounds = bw_probe_up_rounds; ++ bbr->bw_probe_up_acks = 0; ++ bbr->stopped_risky_probe = 0; ++ bbr->ack_phase = BBR_ACKS_REFILLING; ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); ++} ++ ++/* Now probe max deliverable data rate and volume. */ ++static void bbr2_start_bw_probe_up(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->ack_phase = BBR_ACKS_PROBE_STARTING; ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr->cycle_mstamp = tp->tcp_mstamp; ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); ++ bbr2_raise_inflight_hi_slope(sk); ++} ++ ++/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall ++ * clock time at which to probe beyond an inflight that we think to be ++ * safe. This will knowingly risk packet loss, so we want to do this rarely, to ++ * keep packet loss rates low. Also start a round-trip counter, to probe faster ++ * if we estimate a Reno flow at our BDP would probe faster. ++ */ ++static void bbr2_start_bw_probe_down(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr2_reset_congestion_signals(sk); ++ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ ++ bbr2_pick_probe_wait(sk); ++ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ ++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); ++} ++ ++/* Cruise: maintain what we estimate to be a neutral, conservative ++ * operating point, without attempting to probe up for bandwidth or down for ++ * RTT, and only reducing inflight in response to loss/ECN signals. ++ */ ++static void bbr2_start_bw_probe_cruise(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->inflight_lo != ~0U) ++ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); ++ ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); ++} ++ ++/* Loss and/or ECN rate is too high while probing. ++ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. ++ */ ++static void bbr2_handle_inflight_too_high(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ const u32 beta = bbr->params.beta; ++ ++ bbr->prev_probe_too_high = 1; ++ bbr->bw_probe_samples = 0; /* only react once per probe */ ++ bbr->debug.event = 'L'; /* Loss/ECN too high */ ++ /* If we are app-limited then we are not robustly ++ * probing the max volume of inflight data we think ++ * might be safe (analogous to how app-limited bw ++ * samples are not known to be robustly probing bw). ++ */ ++ if (!rs->is_app_limited) ++ bbr->inflight_hi = max_t(u32, rs->tx_in_flight, ++ (u64)bbr2_target_inflight(sk) * ++ (BBR_UNIT - beta) >> BBR_SCALE); ++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) ++ bbr2_start_bw_probe_down(sk); ++} ++ ++/* If we're seeing bw and loss samples reflecting our bw probing, adapt ++ * using the signals we see. If loss or ECN mark rate gets too high, then adapt ++ * inflight_hi downward. If we're able to push inflight higher without such ++ * signals, push higher: adapt inflight_hi upward. ++ */ ++static bool bbr2_adapt_upper_bounds(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ /* Track when we'll see bw/loss samples resulting from our bw probes. */ ++ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) ++ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; ++ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { ++ /* End of samples from bw probing phase. */ ++ bbr->bw_probe_samples = 0; ++ bbr->ack_phase = BBR_ACKS_INIT; ++ /* At this point in the cycle, our current bw sample is also ++ * our best recent chance at finding the highest available bw ++ * for this flow. So now is the best time to forget the bw ++ * samples from the previous cycle, by advancing the window. ++ */ ++ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) ++ bbr2_advance_bw_hi_filter(sk); ++ /* If we had an inflight_hi, then probed and pushed inflight all ++ * the way up to hit that inflight_hi without seeing any ++ * high loss/ECN in all the resulting ACKs from that probing, ++ * then probe up again, this time letting inflight persist at ++ * inflight_hi for a round trip, then accelerating beyond. ++ */ ++ if (bbr->mode == BBR_PROBE_BW && ++ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { ++ bbr->debug.event = 'R'; /* reprobe */ ++ bbr2_start_bw_probe_refill(sk, 0); ++ return true; /* yes, decided state transition */ ++ } ++ } ++ ++ if (bbr2_is_inflight_too_high(sk, rs)) { ++ if (bbr->bw_probe_samples) /* sample is from bw probing? */ ++ bbr2_handle_inflight_too_high(sk, rs); ++ } else { ++ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ ++ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ ++ return false; ++ ++ /* To be resilient to random loss, we must raise inflight_hi ++ * if we observe in any phase that a higher level is safe. ++ */ ++ if (rs->tx_in_flight > bbr->inflight_hi) { ++ bbr->inflight_hi = rs->tx_in_flight; ++ bbr->debug.event = 'U'; /* raise up inflight_hi */ ++ } ++ ++ if (bbr->mode == BBR_PROBE_BW && ++ bbr->cycle_idx == BBR_BW_PROBE_UP) ++ bbr2_probe_inflight_hi_upward(sk, rs); ++ } ++ ++ return false; ++} ++ ++/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ ++static bool bbr2_check_time_to_probe_bw(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 n; ++ ++ /* If we seem to be at an operating point where we are not seeing loss ++ * but we are seeing ECN marks, then when the ECN marks cease we reprobe ++ * quickly (in case a burst of cross-traffic has ceased and freed up bw, ++ * or in case we are sharing with multiplicatively probing traffic). ++ */ ++ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && ++ bbr->ecn_in_cycle && !bbr->loss_in_cycle && ++ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { ++ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ ++ /* Calculate n so that when bbr2_raise_inflight_hi_slope() ++ * computes growth_this_round as 2^n it will be roughly the ++ * desired volume of data (inflight_hi*ecn_reprobe_gain). ++ */ ++ n = ilog2((((u64)bbr->inflight_hi * ++ bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); ++ bbr2_start_bw_probe_refill(sk, n); ++ return true; ++ } ++ ++ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || ++ bbr2_is_reno_coexistence_probe_time(sk)) { ++ bbr2_start_bw_probe_refill(sk, 0); ++ return true; ++ } ++ return false; ++} ++ ++/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ ++static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ bool is_under_bdp, is_long_enough; ++ ++ /* Always need to pull inflight down to leave headroom in queue. */ ++ if (inflight > bbr2_inflight_with_headroom(sk)) ++ return false; ++ ++ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); ++ if (bbr->params.drain_to_target) ++ return is_under_bdp; ++ ++ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); ++ return is_under_bdp || is_long_enough; ++} ++ ++/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ ++static void bbr2_update_cycle_phase(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ bool is_risky = false, is_queuing = false; ++ u32 inflight, bw; ++ ++ if (!bbr_full_bw_reached(sk)) ++ return; ++ ++ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ ++ if (bbr2_adapt_upper_bounds(sk, rs)) ++ return; /* already decided state transition */ ++ ++ if (bbr->mode != BBR_PROBE_BW) ++ return; ++ ++ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); ++ bw = bbr_max_bw(sk); ++ ++ switch (bbr->cycle_idx) { ++ /* First we spend most of our time cruising with a pacing_gain of 1.0, ++ * which paces at the estimated bw, to try to fully use the pipe ++ * without building queue. If we encounter loss/ECN marks, we adapt ++ * by slowing down. ++ */ ++ case BBR_BW_PROBE_CRUISE: ++ if (bbr2_check_time_to_probe_bw(sk)) ++ return; /* already decided state transition */ ++ break; ++ ++ /* After cruising, when it's time to probe, we first "refill": we send ++ * at the estimated bw to fill the pipe, before probing higher and ++ * knowingly risking overflowing the bottleneck buffer (causing loss). ++ */ ++ case BBR_BW_PROBE_REFILL: ++ if (bbr->round_start) { ++ /* After one full round trip of sending in REFILL, we ++ * start to see bw samples reflecting our REFILL, which ++ * may be putting too much data in flight. ++ */ ++ bbr->bw_probe_samples = 1; ++ bbr2_start_bw_probe_up(sk); ++ } ++ break; ++ ++ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to ++ * probe for bw. If we have not seen loss/ECN, we try to raise inflight ++ * to at least pacing_gain*BDP; note that this may take more than ++ * min_rtt if min_rtt is small (e.g. on a LAN). ++ * ++ * We terminate PROBE_UP bandwidth probing upon any of the following: ++ * ++ * (1) We've pushed inflight up to hit the inflight_hi target set in the ++ * most recent previous bw probe phase. Thus we want to start ++ * draining the queue immediately because it's very likely the most ++ * recently sent packets will fill the queue and cause drops. ++ * (checked here) ++ * (2) We have probed for at least 1*min_rtt_us, and the ++ * estimated queue is high enough (inflight > 1.25 * estimated_bdp). ++ * (checked here) ++ * (3) Loss filter says loss rate is "too high". ++ * (checked in bbr_is_inflight_too_high()) ++ * (4) ECN filter says ECN mark rate is "too high". ++ * (checked in bbr_is_inflight_too_high()) ++ */ ++ case BBR_BW_PROBE_UP: ++ if (bbr->prev_probe_too_high && ++ inflight >= bbr->inflight_hi) { ++ bbr->stopped_risky_probe = 1; ++ is_risky = true; ++ bbr->debug.event = 'D'; /* D for danger */ ++ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && ++ inflight >= ++ bbr_inflight(sk, bw, ++ bbr->params.bw_probe_pif_gain)) { ++ is_queuing = true; ++ bbr->debug.event = 'Q'; /* building Queue */ ++ } ++ if (is_risky || is_queuing) { ++ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ ++ bbr2_start_bw_probe_down(sk); /* restart w/ down */ ++ } ++ break; ++ ++ /* After probing in PROBE_UP, we have usually accumulated some data in ++ * the bottleneck buffer (if bw probing didn't find more bw). We next ++ * enter PROBE_DOWN to try to drain any excess data from the queue. To ++ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until ++ * our inflight is less then that target cruising point, which is the ++ * minimum of (a) the amount needed to leave headroom, and (b) the ++ * estimated BDP. Once inflight falls to match the target, we estimate ++ * the queue is drained; persisting would underutilize the pipe. ++ */ ++ case BBR_BW_PROBE_DOWN: ++ if (bbr2_check_time_to_probe_bw(sk)) ++ return; /* already decided state transition */ ++ if (bbr2_check_time_to_cruise(sk, inflight, bw)) ++ bbr2_start_bw_probe_cruise(sk); ++ break; ++ ++ default: ++ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); ++ } ++} ++ ++/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ ++static void bbr2_exit_probe_rtt(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr2_reset_lower_bounds(sk); ++ if (bbr_full_bw_reached(sk)) { ++ bbr->mode = BBR_PROBE_BW; ++ /* Raising inflight after PROBE_RTT may cause loss, so reset ++ * the PROBE_BW clock and schedule the next bandwidth probe for ++ * a friendly and randomized future point in time. ++ */ ++ bbr2_start_bw_probe_down(sk); ++ /* Since we are exiting PROBE_RTT, we know inflight is ++ * below our estimated BDP, so it is reasonable to cruise. ++ */ ++ bbr2_start_bw_probe_cruise(sk); ++ } else { ++ bbr->mode = BBR_STARTUP; ++ } ++} ++ ++/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until ++ * the end of the round in recovery to get a good estimate of how many packets ++ * have been lost, and how many we need to drain with a low pacing rate. ++ */ ++static void bbr2_check_loss_too_high_in_startup(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr_full_bw_reached(sk)) ++ return; ++ ++ /* For STARTUP exit, check the loss rate at the end of each round trip ++ * of Recovery episodes in STARTUP. We check the loss rate at the end ++ * of the round trip to filter out noisy/low loss and have a better ++ * sense of inflight (extent of loss), so we can drain more accurately. ++ */ ++ if (rs->losses && bbr->loss_events_in_round < 0xf) ++ bbr->loss_events_in_round++; /* update saturating counter */ ++ if (bbr->params.full_loss_cnt && bbr->loss_round_start && ++ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && ++ bbr->loss_events_in_round >= bbr->params.full_loss_cnt && ++ bbr2_is_inflight_too_high(sk, rs)) { ++ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ ++ bbr2_handle_queue_too_high_in_startup(sk); ++ return; ++ } ++ if (bbr->loss_round_start) ++ bbr->loss_events_in_round = 0; ++} ++ ++/* If we are done draining, advance into steady state operation in PROBE_BW. */ ++static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, ++ struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr_check_drain(sk, rs, ctx)) { ++ bbr->mode = BBR_PROBE_BW; ++ bbr2_start_bw_probe_down(sk); ++ } ++} ++ ++static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, ++ struct bbr_context *ctx) ++{ ++ bbr2_update_congestion_signals(sk, rs, ctx); ++ bbr_update_ack_aggregation(sk, rs); ++ bbr2_check_loss_too_high_in_startup(sk, rs); ++ bbr_check_full_bw_reached(sk, rs); ++ bbr2_check_drain(sk, rs, ctx); ++ bbr2_update_cycle_phase(sk, rs); ++ bbr_update_min_rtt(sk, rs); ++} ++ ++/* Fast path for app-limited case. ++ * ++ * On each ack, we execute bbr state machine, which primarily consists of: ++ * 1) update model based on new rate sample, and ++ * 2) update control based on updated model or state change. ++ * ++ * There are certain workload/scenarios, e.g. app-limited case, where ++ * either we can skip updating model or we can skip update of both model ++ * as well as control. This provides signifcant softirq cpu savings for ++ * processing incoming acks. ++ * ++ * In case of app-limited, if there is no congestion (loss/ecn) and ++ * if observed bw sample is less than current estimated bw, then we can ++ * skip some of the computation in bbr state processing: ++ * ++ * - if there is no rtt/mode/phase change: In this case, since all the ++ * parameters of the network model are constant, we can skip model ++ * as well control update. ++ * ++ * - else we can skip rest of the model update. But we still need to ++ * update the control to account for the new rtt/mode/phase. ++ * ++ * Returns whether we can take fast path or not. ++ */ ++static bool bbr2_fast_path(struct sock *sk, bool *update_model, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 prev_min_rtt_us, prev_mode; ++ ++ if (bbr->params.fast_path && bbr->try_fast_path && ++ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && ++ !bbr->loss_in_round && !bbr->ecn_in_round) { ++ prev_mode = bbr->mode; ++ prev_min_rtt_us = bbr->min_rtt_us; ++ bbr2_check_drain(sk, rs, ctx); ++ bbr2_update_cycle_phase(sk, rs); ++ bbr_update_min_rtt(sk, rs); ++ ++ if (bbr->mode == prev_mode && ++ bbr->min_rtt_us == prev_min_rtt_us && ++ bbr->try_fast_path) ++ return true; ++ ++ /* Skip model update, but control still needs to be updated */ ++ *update_model = false; ++ } ++ return false; ++} ++ ++static void bbr2_main(struct sock *sk, const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ struct bbr_context ctx = { 0 }; ++ bool update_model = true; ++ u32 bw; ++ int ce_ratio = -1; ++ ++ bbr->debug.event = '.'; /* init to default NOP (no event yet) */ ++ ++ bbr_update_round_start(sk, rs, &ctx); ++ if (bbr->round_start) { ++ bbr->rounds_since_probe = ++ min_t(s32, bbr->rounds_since_probe + 1, 0xFF); ++ ce_ratio = bbr2_update_ecn_alpha(sk); ++ tcp_plb_update_state(sk, &bbr->plb, ce_ratio); ++ tcp_plb_check_rehash(sk, &bbr->plb); ++ } ++ ++ bbr->ecn_in_round |= rs->is_ece; ++ bbr_calculate_bw_sample(sk, rs, &ctx); ++ ++ if (bbr2_fast_path(sk, &update_model, rs, &ctx)) ++ goto out; ++ ++ if (update_model) ++ bbr2_update_model(sk, rs, &ctx); ++ ++ bbr_update_gains(sk); ++ bw = bbr_bw(sk); ++ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); ++ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, ++ tp->snd_cwnd, &ctx); ++ bbr2_bound_cwnd_for_inflight_model(sk); ++ ++out: ++ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; ++ bbr->loss_in_cycle |= rs->lost > 0; ++ bbr->ecn_in_cycle |= rs->delivered_ce > 0; ++ ++ bbr_debug(sk, rs->acked_sacked, rs, &ctx); ++} ++ ++/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared ++ * down here, so that the algorithm functions that use the parameters must use ++ * the per-socket parameters; if they accidentally use the global version ++ * then there will be a compile error. ++ * TODO(ncardwell): move all per-socket parameters down to this section. ++ */ ++ ++/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. ++ * No loss response when 0. Max allwed value is 255. ++ */ ++static u32 bbr_beta = BBR_UNIT * 30 / 100; ++ ++/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. ++ * Max allowed value is 255. ++ */ ++static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ ++ ++/* The initial value for the ecn_alpha state variable. Default and max ++ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly ++ * to congestion if the bottleneck is congested when the flow starts up. ++ */ ++static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ ++ ++/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. ++ * No ECN based bounding when 0. Max allwed value is 255. ++ */ ++static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ ++ ++/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. ++ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. ++ */ ++static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ ++ ++/* Max RTT (in usec) at which to use sender-side ECN logic. ++ * Disabled when 0 (ECN allowed at any RTT). ++ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. ++ */ ++static u32 bbr_ecn_max_rtt_us = 5000; ++ ++/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN ++ * clears then use a multiplicative increase to quickly reprobe bw by ++ * starting inflight probing at the given multiple of inflight_hi. ++ * Default for this experimental knob is 0 (disabled). ++ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. ++ */ ++static u32 bbr_ecn_reprobe_gain; ++ ++/* Estimate bw probing has gone too far if loss rate exceeds this level. */ ++static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ ++ ++/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, ++ * and loss rate is higher than bbr_loss_thresh. ++ * Disabled if 0. Max allowed value is 15 (0xF). ++ */ ++static u32 bbr_full_loss_cnt = 8; ++ ++/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh ++ * meets this count. Max allowed value is 3. ++ */ ++static u32 bbr_full_ecn_cnt = 2; ++ ++/* Fraction of unutilized headroom to try to leave in path upon high loss. */ ++static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; ++ ++/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. ++ * Default is 1.25x, as in BBR v1. Max allowed is 511. ++ */ ++static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; ++ ++/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. ++ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. ++ * Max allowed is 511. ++ */ ++static u32 bbr_bw_probe_reno_gain = BBR_UNIT; ++ ++/* Max number of packet-timed rounds to wait before probing for bandwidth. If ++ * we want to tolerate 1% random loss per round, and not have this cut our ++ * inflight too much, we must probe for bw periodically on roughly this scale. ++ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. ++ * We aim to be fair with Reno/CUBIC up to a BDP of at least: ++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets ++ */ ++static u32 bbr_bw_probe_max_rounds = 63; ++ ++/* Max amount of randomness to inject in round counting for Reno-coexistence. ++ * Max value is 15. ++ */ ++static u32 bbr_bw_probe_rand_rounds = 2; ++ ++/* Use BBR-native probe time scale starting at this many usec. ++ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: ++ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs ++ */ ++static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ ++ ++/* Use BBR-native probes spread over this many usec: */ ++static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ ++ ++/* Undo the model changes made in loss recovery if recovery was spurious? */ ++static bool bbr_undo = true; ++ ++/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ ++static bool bbr_fast_path = true; /* default: enabled */ ++ ++/* Use fast ack mode ? */ ++static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ ++ ++/* How much to additively increase inflight_hi when entering REFILL? */ ++static u32 bbr_refill_add_inc; /* default: disabled */ ++ ++module_param_named(beta, bbr_beta, uint, 0644); ++module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); ++module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); ++module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); ++module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); ++module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); ++module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); ++module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); ++module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664); ++module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664); ++module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664); ++module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664); ++module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664); ++module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664); ++module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664); ++module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664); ++module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664); ++module_param_named(undo, bbr_undo, bool, 0664); ++module_param_named(fast_path, bbr_fast_path, bool, 0664); ++module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664); ++module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664); ++ ++static void bbr2_init(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ const struct net *net = sock_net(sk); ++ ++ bbr_init(sk); /* run shared init code for v1 and v2 */ ++ ++ /* BBR v2 parameters: */ ++ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); ++ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); ++ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); ++ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); ++ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); ++ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); ++ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); ++ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); ++ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); ++ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); ++ bbr->params.inflight_headroom = ++ min_t(u32, 0xFFU, bbr_inflight_headroom); ++ bbr->params.bw_probe_pif_gain = ++ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); ++ bbr->params.bw_probe_reno_gain = ++ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); ++ bbr->params.bw_probe_max_rounds = ++ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); ++ bbr->params.bw_probe_rand_rounds = ++ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); ++ bbr->params.bw_probe_base_us = ++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); ++ bbr->params.bw_probe_rand_us = ++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); ++ bbr->params.undo = bbr_undo; ++ bbr->params.fast_path = bbr_fast_path ? 1 : 0; ++ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); ++ ++ /* BBR v2 state: */ ++ bbr->initialized = 1; ++ /* Start sampling ECN mark rate after first full flight is ACKed: */ ++ bbr->loss_round_delivered = tp->delivered + 1; ++ bbr->loss_round_start = 0; ++ bbr->undo_bw_lo = 0; ++ bbr->undo_inflight_lo = 0; ++ bbr->undo_inflight_hi = 0; ++ bbr->loss_events_in_round = 0; ++ bbr->startup_ecn_rounds = 0; ++ bbr2_reset_congestion_signals(sk); ++ bbr->bw_lo = ~0U; ++ bbr->bw_hi[0] = 0; ++ bbr->bw_hi[1] = 0; ++ bbr->inflight_lo = ~0U; ++ bbr->inflight_hi = ~0U; ++ bbr->bw_probe_up_cnt = ~0U; ++ bbr->bw_probe_up_acks = 0; ++ bbr->bw_probe_up_rounds = 0; ++ bbr->probe_wait_us = 0; ++ bbr->stopped_risky_probe = 0; ++ bbr->ack_phase = BBR_ACKS_INIT; ++ bbr->rounds_since_probe = 0; ++ bbr->bw_probe_samples = 0; ++ bbr->prev_probe_too_high = 0; ++ bbr->ecn_eligible = 0; ++ bbr->ecn_alpha = bbr->params.ecn_alpha_init; ++ bbr->alpha_last_delivered = 0; ++ bbr->alpha_last_delivered_ce = 0; ++ ++ bbr->plb.enabled = 0; ++ bbr->plb.consec_cong_rounds = 0; ++ bbr->plb.pause_until = 0; ++ if ((tp->ecn_flags & TCP_ECN_OK) && ++ net->ipv4.sysctl_tcp_plb_enabled) ++ bbr->plb.enabled = 1; ++ ++ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); ++ ++ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable) ++ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; ++} ++ ++/* Core TCP stack informs us that the given skb was just marked lost. */ ++static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ struct tcp_skb_cb *scb = TCP_SKB_CB(skb); ++ struct rate_sample rs; ++ ++ /* Capture "current" data over the full round trip of loss, ++ * to have a better chance to see the full capacity of the path. ++ */ ++ if (!bbr->loss_in_round) /* first loss in this round trip? */ ++ bbr->loss_round_delivered = tp->delivered; /* set round trip */ ++ bbr->loss_in_round = 1; ++ bbr->loss_in_cycle = 1; ++ ++ if (!bbr->bw_probe_samples) ++ return; /* not an skb sent while probing for bandwidth */ ++ if (unlikely(!scb->tx.delivered_mstamp)) ++ return; /* skb was SACKed, reneged, marked lost; ignore it */ ++ /* We are probing for bandwidth. Construct a rate sample that ++ * estimates what happened in the flight leading up to this lost skb, ++ * then see if the loss rate went too high, and if so at which packet. ++ */ ++ memset(&rs, 0, sizeof(rs)); ++ rs.tx_in_flight = scb->tx.in_flight; ++ rs.lost = tp->lost - scb->tx.lost; ++ rs.is_app_limited = scb->tx.is_app_limited; ++ if (bbr2_is_inflight_too_high(sk, &rs)) { ++ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); ++ bbr2_handle_inflight_too_high(sk, &rs); ++ } ++} ++ ++/* Revert short-term model if current loss recovery event was spurious. */ ++static u32 bbr2_undo_cwnd(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->debug.undo = 1; ++ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ ++ bbr->full_bw_cnt = 0; ++ bbr->loss_in_round = 0; ++ ++ if (!bbr->params.undo) ++ return tp->snd_cwnd; ++ ++ /* Revert to cwnd and other state saved before loss episode. */ ++ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); ++ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); ++ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); ++ return bbr->prior_cwnd; ++} ++ ++/* Entering loss recovery, so save state for when we undo recovery. */ ++static u32 bbr2_ssthresh(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr_save_cwnd(sk); ++ /* For undo, save state that adapts based on loss signal. */ ++ bbr->undo_bw_lo = bbr->bw_lo; ++ bbr->undo_inflight_lo = bbr->inflight_lo; ++ bbr->undo_inflight_hi = bbr->inflight_hi; ++ return tcp_sk(sk)->snd_ssthresh; ++} ++ ++static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) ++{ ++ switch (bbr->mode) { ++ case BBR_STARTUP: ++ return BBR2_PHASE_STARTUP; ++ case BBR_DRAIN: ++ return BBR2_PHASE_DRAIN; ++ case BBR_PROBE_BW: ++ break; ++ case BBR_PROBE_RTT: ++ return BBR2_PHASE_PROBE_RTT; ++ default: ++ return BBR2_PHASE_INVALID; ++ } ++ switch (bbr->cycle_idx) { ++ case BBR_BW_PROBE_UP: ++ return BBR2_PHASE_PROBE_BW_UP; ++ case BBR_BW_PROBE_DOWN: ++ return BBR2_PHASE_PROBE_BW_DOWN; ++ case BBR_BW_PROBE_CRUISE: ++ return BBR2_PHASE_PROBE_BW_CRUISE; ++ case BBR_BW_PROBE_REFILL: ++ return BBR2_PHASE_PROBE_BW_REFILL; ++ default: ++ return BBR2_PHASE_INVALID; ++ } ++} ++ ++static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, ++ union tcp_cc_info *info) ++{ ++ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || ++ ext & (1 << (INET_DIAG_VEGASINFO - 1))) { ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); ++ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); ++ u64 bw_lo = bbr->bw_lo == ~0U ? ++ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); ++ ++ memset(&info->bbr2, 0, sizeof(info->bbr2)); ++ info->bbr2.bbr_bw_lsb = (u32)bw; ++ info->bbr2.bbr_bw_msb = (u32)(bw >> 32); ++ info->bbr2.bbr_min_rtt = bbr->min_rtt_us; ++ info->bbr2.bbr_pacing_gain = bbr->pacing_gain; ++ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; ++ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; ++ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); ++ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; ++ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); ++ info->bbr2.bbr_mode = bbr->mode; ++ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); ++ info->bbr2.bbr_version = (__u8)2; ++ info->bbr2.bbr_inflight_lo = bbr->inflight_lo; ++ info->bbr2.bbr_inflight_hi = bbr->inflight_hi; ++ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); ++ *attr = INET_DIAG_BBRINFO; ++ return sizeof(info->bbr2); ++ } ++ return 0; ++} ++ ++static void bbr2_set_state(struct sock *sk, u8 new_state) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (new_state == TCP_CA_Loss) { ++ struct rate_sample rs = { .losses = 1 }; ++ struct bbr_context ctx = { 0 }; ++ ++ tcp_plb_update_state_upon_rto(sk, &bbr->plb); ++ bbr->prev_ca_state = TCP_CA_Loss; ++ bbr->full_bw = 0; ++ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { ++ /* bbr_adapt_lower_bounds() needs cwnd before ++ * we suffered an RTO, to update inflight_lo: ++ */ ++ bbr->inflight_lo = ++ max(tp->snd_cwnd, bbr->prior_cwnd); ++ } ++ bbr_debug(sk, 0, &rs, &ctx); ++ } else if (bbr->prev_ca_state == TCP_CA_Loss && ++ new_state != TCP_CA_Loss) { ++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); ++ bbr->try_fast_path = 0; /* bound cwnd using latest model */ ++ } ++} ++ ++static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { ++ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, ++ .name = "bbr2", ++ .owner = THIS_MODULE, ++ .init = bbr2_init, ++ .cong_control = bbr2_main, ++ .sndbuf_expand = bbr_sndbuf_expand, ++ .skb_marked_lost = bbr2_skb_marked_lost, ++ .undo_cwnd = bbr2_undo_cwnd, ++ .cwnd_event = bbr_cwnd_event, ++ .ssthresh = bbr2_ssthresh, ++ .tso_segs = bbr_tso_segs, ++ .get_info = bbr2_get_info, ++ .set_state = bbr2_set_state, ++}; ++ ++static int __init bbr_register(void) ++{ ++ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); ++ return tcp_register_congestion_control(&tcp_bbr2_cong_ops); ++} ++ ++static void __exit bbr_unregister(void) ++{ ++ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); ++} ++ ++module_init(bbr_register); ++module_exit(bbr_unregister); ++ ++MODULE_AUTHOR("Van Jacobson "); ++MODULE_AUTHOR("Neal Cardwell "); ++MODULE_AUTHOR("Yuchung Cheng "); ++MODULE_AUTHOR("Soheil Hassas Yeganeh "); ++MODULE_AUTHOR("Priyaranjan Jha "); ++MODULE_AUTHOR("Yousuk Seung "); ++MODULE_AUTHOR("Kevin Yang "); ++MODULE_AUTHOR("Arjun Roy "); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); +diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c +index d3cae40749e8..0f268f2ff2e9 100644 +--- a/net/ipv4/tcp_cong.c ++++ b/net/ipv4/tcp_cong.c +@@ -189,6 +189,7 @@ void tcp_init_congestion_control(struct sock *sk) + struct inet_connection_sock *icsk = inet_csk(sk); + + tcp_sk(sk)->prior_ssthresh = 0; ++ tcp_sk(sk)->fast_ack_mode = 0; + if (icsk->icsk_ca_ops->init) + icsk->icsk_ca_ops->init(sk); + if (tcp_ca_needs_ecn(sk)) +diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c +index bc2ea12221f9..fa5c3e5d9c85 100644 +--- a/net/ipv4/tcp_input.c ++++ b/net/ipv4/tcp_input.c +@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) + tcp_enter_quickack_mode(sk, 2); + break; + case INET_ECN_CE: +- if (tcp_ca_needs_ecn(sk)) ++ if (tcp_ca_wants_ce_events(sk)) + tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); + + if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { +@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) + tp->ecn_flags |= TCP_ECN_SEEN; + break; + default: +- if (tcp_ca_needs_ecn(sk)) ++ if (tcp_ca_wants_ce_events(sk)) + tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); + tp->ecn_flags |= TCP_ECN_SEEN; + break; +@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) + */ + static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) + { ++ struct sock *sk = (struct sock *)tp; ++ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; ++ + tp->lost += tcp_skb_pcount(skb); ++ if (ca_ops->skb_marked_lost) ++ ca_ops->skb_marked_lost(sk, skb); + } + + void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) +@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, + WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); + tcp_skb_pcount_add(skb, -pcount); + ++ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ ++ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, ++ "prev in_flight: %u skb in_flight: %u pcount: %u", ++ TCP_SKB_CB(prev)->tx.in_flight, ++ TCP_SKB_CB(skb)->tx.in_flight, ++ pcount)) ++ TCP_SKB_CB(skb)->tx.in_flight = 0; ++ else ++ TCP_SKB_CB(skb)->tx.in_flight -= pcount; ++ TCP_SKB_CB(prev)->tx.in_flight += pcount; ++ + /* When we're adding to gso_segs == 1, gso_size will be zero, + * in theory this shouldn't be necessary but as long as DSACK + * code can come after this skb later on it's better to keep +@@ -3811,6 +3827,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) + + prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; + rs.prior_in_flight = tcp_packets_in_flight(tp); ++ tcp_rate_check_app_limited(sk); + + /* ts_recent update must be made after we are sure that the packet + * is in window. +@@ -3909,6 +3926,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) + delivered = tcp_newly_delivered(sk, delivered, flag); + lost = tp->lost - lost; /* freshly marked lost */ + rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); ++ rs.is_ece = !!(flag & FLAG_ECE); + tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); + tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); + tcp_xmit_recovery(sk, rexmit); +@@ -5508,13 +5526,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) + + /* More than one full frame received... */ + if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && ++ (tp->fast_ack_mode == 1 || + /* ... and right edge of window advances far enough. + * (tcp_recvmsg() will send ACK otherwise). + * If application uses SO_RCVLOWAT, we want send ack now if + * we have not received enough bytes to satisfy the condition. + */ +- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || +- __tcp_select_window(sk) >= tp->rcv_wnd)) || ++ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || ++ __tcp_select_window(sk) >= tp->rcv_wnd))) || + /* We ACK each frame or... */ + tcp_in_quickack_mode(sk) || + /* Protocol state mandates a one-time immediate ACK */ +diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c +index 5b019ba2b9d2..5884f4388c61 100644 +--- a/net/ipv4/tcp_ipv4.c ++++ b/net/ipv4/tcp_ipv4.c +@@ -3165,6 +3165,13 @@ static int __net_init tcp_sk_init(struct net *net) + net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0; + atomic_set(&net->ipv4.tfo_active_disable_times, 0); + ++ /* Set default values for PLB */ ++ net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */ ++ net->ipv4.sysctl_tcp_plb_cong_thresh = 128; /* 50% congestion */ ++ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3; ++ net->ipv4.sysctl_tcp_plb_rehash_rounds = 12; ++ net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60; ++ + /* Reno is always built in */ + if (!net_eq(net, &init_net) && + bpf_try_module_get(init_net.ipv4.tcp_congestion_control, +diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c +index 290019de766d..15e89addae08 100644 +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -375,7 +375,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, + th->cwr = 1; + skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; + } +- } else if (!tcp_ca_needs_ecn(sk)) { ++ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && ++ !tcp_ca_needs_ecn(sk)) { + /* ACK or retransmitted segment: clear ECT|CE */ + INET_ECN_dontxmit(sk); + } +@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, + { + struct tcp_sock *tp = tcp_sk(sk); + struct sk_buff *buff; +- int nsize, old_factor; ++ int nsize, old_factor, inflight_prev; + long limit; + int nlen; + u8 flags; +@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, + + if (diff) + tcp_adjust_pcount(sk, skb, diff); ++ ++ /* Set buff tx.in_flight as if buff were sent by itself. */ ++ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; ++ if (WARN_ONCE(inflight_prev < 0, ++ "inconsistent: tx.in_flight: %u old_factor: %d", ++ TCP_SKB_CB(skb)->tx.in_flight, old_factor)) ++ inflight_prev = 0; ++ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + ++ tcp_skb_pcount(buff); + } + + /* Link BUFF into the send queue. */ +@@ -1988,13 +1998,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, + static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) + { + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; +- u32 min_tso, tso_segs; +- +- min_tso = ca_ops->min_tso_segs ? +- ca_ops->min_tso_segs(sk) : +- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); ++ u32 tso_segs; + +- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); ++ tso_segs = ca_ops->tso_segs ? ++ ca_ops->tso_segs(sk, mss_now) : ++ tcp_tso_autosize(sk, mss_now, ++ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs)); + return min_t(u32, tso_segs, sk->sk_gso_max_segs); + } + +@@ -2630,6 +2639,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, + skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); + list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); + tcp_init_tso_segs(skb, mss_now); ++ tcp_set_tx_in_flight(sk, skb); + goto repair; /* Skip network transmission */ + } + +diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c +new file mode 100644 +index 000000000000..71b02c0404ce +--- /dev/null ++++ b/net/ipv4/tcp_plb.c +@@ -0,0 +1,100 @@ ++/* Protective Load Balancing (PLB) ++ * ++ * PLB was designed to reduce link load imbalance across datacenter ++ * switches. PLB is a host-based optimization; it leverages congestion ++ * signals from the transport layer to randomly change the path of the ++ * connection experiencing sustained congestion. PLB prefers to repath ++ * after idle periods to minimize packet reordering. It repaths by ++ * changing the IPv6 Flow Label on the packets of a connection, which ++ * datacenter switches include as part of ECMP/WCMP hashing. ++ * ++ * PLB is described in detail in: ++ * ++ * Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu, ++ * Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson, ++ * David Wetherall,Abdul Kabbani: ++ * "PLB: Congestion Signals are Simple and Effective for ++ * Network Load Balancing" ++ * In ACM SIGCOMM 2022, Amsterdam Netherlands. ++ * ++ */ ++ ++#include ++ ++/* Called once per round-trip to update PLB state for a connection. */ ++void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, ++ const int cong_ratio) ++{ ++ struct net *net = sock_net(sk); ++ ++ if (!plb->enabled) ++ return; ++ ++ if (cong_ratio >= 0) { ++ if (cong_ratio < net->ipv4.sysctl_tcp_plb_cong_thresh) ++ plb->consec_cong_rounds = 0; ++ else if (plb->consec_cong_rounds < ++ net->ipv4.sysctl_tcp_plb_rehash_rounds) ++ plb->consec_cong_rounds++; ++ } ++} ++EXPORT_SYMBOL_GPL(tcp_plb_update_state); ++ ++/* Check whether recent congestion has been persistent enough to warrant ++ * a load balancing decision that switches the connection to another path. ++ */ ++void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) ++{ ++ struct net *net = sock_net(sk); ++ bool can_idle_rehash, can_force_rehash; ++ ++ if (!plb->enabled) ++ return; ++ ++ /* Note that tcp_jiffies32 can wrap, so we clear pause_until ++ * to 0 to indicate there is no recent RTO event that constrains ++ * PLB rehashing. ++ */ ++ if (plb->pause_until && ++ !before(tcp_jiffies32, plb->pause_until)) ++ plb->pause_until = 0; ++ ++ can_idle_rehash = net->ipv4.sysctl_tcp_plb_idle_rehash_rounds && ++ !tcp_sk(sk)->packets_out && ++ plb->consec_cong_rounds >= ++ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds; ++ can_force_rehash = plb->consec_cong_rounds >= ++ net->ipv4.sysctl_tcp_plb_rehash_rounds; ++ ++ if (!plb->pause_until && (can_idle_rehash || can_force_rehash)) { ++ sk_rethink_txhash(sk); ++ plb->consec_cong_rounds = 0; ++ tcp_sk(sk)->ecn_rehash++; ++ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPECNREHASH); ++ } ++} ++EXPORT_SYMBOL_GPL(tcp_plb_check_rehash); ++ ++/* Upon RTO, disallow load balancing for a while, to avoid having load ++ * balancing decisions switch traffic to a black-holed path that was ++ * previously avoided with a sk_rethink_txhash() call at RTO time. ++ */ ++void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb) ++{ ++ struct net *net = sock_net(sk); ++ u32 pause; ++ ++ if (!plb->enabled) ++ return; ++ ++ pause = net->ipv4.sysctl_tcp_plb_suspend_rto_sec * HZ; ++ pause += prandom_u32_max(pause); ++ plb->pause_until = tcp_jiffies32 + pause; ++ ++ /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call ++ * that may switch this connection to a path with completely different ++ * congestion characteristics. ++ */ ++ plb->consec_cong_rounds = 0; ++} ++EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto); +diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c +index a8f6d9d06f2e..a8b4c9504570 100644 +--- a/net/ipv4/tcp_rate.c ++++ b/net/ipv4/tcp_rate.c +@@ -34,6 +34,24 @@ + * ready to send in the write queue. + */ + ++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ u32 in_flight; ++ ++ /* Check, sanitize, and record packets in flight after skb was sent. */ ++ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); ++ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, ++ "insane in_flight %u cc %s mss %u " ++ "cwnd %u pif %u %u %u %u\n", ++ in_flight, inet_csk(sk)->icsk_ca_ops->name, ++ tp->mss_cache, tp->snd_cwnd, ++ tp->packets_out, tp->retrans_out, ++ tp->sacked_out, tp->lost_out)) ++ in_flight = TCPCB_IN_FLIGHT_MAX; ++ TCP_SKB_CB(skb)->tx.in_flight = in_flight; ++} ++ + /* Snapshot the current delivery information in the skb, to generate + * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). + */ +@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) + TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; + TCP_SKB_CB(skb)->tx.delivered = tp->delivered; + TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; ++ TCP_SKB_CB(skb)->tx.lost = tp->lost; + TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; ++ tcp_set_tx_in_flight(sk, skb); + } + + /* When an skb is sacked or acked, we fill in the rate sample with the (prior) +@@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, + if (!rs->prior_delivered || + tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, + scb->end_seq, rs->last_end_seq)) { ++ rs->prior_lost = scb->tx.lost; + rs->prior_delivered_ce = scb->tx.delivered_ce; + rs->prior_delivered = scb->tx.delivered; + rs->prior_mstamp = scb->tx.delivered_mstamp; + rs->is_app_limited = scb->tx.is_app_limited; + rs->is_retrans = scb->sacked & TCPCB_RETRANS; + rs->last_end_seq = scb->end_seq; ++ rs->tx_in_flight = scb->tx.in_flight; + + /* Record send time of most recently ACKed packet: */ + tp->first_tx_mstamp = tx_tstamp; + /* Find the duration of the "send phase" of this window: */ +- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, +- scb->tx.first_tx_mstamp); ++ rs->interval_us = tcp_stamp32_us_delta( ++ tp->first_tx_mstamp, ++ scb->tx.first_tx_mstamp); + + } + /* Mark off the skb delivered once it's sacked to avoid being +@@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, + return; + } + rs->delivered = tp->delivered - rs->prior_delivered; ++ rs->lost = tp->lost - rs->prior_lost; + + rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; + /* delivered_ce occupies less than 32 bits in the skb control block */ +@@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, + * longer phase. + */ + snd_us = rs->interval_us; /* send phase */ +- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, ++ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, + rs->prior_mstamp); /* ack phase */ + rs->interval_us = max(snd_us, ack_us); + +diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c +index b4dfb82d6ecb..bb613fc8948a 100644 +--- a/net/ipv4/tcp_timer.c ++++ b/net/ipv4/tcp_timer.c +@@ -605,6 +605,7 @@ void tcp_write_timer_handler(struct sock *sk) + return; + } + ++ tcp_rate_check_app_limited(sk); + tcp_mstamp_refresh(tcp_sk(sk)); + event = icsk->icsk_pending; + +-- +2.38.0 + +From 0d426832aeb2f836b0d96b608bd4a884c0d19220 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 5 Oct 2022 19:08:58 +0200 +Subject: [PATCH 03/17] futex-winesync + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/devices.txt | 3 +- + Documentation/userspace-api/index.rst | 1 + + .../userspace-api/ioctl/ioctl-number.rst | 2 + + Documentation/userspace-api/winesync.rst | 444 +++++ + MAINTAINERS | 9 + + drivers/misc/Kconfig | 11 + + drivers/misc/Makefile | 1 + + drivers/misc/winesync.c | 1212 ++++++++++++++ + include/linux/miscdevice.h | 1 + + include/uapi/linux/futex.h | 13 + + include/uapi/linux/winesync.h | 71 + + kernel/futex/syscalls.c | 75 +- + tools/testing/selftests/Makefile | 1 + + .../selftests/drivers/winesync/Makefile | 8 + + .../testing/selftests/drivers/winesync/config | 1 + + .../selftests/drivers/winesync/winesync.c | 1479 +++++++++++++++++ + 16 files changed, 3330 insertions(+), 2 deletions(-) + create mode 100644 Documentation/userspace-api/winesync.rst + create mode 100644 drivers/misc/winesync.c + create mode 100644 include/uapi/linux/winesync.h + create mode 100644 tools/testing/selftests/drivers/winesync/Makefile + create mode 100644 tools/testing/selftests/drivers/winesync/config + create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c + +diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt +index 9764d6edb189..a4696d3b4a5a 100644 +--- a/Documentation/admin-guide/devices.txt ++++ b/Documentation/admin-guide/devices.txt +@@ -376,8 +376,9 @@ + 240 = /dev/userio Serio driver testing device + 241 = /dev/vhost-vsock Host kernel driver for virtio vsock + 242 = /dev/rfkill Turning off radio transmissions (rfkill) ++ 243 = /dev/winesync Wine synchronization primitive device + +- 243-254 Reserved for local use ++ 244-254 Reserved for local use + 255 Reserved for MISC_DYNAMIC_MINOR + + 11 char Raw keyboard device (Linux/SPARC only) +diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst +index a61eac0c73f8..0bf697ddcb09 100644 +--- a/Documentation/userspace-api/index.rst ++++ b/Documentation/userspace-api/index.rst +@@ -29,6 +29,7 @@ place where this information is gathered. + sysfs-platform_profile + vduse + futex2 ++ winesync + + .. only:: subproject and html + +diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst +index 3b985b19f39d..3f313fd4338c 100644 +--- a/Documentation/userspace-api/ioctl/ioctl-number.rst ++++ b/Documentation/userspace-api/ioctl/ioctl-number.rst +@@ -375,6 +375,8 @@ Code Seq# Include File Comments + + 0xF6 all LTTng Linux Trace Toolkit Next Generation + ++0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives ++ + 0xF8 all arch/x86/include/uapi/asm/amd_hsmp.h AMD HSMP EPYC system management interface driver + + 0xFD all linux/dm-ioctl.h +diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst +new file mode 100644 +index 000000000000..d7055bf41820 +--- /dev/null ++++ b/Documentation/userspace-api/winesync.rst +@@ -0,0 +1,444 @@ ++===================================== ++Wine synchronization primitive driver ++===================================== ++ ++This page documents the user-space API for the winesync driver. ++ ++winesync is a support driver for emulation of NT synchronization ++primitives by the Wine project or other NT emulators. It exists ++because implementation in user-space, using existing tools, cannot ++simultaneously satisfy performance, correctness, and security ++constraints. It is implemented entirely in software, and does not ++drive any hardware device. ++ ++This interface is meant as a compatibility tool only, and should not ++be used for general synchronization. Instead use generic, versatile ++interfaces such as futex(2) and poll(2). ++ ++Synchronization primitives ++========================== ++ ++The winesync driver exposes three types of synchronization primitives: ++semaphores, mutexes, and events. ++ ++A semaphore holds a single volatile 32-bit counter, and a static ++32-bit integer denoting the maximum value. It is considered signaled ++when the counter is nonzero. The counter is decremented by one when a ++wait is satisfied. Both the initial and maximum count are established ++when the semaphore is created. ++ ++A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit ++identifier denoting its owner. A mutex is considered signaled when its ++owner is zero (indicating that it is not owned). The recursion count ++is incremented when a wait is satisfied, and ownership is set to the ++given identifier. ++ ++A mutex also holds an internal flag denoting whether its previous ++owner has died; such a mutex is said to be inconsistent. Owner death ++is not tracked automatically based on thread death, but rather must be ++communicated using ``WINESYNC_IOC_KILL_OWNER``. An inconsistent mutex ++is inherently considered unowned. ++ ++Except for the "unowned" semantics of zero, the actual value of the ++owner identifier is not interpreted by the winesync driver at all. The ++intended use is to store a thread identifier; however, the winesync ++driver does not actually validate that a calling thread provides ++consistent or unique identifiers. ++ ++An event holds a volatile boolean state denoting whether it is ++signaled or not. There are two types of events, auto-reset and ++manual-reset. An auto-reset event is designaled when a wait is ++satisfied; a manual-reset event is not. The event type is specified ++when the event is created. ++ ++Unless specified otherwise, all operations on an object are atomic and ++totally ordered with respect to other operations on the same object. ++ ++Objects are represented by unsigned 32-bit integers. ++ ++Char device ++=========== ++ ++The winesync driver creates a single char device /dev/winesync. Each ++file description opened on the device represents a unique namespace. ++That is, objects created on one open file description are shared ++across all its individual descriptors, but are not shared with other ++open() calls on the same device. The same file description may be ++shared across multiple processes. ++ ++ioctl reference ++=============== ++ ++All operations on the device are done through ioctls. There are three ++structures used in ioctl calls:: ++ ++ struct winesync_sem_args { ++ __u32 sem; ++ __u32 count; ++ __u32 max; ++ }; ++ ++ struct winesync_mutex_args { ++ __u32 mutex; ++ __u32 owner; ++ __u32 count; ++ }; ++ ++ struct winesync_event_args { ++ __u32 event; ++ __u32 signaled; ++ __u32 manual; ++ }; ++ ++ struct winesync_wait_args { ++ __u64 timeout; ++ __u64 objs; ++ __u32 count; ++ __u32 owner; ++ __u32 index; ++ __u32 pad; ++ }; ++ ++Depending on the ioctl, members of the structure may be used as input, ++output, or not at all. All ioctls return 0 on success. ++ ++The ioctls are as follows: ++ ++.. c:macro:: WINESYNC_IOC_CREATE_SEM ++ ++ Create a semaphore object. Takes a pointer to struct ++ :c:type:`winesync_sem_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``sem`` ++ - On output, contains the identifier of the created semaphore. ++ * - ``count`` ++ - Initial count of the semaphore. ++ * - ``max`` ++ - Maximum count of the semaphore. ++ ++ Fails with ``EINVAL`` if ``count`` is greater than ``max``. ++ ++.. c:macro:: WINESYNC_IOC_CREATE_MUTEX ++ ++ Create a mutex object. Takes a pointer to struct ++ :c:type:`winesync_mutex_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``mutex`` ++ - On output, contains the identifier of the created mutex. ++ * - ``count`` ++ - Initial recursion count of the mutex. ++ * - ``owner`` ++ - Initial owner of the mutex. ++ ++ If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is ++ zero and ``count`` is nonzero, the function fails with ``EINVAL``. ++ ++.. c:macro:: WINESYNC_IOC_CREATE_EVENT ++ ++ Create an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - On output, contains the identifier of the created event. ++ * - ``signaled`` ++ - If nonzero, the event is initially signaled, otherwise ++ nonsignaled. ++ * - ``manual`` ++ - If nonzero, the event is a manual-reset event, otherwise ++ auto-reset. ++ ++.. c:macro:: WINESYNC_IOC_DELETE ++ ++ Delete an object of any type. Takes an input-only pointer to a ++ 32-bit integer denoting the object to delete. ++ ++ Wait ioctls currently in progress are not interrupted, and behave as ++ if the object remains valid. ++ ++.. c:macro:: WINESYNC_IOC_PUT_SEM ++ ++ Post to a semaphore object. Takes a pointer to struct ++ :c:type:`winesync_sem_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``sem`` ++ - Semaphore object to post to. ++ * - ``count`` ++ - Count to add to the semaphore. On output, contains the ++ previous count of the semaphore. ++ * - ``max`` ++ - Not used. ++ ++ If adding ``count`` to the semaphore's current count would raise the ++ latter past the semaphore's maximum count, the ioctl fails with ++ ``EOVERFLOW`` and the semaphore is not affected. If raising the ++ semaphore's count causes it to become signaled, eligible threads ++ waiting on this semaphore will be woken and the semaphore's count ++ decremented appropriately. ++ ++.. c:macro:: WINESYNC_IOC_PUT_MUTEX ++ ++ Release a mutex object. Takes a pointer to struct ++ :c:type:`winesync_mutex_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``mutex`` ++ - Mutex object to release. ++ * - ``owner`` ++ - Mutex owner identifier. ++ * - ``count`` ++ - On output, contains the previous recursion count. ++ ++ If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` ++ is not the current owner of the mutex, the ioctl fails with ++ ``EPERM``. ++ ++ The mutex's count will be decremented by one. If decrementing the ++ mutex's count causes it to become zero, the mutex is marked as ++ unowned and signaled, and eligible threads waiting on it will be ++ woken as appropriate. ++ ++.. c:macro:: WINESYNC_IOC_SET_EVENT ++ ++ Signal an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object to set. ++ * - ``signaled`` ++ - On output, contains the previous state of the event. ++ * - ``manual`` ++ - Unused. ++ ++ Eligible threads will be woken, and auto-reset events will be ++ designaled appropriately. ++ ++.. c:macro:: WINESYNC_IOC_RESET_EVENT ++ ++ Designal an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object to reset. ++ * - ``signaled`` ++ - On output, contains the previous state of the event. ++ * - ``manual`` ++ - Unused. ++ ++.. c:macro:: WINESYNC_IOC_PULSE_EVENT ++ ++ Wake threads waiting on an event object without leaving it in a ++ signaled state. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object to pulse. ++ * - ``signaled`` ++ - On output, contains the previous state of the event. ++ * - ``manual`` ++ - Unused. ++ ++ A pulse operation can be thought of as a set followed by a reset, ++ performed as a single atomic operation. If two threads are waiting ++ on an auto-reset event which is pulsed, only one will be woken. If ++ two threads are waiting a manual-reset event which is pulsed, both ++ will be woken. However, in both cases, the event will be unsignaled ++ afterwards, and a simultaneous read operation will always report the ++ event as unsignaled. ++ ++.. c:macro:: WINESYNC_IOC_READ_SEM ++ ++ Read the current state of a semaphore object. Takes a pointer to ++ struct :c:type:`winesync_sem_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``sem`` ++ - Semaphore object to read. ++ * - ``count`` ++ - On output, contains the current count of the semaphore. ++ * - ``max`` ++ - On output, contains the maximum count of the semaphore. ++ ++.. c:macro:: WINESYNC_IOC_READ_MUTEX ++ ++ Read the current state of a mutex object. Takes a pointer to struct ++ :c:type:`winesync_mutex_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``mutex`` ++ - Mutex object to read. ++ * - ``owner`` ++ - On output, contains the current owner of the mutex, or zero ++ if the mutex is not currently owned. ++ * - ``count`` ++ - On output, contains the current recursion count of the mutex. ++ ++ If the mutex is marked as inconsistent, the function fails with ++ ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to ++ zero. ++ ++.. c:macro:: WINESYNC_IOC_READ_EVENT ++ ++ Read the current state of an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object. ++ * - ``signaled`` ++ - On output, contains the current state of the event. ++ * - ``manual`` ++ - On output, contains 1 if the event is a manual-reset event, ++ and 0 otherwise. ++ ++.. c:macro:: WINESYNC_IOC_KILL_OWNER ++ ++ Mark any mutexes owned by the given owner as unowned and ++ inconsistent. Takes an input-only pointer to a 32-bit integer ++ denoting the owner. If the owner is zero, the ioctl fails with ++ ``EINVAL``. ++ ++ For each mutex currently owned by the given owner, eligible threads ++ waiting on said mutex will be woken as appropriate (and such waits ++ will fail with ``EOWNERDEAD``, as described below). ++ ++ The operation as a whole is not atomic; however, the modification of ++ each mutex is atomic and totally ordered with respect to other ++ operations on the same mutex. ++ ++.. c:macro:: WINESYNC_IOC_WAIT_ANY ++ ++ Poll on any of a list of objects, atomically acquiring at most one. ++ Takes a pointer to struct :c:type:`winesync_wait_args`, which is ++ used as follows: ++ ++ .. list-table:: ++ ++ * - ``timeout`` ++ - Optional pointer to a 64-bit struct :c:type:`timespec` ++ (specified as an integer so that the structure has the same ++ size regardless of architecture). The timeout is specified in ++ absolute format, as measured against the MONOTONIC clock. If ++ the timeout is equal to or earlier than the current time, the ++ function returns immediately without sleeping. If ``timeout`` ++ is zero, i.e. NULL, the function will sleep until an object ++ is signaled, and will not fail with ``ETIMEDOUT``. ++ * - ``objs`` ++ - Pointer to an array of ``count`` 32-bit object identifiers ++ (specified as an integer so that the structure has the same ++ size regardless of architecture). If any identifier is ++ invalid, the function fails with ``EINVAL``. ++ * - ``count`` ++ - Number of object identifiers specified in the ``objs`` array. ++ * - ``owner`` ++ - Mutex owner identifier. If any object in ``objs`` is a mutex, ++ the ioctl will attempt to acquire that mutex on behalf of ++ ``owner``. If ``owner`` is zero, the ioctl fails with ++ ``EINVAL``. ++ * - ``index`` ++ - On success, contains the index (into ``objs``) of the object ++ which was signaled. If ``alert`` was signaled instead, ++ this contains ``count``. ++ * - ``alert`` ++ - Optional event object identifier. If nonzero, this specifies ++ an "alert" event object which, if signaled, will terminate ++ the wait. If nonzero, the identifier must point to a valid ++ event. ++ ++ This function attempts to acquire one of the given objects. If ++ unable to do so, it sleeps until an object becomes signaled, ++ subsequently acquiring it, or the timeout expires. In the latter ++ case the ioctl fails with ``ETIMEDOUT``. The function only acquires ++ one object, even if multiple objects are signaled. ++ ++ A semaphore is considered to be signaled if its count is nonzero, ++ and is acquired by decrementing its count by one. A mutex is ++ considered to be signaled if it is unowned or if its owner matches ++ the ``owner`` argument, and is acquired by incrementing its ++ recursion count by one and setting its owner to the ``owner`` ++ argument. An auto-reset event is acquired by designaling it; a ++ manual-reset event is not affected by acquisition. ++ ++ Acquisition is atomic and totally ordered with respect to other ++ operations on the same object. If two wait operations (with ++ different ``owner`` identifiers) are queued on the same mutex, only ++ one is signaled. If two wait operations are queued on the same ++ semaphore, and a value of one is posted to it, only one is signaled. ++ The order in which threads are signaled is not specified. ++ ++ If an inconsistent mutex is acquired, the ioctl fails with ++ ``EOWNERDEAD``. Although this is a failure return, the function may ++ otherwise be considered successful. The mutex is marked as owned by ++ the given owner (with a recursion count of 1) and as no longer ++ inconsistent, and ``index`` is still set to the index of the mutex. ++ ++ The ``alert`` argument is an "extra" event which can terminate the ++ wait, independently of all other objects. If members of ``objs`` and ++ ``alert`` are both simultaneously signaled, a member of ``objs`` ++ will always be given priority and acquired first. Aside from this, ++ for "any" waits, there is no difference between passing an event as ++ this parameter, and passing it as an additional object at the end of ++ the ``objs`` array. For "all" waits, there is an additional ++ difference, as described below. ++ ++ It is valid to pass the same object more than once, including by ++ passing the same event in the ``objs`` array and in ``alert``. If a ++ wakeup occurs due to that object being signaled, ``index`` is set to ++ the lowest index corresponding to that object. ++ ++ The function may fail with ``EINTR`` if a signal is received. ++ ++.. c:macro:: WINESYNC_IOC_WAIT_ALL ++ ++ Poll on a list of objects, atomically acquiring all of them. Takes a ++ pointer to struct :c:type:`winesync_wait_args`, which is used ++ identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is ++ always filled with zero on success if not woken via alert. ++ ++ This function attempts to simultaneously acquire all of the given ++ objects. If unable to do so, it sleeps until all objects become ++ simultaneously signaled, subsequently acquiring them, or the timeout ++ expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and ++ no objects are modified. ++ ++ Objects may become signaled and subsequently designaled (through ++ acquisition by other threads) while this thread is sleeping. Only ++ once all objects are simultaneously signaled does the ioctl acquire ++ them and return. The entire acquisition is atomic and totally ++ ordered with respect to other operations on any of the given ++ objects. ++ ++ If an inconsistent mutex is acquired, the ioctl fails with ++ ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects ++ are nevertheless marked as acquired. Note that if multiple mutex ++ objects are specified, there is no way to know which were marked as ++ inconsistent. ++ ++ As with "any" waits, the ``alert`` argument is an "extra" event ++ which can terminate the wait. Critically, however, an "all" wait ++ will succeed if all members in ``objs`` are signaled, *or* if ++ ``alert`` is signaled. In the latter case ``index`` will be set to ++ ``count``. As with "any" waits, if both conditions are filled, the ++ former takes priority, and objects in ``objs`` will be acquired. ++ ++ Unlike ``WINESYNC_IOC_WAIT_ANY``, it is not valid to pass the same ++ object more than once, nor is it valid to pass the same object in ++ ``objs`` and in ``alert`` If this is attempted, the function fails ++ with ``EINVAL``. +diff --git a/MAINTAINERS b/MAINTAINERS +index 72b9654f764c..ff31beb17835 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -21920,6 +21920,15 @@ M: David Härdeman + S: Maintained + F: drivers/media/rc/winbond-cir.c + ++WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER ++M: Zebediah Figura ++L: wine-devel@winehq.org ++S: Supported ++F: Documentation/userspace-api/winesync.rst ++F: drivers/misc/winesync.c ++F: include/uapi/linux/winesync.h ++F: tools/testing/selftests/drivers/winesync/ ++ + WINSYSTEMS EBC-C384 WATCHDOG DRIVER + M: William Breathitt Gray + L: linux-watchdog@vger.kernel.org +diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig +index 94e9fb4cdd76..bdf56cd530b7 100644 +--- a/drivers/misc/Kconfig ++++ b/drivers/misc/Kconfig +@@ -483,6 +483,17 @@ config OPEN_DICE + + If unsure, say N. + ++config WINESYNC ++ tristate "Synchronization primitives for Wine" ++ help ++ This module provides kernel support for synchronization primitives ++ used by Wine. It is not a hardware driver. ++ ++ To compile this driver as a module, choose M here: the ++ module will be called winesync. ++ ++ If unsure, say N. ++ + config VCPU_STALL_DETECTOR + tristate "Guest vCPU stall detector" + depends on OF && HAS_IOMEM +diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile +index 2be8542616dd..d061fe45407b 100644 +--- a/drivers/misc/Makefile ++++ b/drivers/misc/Makefile +@@ -58,6 +58,7 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ + obj-$(CONFIG_UACCE) += uacce/ + obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o + obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o ++obj-$(CONFIG_WINESYNC) += winesync.o + obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o + obj-$(CONFIG_OPEN_DICE) += open-dice.o + obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o +\ No newline at end of file +diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c +new file mode 100644 +index 000000000000..7a28f58dbbf2 +--- /dev/null ++++ b/drivers/misc/winesync.c +@@ -0,0 +1,1212 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * winesync.c - Kernel driver for Wine synchronization primitives ++ * ++ * Copyright (C) 2021 Zebediah Figura ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define WINESYNC_NAME "winesync" ++ ++enum winesync_type { ++ WINESYNC_TYPE_SEM, ++ WINESYNC_TYPE_MUTEX, ++ WINESYNC_TYPE_EVENT, ++}; ++ ++struct winesync_obj { ++ struct rcu_head rhead; ++ struct kref refcount; ++ spinlock_t lock; ++ ++ /* ++ * any_waiters is protected by the object lock, but all_waiters is ++ * protected by the device wait_all_lock. ++ */ ++ struct list_head any_waiters; ++ struct list_head all_waiters; ++ ++ /* ++ * Hint describing how many tasks are queued on this object in a ++ * wait-all operation. ++ * ++ * Any time we do a wake, we may need to wake "all" waiters as well as ++ * "any" waiters. In order to atomically wake "all" waiters, we must ++ * lock all of the objects, and that means grabbing the wait_all_lock ++ * below (and, due to lock ordering rules, before locking this object). ++ * However, wait-all is a rare operation, and grabbing the wait-all ++ * lock for every wake would create unnecessary contention. Therefore we ++ * first check whether all_hint is zero, and, if it is, we skip trying ++ * to wake "all" waiters. ++ * ++ * This hint isn't protected by any lock. It might change during the ++ * course of a wake, but there's no meaningful race there; it's only a ++ * hint. ++ * ++ * Since wait requests must originate from user-space threads, we're ++ * limited here by PID_MAX_LIMIT, so there's no risk of saturation. ++ */ ++ atomic_t all_hint; ++ ++ enum winesync_type type; ++ ++ /* The following fields are protected by the object lock. */ ++ union { ++ struct { ++ __u32 count; ++ __u32 max; ++ } sem; ++ struct { ++ __u32 count; ++ __u32 owner; ++ bool ownerdead; ++ } mutex; ++ struct { ++ bool manual; ++ bool signaled; ++ } event; ++ } u; ++}; ++ ++struct winesync_q_entry { ++ struct list_head node; ++ struct winesync_q *q; ++ struct winesync_obj *obj; ++ __u32 index; ++}; ++ ++struct winesync_q { ++ struct task_struct *task; ++ __u32 owner; ++ ++ /* ++ * Protected via atomic_cmpxchg(). Only the thread that wins the ++ * compare-and-swap may actually change object states and wake this ++ * task. ++ */ ++ atomic_t signaled; ++ ++ bool all; ++ bool ownerdead; ++ __u32 count; ++ struct winesync_q_entry entries[]; ++}; ++ ++struct winesync_device { ++ /* ++ * Wait-all operations must atomically grab all objects, and be totally ++ * ordered with respect to each other and wait-any operations. If one ++ * thread is trying to acquire several objects, another thread cannot ++ * touch the object at the same time. ++ * ++ * We achieve this by grabbing multiple object locks at the same time. ++ * However, this creates a lock ordering problem. To solve that problem, ++ * wait_all_lock is taken first whenever multiple objects must be locked ++ * at the same time. ++ */ ++ spinlock_t wait_all_lock; ++ ++ struct xarray objects; ++}; ++ ++static struct winesync_obj *get_obj(struct winesync_device *dev, __u32 id) ++{ ++ struct winesync_obj *obj; ++ ++ rcu_read_lock(); ++ obj = xa_load(&dev->objects, id); ++ if (obj && !kref_get_unless_zero(&obj->refcount)) ++ obj = NULL; ++ rcu_read_unlock(); ++ ++ return obj; ++} ++ ++static void destroy_obj(struct kref *ref) ++{ ++ struct winesync_obj *obj = container_of(ref, struct winesync_obj, refcount); ++ ++ kfree_rcu(obj, rhead); ++} ++ ++static void put_obj(struct winesync_obj *obj) ++{ ++ kref_put(&obj->refcount, destroy_obj); ++} ++ ++static struct winesync_obj *get_obj_typed(struct winesync_device *dev, __u32 id, ++ enum winesync_type type) ++{ ++ struct winesync_obj *obj = get_obj(dev, id); ++ ++ if (obj && obj->type != type) { ++ put_obj(obj); ++ return NULL; ++ } ++ return obj; ++} ++ ++static int winesync_char_open(struct inode *inode, struct file *file) ++{ ++ struct winesync_device *dev; ++ ++ dev = kzalloc(sizeof(*dev), GFP_KERNEL); ++ if (!dev) ++ return -ENOMEM; ++ ++ spin_lock_init(&dev->wait_all_lock); ++ ++ xa_init_flags(&dev->objects, XA_FLAGS_ALLOC); ++ ++ file->private_data = dev; ++ return nonseekable_open(inode, file); ++} ++ ++static int winesync_char_release(struct inode *inode, struct file *file) ++{ ++ struct winesync_device *dev = file->private_data; ++ struct winesync_obj *obj; ++ unsigned long id; ++ ++ xa_for_each(&dev->objects, id, obj) ++ put_obj(obj); ++ ++ xa_destroy(&dev->objects); ++ ++ kfree(dev); ++ ++ return 0; ++} ++ ++static void init_obj(struct winesync_obj *obj) ++{ ++ kref_init(&obj->refcount); ++ atomic_set(&obj->all_hint, 0); ++ spin_lock_init(&obj->lock); ++ INIT_LIST_HEAD(&obj->any_waiters); ++ INIT_LIST_HEAD(&obj->all_waiters); ++} ++ ++static bool is_signaled(struct winesync_obj *obj, __u32 owner) ++{ ++ lockdep_assert_held(&obj->lock); ++ ++ switch (obj->type) { ++ case WINESYNC_TYPE_SEM: ++ return !!obj->u.sem.count; ++ case WINESYNC_TYPE_MUTEX: ++ if (obj->u.mutex.owner && obj->u.mutex.owner != owner) ++ return false; ++ return obj->u.mutex.count < UINT_MAX; ++ case WINESYNC_TYPE_EVENT: ++ return obj->u.event.signaled; ++ } ++ ++ WARN(1, "bad object type %#x\n", obj->type); ++ return false; ++} ++ ++/* ++ * "locked_obj" is an optional pointer to an object which is already locked and ++ * should not be locked again. This is necessary so that changing an object's ++ * state and waking it can be a single atomic operation. ++ */ ++static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, ++ struct winesync_obj *locked_obj) ++{ ++ __u32 count = q->count; ++ bool can_wake = true; ++ __u32 i; ++ ++ lockdep_assert_held(&dev->wait_all_lock); ++ if (locked_obj) ++ lockdep_assert_held(&locked_obj->lock); ++ ++ for (i = 0; i < count; i++) { ++ if (q->entries[i].obj != locked_obj) ++ spin_lock(&q->entries[i].obj->lock); ++ } ++ ++ for (i = 0; i < count; i++) { ++ if (!is_signaled(q->entries[i].obj, q->owner)) { ++ can_wake = false; ++ break; ++ } ++ } ++ ++ if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { ++ for (i = 0; i < count; i++) { ++ struct winesync_obj *obj = q->entries[i].obj; ++ ++ switch (obj->type) { ++ case WINESYNC_TYPE_SEM: ++ obj->u.sem.count--; ++ break; ++ case WINESYNC_TYPE_MUTEX: ++ if (obj->u.mutex.ownerdead) ++ q->ownerdead = true; ++ obj->u.mutex.ownerdead = false; ++ obj->u.mutex.count++; ++ obj->u.mutex.owner = q->owner; ++ break; ++ case WINESYNC_TYPE_EVENT: ++ if (!obj->u.event.manual) ++ obj->u.event.signaled = false; ++ break; ++ } ++ } ++ wake_up_process(q->task); ++ } ++ ++ for (i = 0; i < count; i++) { ++ if (q->entries[i].obj != locked_obj) ++ spin_unlock(&q->entries[i].obj->lock); ++ } ++} ++ ++static void try_wake_all_obj(struct winesync_device *dev, ++ struct winesync_obj *obj) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&dev->wait_all_lock); ++ lockdep_assert_held(&obj->lock); ++ ++ list_for_each_entry(entry, &obj->all_waiters, node) ++ try_wake_all(dev, entry->q, obj); ++} ++ ++static void try_wake_any_sem(struct winesync_obj *sem) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&sem->lock); ++ ++ list_for_each_entry(entry, &sem->any_waiters, node) { ++ struct winesync_q *q = entry->q; ++ ++ if (!sem->u.sem.count) ++ break; ++ ++ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { ++ sem->u.sem.count--; ++ wake_up_process(q->task); ++ } ++ } ++} ++ ++static void try_wake_any_mutex(struct winesync_obj *mutex) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&mutex->lock); ++ ++ list_for_each_entry(entry, &mutex->any_waiters, node) { ++ struct winesync_q *q = entry->q; ++ ++ if (mutex->u.mutex.count == UINT_MAX) ++ break; ++ if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) ++ continue; ++ ++ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { ++ if (mutex->u.mutex.ownerdead) ++ q->ownerdead = true; ++ mutex->u.mutex.ownerdead = false; ++ mutex->u.mutex.count++; ++ mutex->u.mutex.owner = q->owner; ++ wake_up_process(q->task); ++ } ++ } ++} ++ ++static void try_wake_any_event(struct winesync_obj *event) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&event->lock); ++ ++ list_for_each_entry(entry, &event->any_waiters, node) { ++ struct winesync_q *q = entry->q; ++ ++ if (!event->u.event.signaled) ++ break; ++ ++ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { ++ if (!event->u.event.manual) ++ event->u.event.signaled = false; ++ wake_up_process(q->task); ++ } ++ } ++} ++ ++static int winesync_create_sem(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_sem_args __user *user_args = argp; ++ struct winesync_sem_args args; ++ struct winesync_obj *sem; ++ __u32 id; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ if (args.count > args.max) ++ return -EINVAL; ++ ++ sem = kzalloc(sizeof(*sem), GFP_KERNEL); ++ if (!sem) ++ return -ENOMEM; ++ ++ init_obj(sem); ++ sem->type = WINESYNC_TYPE_SEM; ++ sem->u.sem.count = args.count; ++ sem->u.sem.max = args.max; ++ ++ ret = xa_alloc(&dev->objects, &id, sem, xa_limit_32b, GFP_KERNEL); ++ if (ret < 0) { ++ kfree(sem); ++ return ret; ++ } ++ ++ return put_user(id, &user_args->sem); ++} ++ ++static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_mutex_args __user *user_args = argp; ++ struct winesync_mutex_args args; ++ struct winesync_obj *mutex; ++ __u32 id; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ if (!args.owner != !args.count) ++ return -EINVAL; ++ ++ mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); ++ if (!mutex) ++ return -ENOMEM; ++ ++ init_obj(mutex); ++ mutex->type = WINESYNC_TYPE_MUTEX; ++ mutex->u.mutex.count = args.count; ++ mutex->u.mutex.owner = args.owner; ++ ++ ret = xa_alloc(&dev->objects, &id, mutex, xa_limit_32b, GFP_KERNEL); ++ if (ret < 0) { ++ kfree(mutex); ++ return ret; ++ } ++ ++ return put_user(id, &user_args->mutex); ++} ++ ++static int winesync_create_event(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ __u32 id; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ event = kzalloc(sizeof(*event), GFP_KERNEL); ++ if (!event) ++ return -ENOMEM; ++ ++ init_obj(event); ++ event->type = WINESYNC_TYPE_EVENT; ++ event->u.event.manual = args.manual; ++ event->u.event.signaled = args.signaled; ++ ++ ret = xa_alloc(&dev->objects, &id, event, xa_limit_32b, GFP_KERNEL); ++ if (ret < 0) { ++ kfree(event); ++ return ret; ++ } ++ ++ return put_user(id, &user_args->event); ++} ++ ++static int winesync_delete(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_obj *obj; ++ __u32 id; ++ ++ if (get_user(id, (__u32 __user *)argp)) ++ return -EFAULT; ++ ++ obj = xa_erase(&dev->objects, id); ++ if (!obj) ++ return -EINVAL; ++ ++ put_obj(obj); ++ return 0; ++} ++ ++/* ++ * Actually change the semaphore state, returning -EOVERFLOW if it is made ++ * invalid. ++ */ ++static int put_sem_state(struct winesync_obj *sem, __u32 count) ++{ ++ lockdep_assert_held(&sem->lock); ++ ++ if (sem->u.sem.count + count < sem->u.sem.count || ++ sem->u.sem.count + count > sem->u.sem.max) ++ return -EOVERFLOW; ++ ++ sem->u.sem.count += count; ++ return 0; ++} ++ ++static int winesync_put_sem(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_sem_args __user *user_args = argp; ++ struct winesync_sem_args args; ++ struct winesync_obj *sem; ++ __u32 prev_count; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ sem = get_obj_typed(dev, args.sem, WINESYNC_TYPE_SEM); ++ if (!sem) ++ return -EINVAL; ++ ++ if (atomic_read(&sem->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&sem->lock); ++ ++ prev_count = sem->u.sem.count; ++ ret = put_sem_state(sem, args.count); ++ if (!ret) { ++ try_wake_all_obj(dev, sem); ++ try_wake_any_sem(sem); ++ } ++ ++ spin_unlock(&sem->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&sem->lock); ++ ++ prev_count = sem->u.sem.count; ++ ret = put_sem_state(sem, args.count); ++ if (!ret) ++ try_wake_any_sem(sem); ++ ++ spin_unlock(&sem->lock); ++ } ++ ++ put_obj(sem); ++ ++ if (!ret && put_user(prev_count, &user_args->count)) ++ ret = -EFAULT; ++ ++ return ret; ++} ++ ++/* ++ * Actually change the mutex state, returning -EPERM if not the owner. ++ */ ++static int put_mutex_state(struct winesync_obj *mutex, ++ const struct winesync_mutex_args *args) ++{ ++ lockdep_assert_held(&mutex->lock); ++ ++ if (mutex->u.mutex.owner != args->owner) ++ return -EPERM; ++ ++ if (!--mutex->u.mutex.count) ++ mutex->u.mutex.owner = 0; ++ return 0; ++} ++ ++static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_mutex_args __user *user_args = argp; ++ struct winesync_mutex_args args; ++ struct winesync_obj *mutex; ++ __u32 prev_count; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ if (!args.owner) ++ return -EINVAL; ++ ++ mutex = get_obj_typed(dev, args.mutex, WINESYNC_TYPE_MUTEX); ++ if (!mutex) ++ return -EINVAL; ++ ++ if (atomic_read(&mutex->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&mutex->lock); ++ ++ prev_count = mutex->u.mutex.count; ++ ret = put_mutex_state(mutex, &args); ++ if (!ret) { ++ try_wake_all_obj(dev, mutex); ++ try_wake_any_mutex(mutex); ++ } ++ ++ spin_unlock(&mutex->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&mutex->lock); ++ ++ prev_count = mutex->u.mutex.count; ++ ret = put_mutex_state(mutex, &args); ++ if (!ret) ++ try_wake_any_mutex(mutex); ++ ++ spin_unlock(&mutex->lock); ++ } ++ ++ put_obj(mutex); ++ ++ if (!ret && put_user(prev_count, &user_args->count)) ++ ret = -EFAULT; ++ ++ return ret; ++} ++ ++static int winesync_read_sem(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_sem_args __user *user_args = argp; ++ struct winesync_sem_args args; ++ struct winesync_obj *sem; ++ __u32 id; ++ ++ if (get_user(id, &user_args->sem)) ++ return -EFAULT; ++ ++ sem = get_obj_typed(dev, id, WINESYNC_TYPE_SEM); ++ if (!sem) ++ return -EINVAL; ++ ++ args.sem = id; ++ spin_lock(&sem->lock); ++ args.count = sem->u.sem.count; ++ args.max = sem->u.sem.max; ++ spin_unlock(&sem->lock); ++ ++ put_obj(sem); ++ ++ if (copy_to_user(user_args, &args, sizeof(args))) ++ return -EFAULT; ++ return 0; ++} ++ ++static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_mutex_args __user *user_args = argp; ++ struct winesync_mutex_args args; ++ struct winesync_obj *mutex; ++ __u32 id; ++ int ret; ++ ++ if (get_user(id, &user_args->mutex)) ++ return -EFAULT; ++ ++ mutex = get_obj_typed(dev, id, WINESYNC_TYPE_MUTEX); ++ if (!mutex) ++ return -EINVAL; ++ ++ args.mutex = id; ++ spin_lock(&mutex->lock); ++ args.count = mutex->u.mutex.count; ++ args.owner = mutex->u.mutex.owner; ++ ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; ++ spin_unlock(&mutex->lock); ++ ++ put_obj(mutex); ++ ++ if (copy_to_user(user_args, &args, sizeof(args))) ++ return -EFAULT; ++ return ret; ++} ++ ++static int winesync_read_event(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ __u32 id; ++ ++ if (get_user(id, &user_args->event)) ++ return -EFAULT; ++ ++ event = get_obj_typed(dev, id, WINESYNC_TYPE_EVENT); ++ if (!event) ++ return -EINVAL; ++ ++ args.event = id; ++ spin_lock(&event->lock); ++ args.manual = event->u.event.manual; ++ args.signaled = event->u.event.signaled; ++ spin_unlock(&event->lock); ++ ++ put_obj(event); ++ ++ if (copy_to_user(user_args, &args, sizeof(args))) ++ return -EFAULT; ++ return 0; ++} ++ ++/* ++ * Actually change the mutex state to mark its owner as dead. ++ */ ++static void put_mutex_ownerdead_state(struct winesync_obj *mutex) ++{ ++ lockdep_assert_held(&mutex->lock); ++ ++ mutex->u.mutex.ownerdead = true; ++ mutex->u.mutex.owner = 0; ++ mutex->u.mutex.count = 0; ++} ++ ++static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_obj *obj; ++ unsigned long id; ++ __u32 owner; ++ ++ if (get_user(owner, (__u32 __user *)argp)) ++ return -EFAULT; ++ if (!owner) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ ++ xa_for_each(&dev->objects, id, obj) { ++ if (!kref_get_unless_zero(&obj->refcount)) ++ continue; ++ ++ if (obj->type != WINESYNC_TYPE_MUTEX) { ++ put_obj(obj); ++ continue; ++ } ++ ++ if (atomic_read(&obj->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&obj->lock); ++ ++ if (obj->u.mutex.owner == owner) { ++ put_mutex_ownerdead_state(obj); ++ try_wake_all_obj(dev, obj); ++ try_wake_any_mutex(obj); ++ } ++ ++ spin_unlock(&obj->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&obj->lock); ++ ++ if (obj->u.mutex.owner == owner) { ++ put_mutex_ownerdead_state(obj); ++ try_wake_any_mutex(obj); ++ } ++ ++ spin_unlock(&obj->lock); ++ } ++ ++ put_obj(obj); ++ } ++ ++ rcu_read_unlock(); ++ ++ return 0; ++} ++ ++static int winesync_set_event(struct winesync_device *dev, void __user *argp, ++ bool pulse) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ bool prev_state; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); ++ if (!event) ++ return -EINVAL; ++ ++ if (atomic_read(&event->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&event->lock); ++ ++ prev_state = event->u.event.signaled; ++ event->u.event.signaled = true; ++ try_wake_all_obj(dev, event); ++ try_wake_any_event(event); ++ if (pulse) ++ event->u.event.signaled = false; ++ ++ spin_unlock(&event->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&event->lock); ++ ++ prev_state = event->u.event.signaled; ++ event->u.event.signaled = true; ++ try_wake_any_event(event); ++ if (pulse) ++ event->u.event.signaled = false; ++ ++ spin_unlock(&event->lock); ++ } ++ ++ put_obj(event); ++ ++ if (put_user(prev_state, &user_args->signaled)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int winesync_reset_event(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ bool prev_state; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); ++ if (!event) ++ return -EINVAL; ++ ++ spin_lock(&event->lock); ++ ++ prev_state = event->u.event.signaled; ++ event->u.event.signaled = false; ++ ++ spin_unlock(&event->lock); ++ ++ put_obj(event); ++ ++ if (put_user(prev_state, &user_args->signaled)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) ++{ ++ int ret = 0; ++ ++ do { ++ if (signal_pending(current)) { ++ ret = -ERESTARTSYS; ++ break; ++ } ++ ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (atomic_read(&q->signaled) != -1) { ++ ret = 0; ++ break; ++ } ++ ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); ++ } while (ret < 0); ++ __set_current_state(TASK_RUNNING); ++ ++ return ret; ++} ++ ++/* ++ * Allocate and initialize the winesync_q structure, but do not queue us yet. ++ * Also, calculate the relative timeout. ++ */ ++static int setup_wait(struct winesync_device *dev, ++ const struct winesync_wait_args *args, bool all, ++ ktime_t *ret_timeout, struct winesync_q **ret_q) ++{ ++ const __u32 count = args->count; ++ struct winesync_q *q; ++ ktime_t timeout = 0; ++ __u32 total_count; ++ __u32 *ids; ++ __u32 i, j; ++ ++ if (!args->owner) ++ return -EINVAL; ++ ++ if (args->timeout) { ++ struct timespec64 to; ++ ++ if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) ++ return -EFAULT; ++ if (!timespec64_valid(&to)) ++ return -EINVAL; ++ ++ timeout = timespec64_to_ns(&to); ++ } ++ ++ total_count = count; ++ if (args->alert) ++ total_count++; ++ ++ ids = kmalloc_array(total_count, sizeof(*ids), GFP_KERNEL); ++ if (!ids) ++ return -ENOMEM; ++ if (copy_from_user(ids, u64_to_user_ptr(args->objs), ++ array_size(count, sizeof(*ids)))) { ++ kfree(ids); ++ return -EFAULT; ++ } ++ if (args->alert) ++ ids[count] = args->alert; ++ ++ q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); ++ if (!q) { ++ kfree(ids); ++ return -ENOMEM; ++ } ++ q->task = current; ++ q->owner = args->owner; ++ atomic_set(&q->signaled, -1); ++ q->all = all; ++ q->ownerdead = false; ++ q->count = count; ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = get_obj(dev, ids[i]); ++ ++ if (!obj) ++ goto err; ++ ++ if (all) { ++ /* Check that the objects are all distinct. */ ++ for (j = 0; j < i; j++) { ++ if (obj == q->entries[j].obj) { ++ put_obj(obj); ++ goto err; ++ } ++ } ++ } ++ ++ entry->obj = obj; ++ entry->q = q; ++ entry->index = i; ++ } ++ ++ kfree(ids); ++ ++ *ret_q = q; ++ *ret_timeout = timeout; ++ return 0; ++ ++err: ++ for (j = 0; j < i; j++) ++ put_obj(q->entries[j].obj); ++ kfree(ids); ++ kfree(q); ++ return -EINVAL; ++} ++ ++static void try_wake_any_obj(struct winesync_obj *obj) ++{ ++ switch (obj->type) { ++ case WINESYNC_TYPE_SEM: ++ try_wake_any_sem(obj); ++ break; ++ case WINESYNC_TYPE_MUTEX: ++ try_wake_any_mutex(obj); ++ break; ++ case WINESYNC_TYPE_EVENT: ++ try_wake_any_event(obj); ++ break; ++ } ++} ++ ++static int winesync_wait_any(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_wait_args args; ++ struct winesync_q *q; ++ __u32 i, total_count; ++ ktime_t timeout; ++ int signaled; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ ret = setup_wait(dev, &args, false, &timeout, &q); ++ if (ret < 0) ++ return ret; ++ ++ total_count = args.count; ++ if (args.alert) ++ total_count++; ++ ++ /* queue ourselves */ ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_add_tail(&entry->node, &obj->any_waiters); ++ spin_unlock(&obj->lock); ++ } ++ ++ /* ++ * Check if we are already signaled. ++ * ++ * Note that the API requires that normal objects are checked before ++ * the alert event. Hence we queue the alert event last, and check ++ * objects in order. ++ */ ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_obj *obj = q->entries[i].obj; ++ ++ if (atomic_read(&q->signaled) != -1) ++ break; ++ ++ spin_lock(&obj->lock); ++ try_wake_any_obj(obj); ++ spin_unlock(&obj->lock); ++ } ++ ++ /* sleep */ ++ ++ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); ++ ++ /* and finally, unqueue */ ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_del(&entry->node); ++ spin_unlock(&obj->lock); ++ ++ put_obj(obj); ++ } ++ ++ signaled = atomic_read(&q->signaled); ++ if (signaled != -1) { ++ struct winesync_wait_args __user *user_args = argp; ++ ++ /* even if we caught a signal, we need to communicate success */ ++ ret = q->ownerdead ? -EOWNERDEAD : 0; ++ ++ if (put_user(signaled, &user_args->index)) ++ ret = -EFAULT; ++ } else if (!ret) { ++ ret = -ETIMEDOUT; ++ } ++ ++ kfree(q); ++ return ret; ++} ++ ++static int winesync_wait_all(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_wait_args args; ++ struct winesync_q *q; ++ ktime_t timeout; ++ int signaled; ++ __u32 i; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ ret = setup_wait(dev, &args, true, &timeout, &q); ++ if (ret < 0) ++ return ret; ++ ++ /* queue ourselves */ ++ ++ spin_lock(&dev->wait_all_lock); ++ ++ for (i = 0; i < args.count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ atomic_inc(&obj->all_hint); ++ ++ /* ++ * obj->all_waiters is protected by dev->wait_all_lock rather ++ * than obj->lock, so there is no need to acquire it here. ++ */ ++ list_add_tail(&entry->node, &obj->all_waiters); ++ } ++ if (args.alert) { ++ struct winesync_q_entry *entry = &q->entries[args.count]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_add_tail(&entry->node, &obj->any_waiters); ++ spin_unlock(&obj->lock); ++ } ++ ++ /* check if we are already signaled */ ++ ++ try_wake_all(dev, q, NULL); ++ ++ spin_unlock(&dev->wait_all_lock); ++ ++ /* ++ * Check if the alert event is signaled, making sure to do so only ++ * after checking if the other objects are signaled. ++ */ ++ ++ if (args.alert) { ++ struct winesync_obj *obj = q->entries[args.count].obj; ++ ++ if (atomic_read(&q->signaled) == -1) { ++ spin_lock(&obj->lock); ++ try_wake_any_obj(obj); ++ spin_unlock(&obj->lock); ++ } ++ } ++ ++ /* sleep */ ++ ++ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); ++ ++ /* and finally, unqueue */ ++ ++ spin_lock(&dev->wait_all_lock); ++ ++ for (i = 0; i < args.count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ /* ++ * obj->all_waiters is protected by dev->wait_all_lock rather ++ * than obj->lock, so there is no need to acquire it here. ++ */ ++ list_del(&entry->node); ++ ++ atomic_dec(&obj->all_hint); ++ ++ put_obj(obj); ++ } ++ if (args.alert) { ++ struct winesync_q_entry *entry = &q->entries[args.count]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_del(&entry->node); ++ spin_unlock(&obj->lock); ++ ++ put_obj(obj); ++ } ++ ++ spin_unlock(&dev->wait_all_lock); ++ ++ signaled = atomic_read(&q->signaled); ++ if (signaled != -1) { ++ struct winesync_wait_args __user *user_args = argp; ++ ++ /* even if we caught a signal, we need to communicate success */ ++ ret = q->ownerdead ? -EOWNERDEAD : 0; ++ ++ if (put_user(signaled, &user_args->index)) ++ ret = -EFAULT; ++ } else if (!ret) { ++ ret = -ETIMEDOUT; ++ } ++ ++ kfree(q); ++ return ret; ++} ++ ++static long winesync_char_ioctl(struct file *file, unsigned int cmd, ++ unsigned long parm) ++{ ++ struct winesync_device *dev = file->private_data; ++ void __user *argp = (void __user *)parm; ++ ++ switch (cmd) { ++ case WINESYNC_IOC_CREATE_EVENT: ++ return winesync_create_event(dev, argp); ++ case WINESYNC_IOC_CREATE_MUTEX: ++ return winesync_create_mutex(dev, argp); ++ case WINESYNC_IOC_CREATE_SEM: ++ return winesync_create_sem(dev, argp); ++ case WINESYNC_IOC_DELETE: ++ return winesync_delete(dev, argp); ++ case WINESYNC_IOC_KILL_OWNER: ++ return winesync_kill_owner(dev, argp); ++ case WINESYNC_IOC_PULSE_EVENT: ++ return winesync_set_event(dev, argp, true); ++ case WINESYNC_IOC_PUT_MUTEX: ++ return winesync_put_mutex(dev, argp); ++ case WINESYNC_IOC_PUT_SEM: ++ return winesync_put_sem(dev, argp); ++ case WINESYNC_IOC_READ_EVENT: ++ return winesync_read_event(dev, argp); ++ case WINESYNC_IOC_READ_MUTEX: ++ return winesync_read_mutex(dev, argp); ++ case WINESYNC_IOC_READ_SEM: ++ return winesync_read_sem(dev, argp); ++ case WINESYNC_IOC_RESET_EVENT: ++ return winesync_reset_event(dev, argp); ++ case WINESYNC_IOC_SET_EVENT: ++ return winesync_set_event(dev, argp, false); ++ case WINESYNC_IOC_WAIT_ALL: ++ return winesync_wait_all(dev, argp); ++ case WINESYNC_IOC_WAIT_ANY: ++ return winesync_wait_any(dev, argp); ++ default: ++ return -ENOSYS; ++ } ++} ++ ++static const struct file_operations winesync_fops = { ++ .owner = THIS_MODULE, ++ .open = winesync_char_open, ++ .release = winesync_char_release, ++ .unlocked_ioctl = winesync_char_ioctl, ++ .compat_ioctl = winesync_char_ioctl, ++ .llseek = no_llseek, ++}; ++ ++static struct miscdevice winesync_misc = { ++ .minor = WINESYNC_MINOR, ++ .name = WINESYNC_NAME, ++ .fops = &winesync_fops, ++}; ++ ++static int __init winesync_init(void) ++{ ++ return misc_register(&winesync_misc); ++} ++ ++static void __exit winesync_exit(void) ++{ ++ misc_deregister(&winesync_misc); ++} ++ ++module_init(winesync_init); ++module_exit(winesync_exit); ++ ++MODULE_AUTHOR("Zebediah Figura"); ++MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); ++MODULE_LICENSE("GPL"); ++MODULE_ALIAS("devname:" WINESYNC_NAME); ++MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); +diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h +index c0fea6ca5076..36fc5d5315a4 100644 +--- a/include/linux/miscdevice.h ++++ b/include/linux/miscdevice.h +@@ -71,6 +71,7 @@ + #define USERIO_MINOR 240 + #define VHOST_VSOCK_MINOR 241 + #define RFKILL_MINOR 242 ++#define WINESYNC_MINOR 243 + #define MISC_DYNAMIC_MINOR 255 + + struct device; +diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h +index 71a5df8d2689..d375ab21cbf8 100644 +--- a/include/uapi/linux/futex.h ++++ b/include/uapi/linux/futex.h +@@ -22,6 +22,7 @@ + #define FUTEX_WAIT_REQUEUE_PI 11 + #define FUTEX_CMP_REQUEUE_PI 12 + #define FUTEX_LOCK_PI2 13 ++#define FUTEX_WAIT_MULTIPLE 31 + + #define FUTEX_PRIVATE_FLAG 128 + #define FUTEX_CLOCK_REALTIME 256 +@@ -68,6 +69,18 @@ struct futex_waitv { + __u32 __reserved; + }; + ++/** ++ * struct futex_wait_block - Block of futexes to be waited for ++ * @uaddr: User address of the futex ++ * @val: Futex value expected by userspace ++ * @bitset: Bitset for the optional bitmasked wakeup ++ */ ++struct futex_wait_block { ++ __u32 __user *uaddr; ++ __u32 val; ++ __u32 bitset; ++}; ++ + /* + * Support for robust futexes: the kernel cleans up held futexes at + * thread exit time. +diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h +new file mode 100644 +index 000000000000..5b4e369f7469 +--- /dev/null ++++ b/include/uapi/linux/winesync.h +@@ -0,0 +1,71 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* ++ * Kernel support for Wine synchronization primitives ++ * ++ * Copyright (C) 2021 Zebediah Figura ++ */ ++ ++#ifndef __LINUX_WINESYNC_H ++#define __LINUX_WINESYNC_H ++ ++#include ++ ++struct winesync_sem_args { ++ __u32 sem; ++ __u32 count; ++ __u32 max; ++}; ++ ++struct winesync_mutex_args { ++ __u32 mutex; ++ __u32 owner; ++ __u32 count; ++}; ++ ++struct winesync_event_args { ++ __u32 event; ++ __u32 manual; ++ __u32 signaled; ++}; ++ ++struct winesync_wait_args { ++ __u64 timeout; ++ __u64 objs; ++ __u32 count; ++ __u32 owner; ++ __u32 index; ++ __u32 alert; ++}; ++ ++#define WINESYNC_IOC_BASE 0xf7 ++ ++#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ ++ struct winesync_sem_args) ++#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 1, __u32) ++#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 2, \ ++ struct winesync_sem_args) ++#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 3, \ ++ struct winesync_wait_args) ++#define WINESYNC_IOC_WAIT_ALL _IOWR(WINESYNC_IOC_BASE, 4, \ ++ struct winesync_wait_args) ++#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 5, \ ++ struct winesync_mutex_args) ++#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ ++ struct winesync_mutex_args) ++#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 7, __u32) ++#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 8, \ ++ struct winesync_sem_args) ++#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 9, \ ++ struct winesync_mutex_args) ++#define WINESYNC_IOC_CREATE_EVENT _IOWR(WINESYNC_IOC_BASE, 10, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_SET_EVENT _IOWR(WINESYNC_IOC_BASE, 11, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_RESET_EVENT _IOWR(WINESYNC_IOC_BASE, 12, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_PULSE_EVENT _IOWR(WINESYNC_IOC_BASE, 13, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_READ_EVENT _IOWR(WINESYNC_IOC_BASE, 14, \ ++ struct winesync_event_args) ++ ++#endif +diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c +index 086a22d1adb7..c6f5f1e84e09 100644 +--- a/kernel/futex/syscalls.c ++++ b/kernel/futex/syscalls.c +@@ -142,6 +142,7 @@ static __always_inline bool futex_cmd_has_timeout(u32 cmd) + case FUTEX_LOCK_PI2: + case FUTEX_WAIT_BITSET: + case FUTEX_WAIT_REQUEUE_PI: ++ case FUTEX_WAIT_MULTIPLE: + return true; + } + return false; +@@ -154,13 +155,79 @@ futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t) + return -EINVAL; + + *t = timespec64_to_ktime(*ts); +- if (cmd == FUTEX_WAIT) ++ if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE) + *t = ktime_add_safe(ktime_get(), *t); + else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) + *t = timens_ktime_to_host(CLOCK_MONOTONIC, *t); + return 0; + } + ++/** ++ * futex_read_wait_block - Read an array of futex_wait_block from userspace ++ * @uaddr: Userspace address of the block ++ * @count: Number of blocks to be read ++ * ++ * This function creates and allocate an array of futex_q (we zero it to ++ * initialize the fields) and then, for each futex_wait_block element from ++ * userspace, fill a futex_q element with proper values. ++ */ ++inline struct futex_vector *futex_read_wait_block(u32 __user *uaddr, u32 count) ++{ ++ unsigned int i; ++ struct futex_vector *futexv; ++ struct futex_wait_block fwb; ++ struct futex_wait_block __user *entry = ++ (struct futex_wait_block __user *)uaddr; ++ ++ if (!count || count > FUTEX_WAITV_MAX) ++ return ERR_PTR(-EINVAL); ++ ++ futexv = kcalloc(count, sizeof(*futexv), GFP_KERNEL); ++ if (!futexv) ++ return ERR_PTR(-ENOMEM); ++ ++ for (i = 0; i < count; i++) { ++ if (copy_from_user(&fwb, &entry[i], sizeof(fwb))) { ++ kfree(futexv); ++ return ERR_PTR(-EFAULT); ++ } ++ ++ futexv[i].w.flags = FUTEX_32; ++ futexv[i].w.val = fwb.val; ++ futexv[i].w.uaddr = (uintptr_t) (fwb.uaddr); ++ futexv[i].q = futex_q_init; ++ } ++ ++ return futexv; ++} ++ ++int futex_wait_multiple(struct futex_vector *vs, unsigned int count, ++ struct hrtimer_sleeper *to); ++ ++int futex_opcode_31(ktime_t *abs_time, u32 __user *uaddr, int count) ++{ ++ int ret; ++ struct futex_vector *vs; ++ struct hrtimer_sleeper *to = NULL, timeout; ++ ++ to = futex_setup_timer(abs_time, &timeout, 0, 0); ++ ++ vs = futex_read_wait_block(uaddr, count); ++ ++ if (IS_ERR(vs)) ++ return PTR_ERR(vs); ++ ++ ret = futex_wait_multiple(vs, count, abs_time ? to : NULL); ++ kfree(vs); ++ ++ if (to) { ++ hrtimer_cancel(&to->timer); ++ destroy_hrtimer_on_stack(&to->timer); ++ } ++ ++ return ret; ++} ++ + SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, + const struct __kernel_timespec __user *, utime, + u32 __user *, uaddr2, u32, val3) +@@ -180,6 +247,9 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, + tp = &t; + } + ++ if (cmd == FUTEX_WAIT_MULTIPLE) ++ return futex_opcode_31(tp, uaddr, val); ++ + return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); + } + +@@ -370,6 +440,9 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, + tp = &t; + } + ++ if (cmd == FUTEX_WAIT_MULTIPLE) ++ return futex_opcode_31(tp, uaddr, val); ++ + return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); + } + #endif /* CONFIG_COMPAT_32BIT_TIME */ +diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile +index 1fc89b8ef433..c7d3d9f5e0a0 100644 +--- a/tools/testing/selftests/Makefile ++++ b/tools/testing/selftests/Makefile +@@ -14,6 +14,7 @@ TARGETS += drivers/dma-buf + TARGETS += drivers/s390x/uvdevice + TARGETS += drivers/net/bonding + TARGETS += drivers/net/team ++TARGETS += drivers/winesync + TARGETS += efivarfs + TARGETS += exec + TARGETS += filesystems +diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile +new file mode 100644 +index 000000000000..43b39fdeea10 +--- /dev/null ++++ b/tools/testing/selftests/drivers/winesync/Makefile +@@ -0,0 +1,8 @@ ++# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only ++TEST_GEN_PROGS := winesync ++ ++top_srcdir =../../../../.. ++CFLAGS += -I$(top_srcdir)/usr/include ++LDLIBS += -lpthread ++ ++include ../../lib.mk +diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config +new file mode 100644 +index 000000000000..60539c826d06 +--- /dev/null ++++ b/tools/testing/selftests/drivers/winesync/config +@@ -0,0 +1 @@ ++CONFIG_WINESYNC=y +diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c +new file mode 100644 +index 000000000000..169e922484b0 +--- /dev/null ++++ b/tools/testing/selftests/drivers/winesync/winesync.c +@@ -0,0 +1,1479 @@ ++// SPDX-License-Identifier: GPL-2.0-or-later ++/* ++ * Various unit tests for the "winesync" synchronization primitive driver. ++ * ++ * Copyright (C) 2021 Zebediah Figura ++ */ ++ ++#define _GNU_SOURCE ++#include ++#include ++#include ++#include ++#include ++#include ++#include "../../kselftest_harness.h" ++ ++static int read_sem_state(int fd, __u32 sem, __u32 *count, __u32 *max) ++{ ++ struct winesync_sem_args args; ++ int ret; ++ ++ args.sem = sem; ++ args.count = 0xdeadbeef; ++ args.max = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &args); ++ *count = args.count; ++ *max = args.max; ++ return ret; ++} ++ ++#define check_sem_state(fd, sem, count, max) \ ++ ({ \ ++ __u32 __count, __max; \ ++ int ret = read_sem_state((fd), (sem), &__count, &__max); \ ++ EXPECT_EQ(0, ret); \ ++ EXPECT_EQ((count), __count); \ ++ EXPECT_EQ((max), __max); \ ++ }) ++ ++static int put_sem(int fd, __u32 sem, __u32 *count) ++{ ++ struct winesync_sem_args args; ++ int ret; ++ ++ args.sem = sem; ++ args.count = *count; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &args); ++ *count = args.count; ++ return ret; ++} ++ ++static int read_mutex_state(int fd, __u32 mutex, __u32 *count, __u32 *owner) ++{ ++ struct winesync_mutex_args args; ++ int ret; ++ ++ args.mutex = mutex; ++ args.count = 0xdeadbeef; ++ args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &args); ++ *count = args.count; ++ *owner = args.owner; ++ return ret; ++} ++ ++#define check_mutex_state(fd, mutex, count, owner) \ ++ ({ \ ++ __u32 __count, __owner; \ ++ int ret = read_mutex_state((fd), (mutex), &__count, &__owner); \ ++ EXPECT_EQ(0, ret); \ ++ EXPECT_EQ((count), __count); \ ++ EXPECT_EQ((owner), __owner); \ ++ }) ++ ++static int put_mutex(int fd, __u32 mutex, __u32 owner, __u32 *count) ++{ ++ struct winesync_mutex_args args; ++ int ret; ++ ++ args.mutex = mutex; ++ args.owner = owner; ++ args.count = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &args); ++ *count = args.count; ++ return ret; ++} ++ ++static int read_event_state(int fd, __u32 event, __u32 *signaled, __u32 *manual) ++{ ++ struct winesync_event_args args; ++ int ret; ++ ++ args.event = event; ++ args.signaled = 0xdeadbeef; ++ args.manual = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &args); ++ *signaled = args.signaled; ++ *manual = args.manual; ++ return ret; ++} ++ ++#define check_event_state(fd, event, signaled, manual) \ ++ ({ \ ++ __u32 __signaled, __manual; \ ++ int ret = read_event_state((fd), (event), \ ++ &__signaled, &__manual); \ ++ EXPECT_EQ(0, ret); \ ++ EXPECT_EQ((signaled), __signaled); \ ++ EXPECT_EQ((manual), __manual); \ ++ }) ++ ++static int wait_objs(int fd, unsigned long request, __u32 count, ++ const __u32 *objs, __u32 owner, __u32 alert, __u32 *index) ++{ ++ struct winesync_wait_args args = {0}; ++ struct timespec timeout; ++ int ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ args.timeout = (uintptr_t)&timeout; ++ args.count = count; ++ args.objs = (uintptr_t)objs; ++ args.owner = owner; ++ args.index = 0xdeadbeef; ++ args.alert = alert; ++ ret = ioctl(fd, request, &args); ++ *index = args.index; ++ return ret; ++} ++ ++static int wait_any(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, ++ count, objs, owner, 0, index); ++} ++ ++static int wait_all(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, ++ count, objs, owner, 0, index); ++} ++ ++static int wait_any_alert(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 alert, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, ++ count, objs, owner, alert, index); ++} ++ ++static int wait_all_alert(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 alert, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, ++ count, objs, owner, alert, index); ++} ++ ++TEST(semaphore_state) ++{ ++ struct winesync_sem_args sem_args; ++ struct timespec timeout; ++ __u32 sem, count, index; ++ int fd, ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 3; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ sem_args.count = 2; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ check_sem_state(fd, sem, 2, 2); ++ ++ count = 0; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ check_sem_state(fd, sem, 2, 2); ++ ++ count = 1; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOVERFLOW, errno); ++ check_sem_state(fd, sem, 2, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem, 1, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem, 0, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ count = 3; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOVERFLOW, errno); ++ check_sem_state(fd, sem, 0, 2); ++ ++ count = 2; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, count); ++ check_sem_state(fd, sem, 2, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ ++ count = 1; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, count); ++ check_sem_state(fd, sem, 1, 2); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(mutex_state) ++{ ++ struct winesync_mutex_args mutex_args; ++ __u32 mutex, owner, count, index; ++ struct timespec timeout; ++ int fd, ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 0; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 2; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ mutex = mutex_args.mutex; ++ check_mutex_state(fd, mutex, 2, 123); ++ ++ ret = put_mutex(fd, mutex, 0, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = put_mutex(fd, mutex, 456, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EPERM, errno); ++ check_mutex_state(fd, mutex, 2, 123); ++ ++ ret = put_mutex(fd, mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ ret = put_mutex(fd, mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, count); ++ check_mutex_state(fd, mutex, 0, 0); ++ ++ ret = put_mutex(fd, mutex, 123, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EPERM, errno); ++ ++ ret = wait_any(fd, 1, &mutex, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 456); ++ ++ ret = wait_any(fd, 1, &mutex, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 2, 456); ++ ++ ret = put_mutex(fd, mutex, 456, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ check_mutex_state(fd, mutex, 1, 456); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ owner = 0; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ check_mutex_state(fd, mutex, 1, 456); ++ ++ owner = 456; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.count = 0xdeadbeef; ++ mutex_args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, mutex_args.count); ++ EXPECT_EQ(0, mutex_args.owner); ++ ++ mutex_args.count = 0xdeadbeef; ++ mutex_args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, mutex_args.count); ++ EXPECT_EQ(0, mutex_args.owner); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.count = 0xdeadbeef; ++ mutex_args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, mutex_args.count); ++ EXPECT_EQ(0, mutex_args.owner); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 0; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ mutex = mutex_args.mutex; ++ check_mutex_state(fd, mutex, 0, 0); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(manual_event_state) ++{ ++ struct winesync_event_args event_args; ++ __u32 index; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ event_args.manual = 1; ++ event_args.signaled = 0; ++ event_args.event = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, event_args.event); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(auto_event_state) ++{ ++ struct winesync_event_args event_args; ++ __u32 index; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ event_args.manual = 0; ++ event_args.signaled = 1; ++ event_args.event = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, event_args.event); ++ ++ check_event_state(fd, event_args.event, 1, 0); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 0); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(test_wait_any) ++{ ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ __u32 objs[2], owner, index; ++ struct timespec timeout; ++ int fd, ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 2; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 0; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++ ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(1, index); ++ ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ ++ /* test waiting on the same object twice */ ++ sem_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ objs[0] = objs[1] = sem_args.sem; ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, wait_args.index); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ ++ ret = wait_any(fd, 0, NULL, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(test_wait_all) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ __u32 objs[2], owner, index; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 2; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 0; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ event_args.manual = true; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ ret = wait_all(fd, 2, objs, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++ ++ sem_args.count = 3; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 2, 3); ++ check_mutex_state(fd, mutex_args.mutex, 3, 123); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = event_args.event; ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ /* test waiting on the same object twice */ ++ objs[0] = objs[1] = sem_args.sem; ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(invalid_objects) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ __u32 objs[2] = {0}; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ sem_args.max = 1; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.mutex = sem_args.sem; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ event_args.event = sem_args.sem; ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = sem_args.sem + 1; ++ wait_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ objs[0] = sem_args.sem + 1; ++ objs[1] = sem_args.sem; ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ ++ sem_args.sem = mutex_args.mutex; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++struct wake_args ++{ ++ int fd; ++ __u32 obj; ++}; ++ ++struct wait_args ++{ ++ int fd; ++ unsigned long request; ++ struct winesync_wait_args *args; ++ int ret; ++ int err; ++}; ++ ++static void *wait_thread(void *arg) ++{ ++ struct wait_args *args = arg; ++ ++ args->ret = ioctl(args->fd, args->request, args->args); ++ args->err = errno; ++ return NULL; ++} ++ ++static void get_abs_timeout(struct timespec *timeout, clockid_t clock, ++ unsigned int ms) ++{ ++ clock_gettime(clock, timeout); ++ timeout->tv_nsec += ms * 1000000; ++ timeout->tv_sec += (timeout->tv_nsec / 1000000000); ++ timeout->tv_nsec %= 1000000000; ++} ++ ++static int wait_for_thread(pthread_t thread, unsigned int ms) ++{ ++ struct timespec timeout; ++ get_abs_timeout(&timeout, CLOCK_REALTIME, ms); ++ return pthread_timedjoin_np(thread, NULL, &timeout); ++} ++ ++TEST(wake_any) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ __u32 objs[2], count, index; ++ struct timespec timeout; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 0; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 1; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ ++ /* test waking the semaphore */ ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 2; ++ wait_args.owner = 456; ++ wait_args.index = 0xdeadbeef; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ANY; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(0, wait_args.index); ++ ++ /* test waking the mutex */ ++ ++ /* first grab it again for owner 123 */ ++ ret = wait_any(fd, 1, &mutex_args.mutex, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.owner = 456; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = put_mutex(fd, mutex_args.mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ ++ ret = pthread_tryjoin_np(thread, NULL); ++ EXPECT_EQ(EBUSY, ret); ++ ++ ret = put_mutex(fd, mutex_args.mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, mutex_args.count); ++ check_mutex_state(fd, mutex_args.mutex, 1, 456); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ /* test waking events */ ++ ++ event_args.manual = false; ++ event_args.signaled = false; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[1] = event_args.event; ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ event_args.manual = true; ++ event_args.signaled = false; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[1] = event_args.event; ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ /* delete an object while it's being waited on */ ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); ++ wait_args.owner = 123; ++ objs[1] = mutex_args.mutex; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 200); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(-1, thread_args.ret); ++ EXPECT_EQ(ETIMEDOUT, thread_args.err); ++ ++ close(fd); ++} ++ ++TEST(wake_all) ++{ ++ struct winesync_event_args manual_event_args = {0}; ++ struct winesync_event_args auto_event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ __u32 objs[4], count, index; ++ struct timespec timeout; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 0; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 1; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ manual_event_args.manual = true; ++ manual_event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &manual_event_args); ++ EXPECT_EQ(0, ret); ++ ++ auto_event_args.manual = false; ++ auto_event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &auto_event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ objs[2] = manual_event_args.event; ++ objs[3] = auto_event_args.event; ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 4; ++ wait_args.owner = 456; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ALL; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ ret = pthread_tryjoin_np(thread, NULL); ++ EXPECT_EQ(EBUSY, ret); ++ ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ ++ ret = wait_any(fd, 1, &sem_args.sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = put_mutex(fd, mutex_args.mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, count); ++ ++ ret = pthread_tryjoin_np(thread, NULL); ++ EXPECT_EQ(EBUSY, ret); ++ ++ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &manual_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, manual_event_args.signaled); ++ ++ sem_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ check_sem_state(fd, sem_args.sem, 2, 3); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &auto_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, auto_event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &manual_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, manual_event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &auto_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, auto_event_args.signaled); ++ ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 456); ++ check_event_state(fd, manual_event_args.event, 1, 1); ++ check_event_state(fd, auto_event_args.event, 0, 0); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ ++ /* delete an object while it's being waited on */ ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); ++ wait_args.owner = 123; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &manual_event_args.event); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &auto_event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 200); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(-1, thread_args.ret); ++ EXPECT_EQ(ETIMEDOUT, thread_args.err); ++ ++ close(fd); ++} ++ ++TEST(alert_any) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ struct timespec timeout; ++ __u32 objs[2], index; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 0; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[0] = sem_args.sem; ++ ++ sem_args.count = 1; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[1] = sem_args.sem; ++ ++ event_args.manual = true; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ /* test wakeup via alert */ ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 2; ++ wait_args.owner = 123; ++ wait_args.index = 0xdeadbeef; ++ wait_args.alert = event_args.event; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ANY; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(2, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ /* test with an auto-reset event */ ++ ++ event_args.manual = false; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ sem_args.sem = objs[0]; ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(alert_all) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ struct timespec timeout; ++ __u32 objs[2], index; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 2; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[0] = sem_args.sem; ++ ++ sem_args.count = 1; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[1] = sem_args.sem; ++ ++ event_args.manual = true; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ /* test wakeup via alert */ ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 2; ++ wait_args.owner = 123; ++ wait_args.index = 0xdeadbeef; ++ wait_args.alert = event_args.event; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ALL; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(2, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ /* test with an auto-reset event */ ++ ++ event_args.manual = false; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ sem_args.sem = objs[1]; ++ sem_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST_HARNESS_MAIN +-- +2.38.0 + +From b5f6f1f7ab1cd65a7e8a40ac3eae83835402e8e8 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Fri, 5 Aug 2022 19:33:47 +0200 +Subject: [PATCH 04/17] Introducing-OpenVPN-Data-Channel-Offload + +Signed-off-by: Peter Jung +--- + MAINTAINERS | 8 + + drivers/net/Kconfig | 19 + + drivers/net/Makefile | 1 + + drivers/net/ovpn-dco/Makefile | 21 + + drivers/net/ovpn-dco/addr.h | 41 + + drivers/net/ovpn-dco/bind.c | 62 ++ + drivers/net/ovpn-dco/bind.h | 67 ++ + drivers/net/ovpn-dco/crypto.c | 154 ++++ + drivers/net/ovpn-dco/crypto.h | 144 ++++ + drivers/net/ovpn-dco/crypto_aead.c | 367 +++++++++ + drivers/net/ovpn-dco/crypto_aead.h | 27 + + drivers/net/ovpn-dco/main.c | 271 +++++++ + drivers/net/ovpn-dco/main.h | 32 + + drivers/net/ovpn-dco/netlink.c | 1143 ++++++++++++++++++++++++++++ + drivers/net/ovpn-dco/netlink.h | 22 + + drivers/net/ovpn-dco/ovpn.c | 600 +++++++++++++++ + drivers/net/ovpn-dco/ovpn.h | 43 ++ + drivers/net/ovpn-dco/ovpnstruct.h | 59 ++ + drivers/net/ovpn-dco/peer.c | 906 ++++++++++++++++++++++ + drivers/net/ovpn-dco/peer.h | 168 ++++ + drivers/net/ovpn-dco/pktid.c | 127 ++++ + drivers/net/ovpn-dco/pktid.h | 116 +++ + drivers/net/ovpn-dco/proto.h | 101 +++ + drivers/net/ovpn-dco/rcu.h | 21 + + drivers/net/ovpn-dco/skb.h | 54 ++ + drivers/net/ovpn-dco/sock.c | 134 ++++ + drivers/net/ovpn-dco/sock.h | 54 ++ + drivers/net/ovpn-dco/stats.c | 20 + + drivers/net/ovpn-dco/stats.h | 67 ++ + drivers/net/ovpn-dco/tcp.c | 326 ++++++++ + drivers/net/ovpn-dco/tcp.h | 38 + + drivers/net/ovpn-dco/udp.c | 343 +++++++++ + drivers/net/ovpn-dco/udp.h | 25 + + include/net/netlink.h | 1 + + include/uapi/linux/ovpn_dco.h | 265 +++++++ + include/uapi/linux/udp.h | 1 + + 36 files changed, 5848 insertions(+) + create mode 100644 drivers/net/ovpn-dco/Makefile + create mode 100644 drivers/net/ovpn-dco/addr.h + create mode 100644 drivers/net/ovpn-dco/bind.c + create mode 100644 drivers/net/ovpn-dco/bind.h + create mode 100644 drivers/net/ovpn-dco/crypto.c + create mode 100644 drivers/net/ovpn-dco/crypto.h + create mode 100644 drivers/net/ovpn-dco/crypto_aead.c + create mode 100644 drivers/net/ovpn-dco/crypto_aead.h + create mode 100644 drivers/net/ovpn-dco/main.c + create mode 100644 drivers/net/ovpn-dco/main.h + create mode 100644 drivers/net/ovpn-dco/netlink.c + create mode 100644 drivers/net/ovpn-dco/netlink.h + create mode 100644 drivers/net/ovpn-dco/ovpn.c + create mode 100644 drivers/net/ovpn-dco/ovpn.h + create mode 100644 drivers/net/ovpn-dco/ovpnstruct.h + create mode 100644 drivers/net/ovpn-dco/peer.c + create mode 100644 drivers/net/ovpn-dco/peer.h + create mode 100644 drivers/net/ovpn-dco/pktid.c + create mode 100644 drivers/net/ovpn-dco/pktid.h + create mode 100644 drivers/net/ovpn-dco/proto.h + create mode 100644 drivers/net/ovpn-dco/rcu.h + create mode 100644 drivers/net/ovpn-dco/skb.h + create mode 100644 drivers/net/ovpn-dco/sock.c + create mode 100644 drivers/net/ovpn-dco/sock.h + create mode 100644 drivers/net/ovpn-dco/stats.c + create mode 100644 drivers/net/ovpn-dco/stats.h + create mode 100644 drivers/net/ovpn-dco/tcp.c + create mode 100644 drivers/net/ovpn-dco/tcp.h + create mode 100644 drivers/net/ovpn-dco/udp.c + create mode 100644 drivers/net/ovpn-dco/udp.h + create mode 100644 include/uapi/linux/ovpn_dco.h + +diff --git a/MAINTAINERS b/MAINTAINERS +index ff31beb17835..594e31ec15cb 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -15319,6 +15319,14 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git + F: Documentation/filesystems/overlayfs.rst + F: fs/overlayfs/ + ++OVPN-DCO NETWORK DRIVER ++M: Antonio Quartulli ++L: openvpn-devel@lists.sourceforge.net (moderated for non-subscribers) ++L: netdev@vger.kernel.org ++S: Maintained ++F: drivers/net/ovpn-dco/ ++F: include/uapi/linux/ovpn_dco.h ++ + P54 WIRELESS DRIVER + M: Christian Lamparter + L: linux-wireless@vger.kernel.org +diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig +index 94c889802566..349866bd4448 100644 +--- a/drivers/net/Kconfig ++++ b/drivers/net/Kconfig +@@ -116,6 +116,25 @@ config WIREGUARD_DEBUG + + Say N here unless you know what you're doing. + ++config OVPN_DCO ++ tristate "OpenVPN data channel offload" ++ depends on NET && INET ++ select NET_UDP_TUNNEL ++ select DST_CACHE ++ select CRYPTO ++ select CRYPTO_AES ++ select CRYPTO_GCM ++ select CRYPTO_CHACHA20POLY1305 ++ help ++ This module enhances the performance of an OpenVPN connection by ++ allowing the user to offload the data channel processing to ++ kernelspace. ++ Connection handshake, parameters negotiation and other non-data ++ related mechanisms are still performed in userspace. ++ ++ The OpenVPN userspace software at version 2.6 or higher is required ++ to use this functionality. ++ + config EQUALIZER + tristate "EQL (serial line load balancing) support" + help +diff --git a/drivers/net/Makefile b/drivers/net/Makefile +index 3f1192d3c52d..8ed151e8d233 100644 +--- a/drivers/net/Makefile ++++ b/drivers/net/Makefile +@@ -11,6 +11,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/ + obj-$(CONFIG_IPVTAP) += ipvlan/ + obj-$(CONFIG_DUMMY) += dummy.o + obj-$(CONFIG_WIREGUARD) += wireguard/ ++obj-$(CONFIG_OVPN_DCO) += ovpn-dco/ + obj-$(CONFIG_EQUALIZER) += eql.o + obj-$(CONFIG_IFB) += ifb.o + obj-$(CONFIG_MACSEC) += macsec.o +diff --git a/drivers/net/ovpn-dco/Makefile b/drivers/net/ovpn-dco/Makefile +new file mode 100644 +index 000000000000..7efefe8f13a9 +--- /dev/null ++++ b/drivers/net/ovpn-dco/Makefile +@@ -0,0 +1,21 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ovpn-dco -- OpenVPN data channel offload ++# ++# Copyright (C) 2020-2022 OpenVPN, Inc. ++# ++# Author: Antonio Quartulli ++ ++obj-$(CONFIG_OVPN_DCO) += ovpn-dco.o ++ovpn-dco-y += main.o ++ovpn-dco-y += bind.o ++ovpn-dco-y += crypto.o ++ovpn-dco-y += ovpn.o ++ovpn-dco-y += peer.o ++ovpn-dco-y += sock.o ++ovpn-dco-y += stats.o ++ovpn-dco-y += netlink.o ++ovpn-dco-y += crypto_aead.o ++ovpn-dco-y += pktid.o ++ovpn-dco-y += tcp.o ++ovpn-dco-y += udp.o +diff --git a/drivers/net/ovpn-dco/addr.h b/drivers/net/ovpn-dco/addr.h +new file mode 100644 +index 000000000000..3d6ad0fc15af +--- /dev/null ++++ b/drivers/net/ovpn-dco/addr.h +@@ -0,0 +1,41 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNADDR_H_ ++#define _NET_OVPN_DCO_OVPNADDR_H_ ++ ++#include "crypto.h" ++ ++#include ++#include ++#include ++#include ++ ++/* our basic transport layer address */ ++struct ovpn_sockaddr { ++ union { ++ struct sockaddr_in in4; ++ struct sockaddr_in6 in6; ++ }; ++}; ++ ++/* Translate skb->protocol value to AF_INET or AF_INET6 */ ++static inline unsigned short skb_protocol_to_family(const struct sk_buff *skb) ++{ ++ switch (skb->protocol) { ++ case htons(ETH_P_IP): ++ return AF_INET; ++ case htons(ETH_P_IPV6): ++ return AF_INET6; ++ default: ++ return 0; ++ } ++} ++ ++#endif /* _NET_OVPN_DCO_OVPNADDR_H_ */ +diff --git a/drivers/net/ovpn-dco/bind.c b/drivers/net/ovpn-dco/bind.c +new file mode 100644 +index 000000000000..107697ea983e +--- /dev/null ++++ b/drivers/net/ovpn-dco/bind.c +@@ -0,0 +1,62 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2012-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "ovpn.h" ++#include "bind.h" ++#include "peer.h" ++ ++#include ++#include ++#include ++#include ++ ++/* Given a remote sockaddr, compute the skb hash ++ * and get a dst_entry so we can send packets to the remote. ++ * Called from process context or softirq (must be indicated with ++ * process_context bool). ++ */ ++struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *ss) ++{ ++ struct ovpn_bind *bind; ++ size_t sa_len; ++ ++ if (ss->ss_family == AF_INET) ++ sa_len = sizeof(struct sockaddr_in); ++ else if (ss->ss_family == AF_INET6) ++ sa_len = sizeof(struct sockaddr_in6); ++ else ++ return ERR_PTR(-EAFNOSUPPORT); ++ ++ bind = kzalloc(sizeof(*bind), GFP_ATOMIC); ++ if (unlikely(!bind)) ++ return ERR_PTR(-ENOMEM); ++ ++ memcpy(&bind->sa, ss, sa_len); ++ ++ return bind; ++} ++ ++static void ovpn_bind_release_rcu(struct rcu_head *head) ++{ ++ struct ovpn_bind *bind = container_of(head, struct ovpn_bind, rcu); ++ ++ kfree(bind); ++} ++ ++void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *new) ++{ ++ struct ovpn_bind *old; ++ ++ spin_lock_bh(&peer->lock); ++ old = rcu_replace_pointer(peer->bind, new, true); ++ spin_unlock_bh(&peer->lock); ++ ++ if (old) ++ call_rcu(&old->rcu, ovpn_bind_release_rcu); ++} +diff --git a/drivers/net/ovpn-dco/bind.h b/drivers/net/ovpn-dco/bind.h +new file mode 100644 +index 000000000000..a562e471acae +--- /dev/null ++++ b/drivers/net/ovpn-dco/bind.h +@@ -0,0 +1,67 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OVPN -- OpenVPN protocol accelerator for Linux ++ * Copyright (C) 2012-2022 OpenVPN, Inc. ++ * All rights reserved. ++ * Author: James Yonan ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNBIND_H_ ++#define _NET_OVPN_DCO_OVPNBIND_H_ ++ ++#include "addr.h" ++#include "rcu.h" ++ ++#include ++#include ++#include ++ ++struct ovpn_peer; ++ ++struct ovpn_bind { ++ struct ovpn_sockaddr sa; /* remote sockaddr */ ++ ++ union { ++ struct in_addr ipv4; ++ struct in6_addr ipv6; ++ } local; ++ ++ struct rcu_head rcu; ++}; ++ ++static inline bool ovpn_bind_skb_src_match(const struct ovpn_bind *bind, struct sk_buff *skb) ++{ ++ const unsigned short family = skb_protocol_to_family(skb); ++ const struct ovpn_sockaddr *sa = &bind->sa; ++ ++ if (unlikely(!bind)) ++ return false; ++ ++ if (unlikely(sa->in4.sin_family != family)) ++ return false; ++ ++ switch (family) { ++ case AF_INET: ++ if (unlikely(sa->in4.sin_addr.s_addr != ip_hdr(skb)->saddr)) ++ return false; ++ ++ if (unlikely(sa->in4.sin_port != udp_hdr(skb)->source)) ++ return false; ++ break; ++ case AF_INET6: ++ if (unlikely(!ipv6_addr_equal(&sa->in6.sin6_addr, &ipv6_hdr(skb)->saddr))) ++ return false; ++ ++ if (unlikely(sa->in6.sin6_port != udp_hdr(skb)->source)) ++ return false; ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++} ++ ++struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *sa); ++void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *bind); ++ ++#endif /* _NET_OVPN_DCO_OVPNBIND_H_ */ +diff --git a/drivers/net/ovpn-dco/crypto.c b/drivers/net/ovpn-dco/crypto.c +new file mode 100644 +index 000000000000..fcc3a351ba9d +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto.c +@@ -0,0 +1,154 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "crypto_aead.h" ++#include "crypto.h" ++ ++#include ++ ++static void ovpn_ks_destroy_rcu(struct rcu_head *head) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ ks = container_of(head, struct ovpn_crypto_key_slot, rcu); ++ ovpn_aead_crypto_key_slot_destroy(ks); ++} ++ ++void ovpn_crypto_key_slot_release(struct kref *kref) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ ks = container_of(kref, struct ovpn_crypto_key_slot, refcount); ++ call_rcu(&ks->rcu, ovpn_ks_destroy_rcu); ++} ++ ++/* can only be invoked when all peer references have been dropped (i.e. RCU ++ * release routine) ++ */ ++void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ ks = rcu_access_pointer(cs->primary); ++ if (ks) { ++ RCU_INIT_POINTER(cs->primary, NULL); ++ ovpn_crypto_key_slot_put(ks); ++ } ++ ++ ks = rcu_access_pointer(cs->secondary); ++ if (ks) { ++ RCU_INIT_POINTER(cs->secondary, NULL); ++ ovpn_crypto_key_slot_put(ks); ++ } ++ ++ mutex_destroy(&cs->mutex); ++} ++ ++/* removes the primary key from the crypto context */ ++void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ mutex_lock(&cs->mutex); ++ ks = rcu_replace_pointer(cs->primary, NULL, lockdep_is_held(&cs->mutex)); ++ ovpn_crypto_key_slot_put(ks); ++ mutex_unlock(&cs->mutex); ++} ++ ++/* Reset the ovpn_crypto_state object in a way that is atomic ++ * to RCU readers. ++ */ ++int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, ++ const struct ovpn_peer_key_reset *pkr) ++ __must_hold(cs->mutex) ++{ ++ struct ovpn_crypto_key_slot *old = NULL; ++ struct ovpn_crypto_key_slot *new; ++ ++ lockdep_assert_held(&cs->mutex); ++ ++ new = ovpn_aead_crypto_key_slot_new(&pkr->key); ++ if (IS_ERR(new)) ++ return PTR_ERR(new); ++ ++ switch (pkr->slot) { ++ case OVPN_KEY_SLOT_PRIMARY: ++ old = rcu_replace_pointer(cs->primary, new, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ case OVPN_KEY_SLOT_SECONDARY: ++ old = rcu_replace_pointer(cs->secondary, new, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ default: ++ goto free_key; ++ } ++ ++ if (old) ++ ovpn_crypto_key_slot_put(old); ++ ++ return 0; ++free_key: ++ ovpn_crypto_key_slot_put(new); ++ return -EINVAL; ++} ++ ++void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, ++ enum ovpn_key_slot slot) ++{ ++ struct ovpn_crypto_key_slot *ks = NULL; ++ ++ mutex_lock(&cs->mutex); ++ switch (slot) { ++ case OVPN_KEY_SLOT_PRIMARY: ++ ks = rcu_replace_pointer(cs->primary, NULL, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ case OVPN_KEY_SLOT_SECONDARY: ++ ks = rcu_replace_pointer(cs->secondary, NULL, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ default: ++ pr_warn("Invalid slot to release: %u\n", slot); ++ break; ++ } ++ mutex_unlock(&cs->mutex); ++ ++ if (!ks) { ++ pr_debug("Key slot already released: %u\n", slot); ++ return; ++ } ++ pr_debug("deleting key slot %u, key_id=%u\n", slot, ks->key_id); ++ ++ ovpn_crypto_key_slot_put(ks); ++} ++ ++/* this swap is not atomic, but there will be a very short time frame where the ++ * old_secondary key won't be available. This should not be a big deal as most ++ * likely both peers are already using the new primary at this point. ++ */ ++void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs) ++{ ++ const struct ovpn_crypto_key_slot *old_primary, *old_secondary; ++ ++ mutex_lock(&cs->mutex); ++ ++ old_secondary = rcu_dereference_protected(cs->secondary, ++ lockdep_is_held(&cs->mutex)); ++ old_primary = rcu_replace_pointer(cs->primary, old_secondary, ++ lockdep_is_held(&cs->mutex)); ++ rcu_assign_pointer(cs->secondary, old_primary); ++ ++ pr_debug("key swapped: %u <-> %u\n", ++ old_primary ? old_primary->key_id : 0, ++ old_secondary ? old_secondary->key_id : 0); ++ ++ mutex_unlock(&cs->mutex); ++} +diff --git a/drivers/net/ovpn-dco/crypto.h b/drivers/net/ovpn-dco/crypto.h +new file mode 100644 +index 000000000000..79f580e54a63 +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNCRYPTO_H_ ++#define _NET_OVPN_DCO_OVPNCRYPTO_H_ ++ ++#include "main.h" ++#include "pktid.h" ++ ++#include ++#include ++ ++struct ovpn_peer; ++struct ovpn_crypto_key_slot; ++ ++/* info needed for both encrypt and decrypt directions */ ++struct ovpn_key_direction { ++ const u8 *cipher_key; ++ size_t cipher_key_size; ++ const u8 *nonce_tail; /* only needed for GCM modes */ ++ size_t nonce_tail_size; /* only needed for GCM modes */ ++}; ++ ++/* all info for a particular symmetric key (primary or secondary) */ ++struct ovpn_key_config { ++ enum ovpn_cipher_alg cipher_alg; ++ u8 key_id; ++ struct ovpn_key_direction encrypt; ++ struct ovpn_key_direction decrypt; ++}; ++ ++/* used to pass settings from netlink to the crypto engine */ ++struct ovpn_peer_key_reset { ++ enum ovpn_key_slot slot; ++ struct ovpn_key_config key; ++}; ++ ++struct ovpn_crypto_key_slot { ++ u8 key_id; ++ ++ struct crypto_aead *encrypt; ++ struct crypto_aead *decrypt; ++ struct ovpn_nonce_tail nonce_tail_xmit; ++ struct ovpn_nonce_tail nonce_tail_recv; ++ ++ struct ovpn_pktid_recv pid_recv ____cacheline_aligned_in_smp; ++ struct ovpn_pktid_xmit pid_xmit ____cacheline_aligned_in_smp; ++ struct kref refcount; ++ struct rcu_head rcu; ++}; ++ ++struct ovpn_crypto_state { ++ struct ovpn_crypto_key_slot __rcu *primary; ++ struct ovpn_crypto_key_slot __rcu *secondary; ++ ++ /* protects primary and secondary slots */ ++ struct mutex mutex; ++}; ++ ++static inline bool ovpn_crypto_key_slot_hold(struct ovpn_crypto_key_slot *ks) ++{ ++ return kref_get_unless_zero(&ks->refcount); ++} ++ ++static inline void ovpn_crypto_state_init(struct ovpn_crypto_state *cs) ++{ ++ RCU_INIT_POINTER(cs->primary, NULL); ++ RCU_INIT_POINTER(cs->secondary, NULL); ++ mutex_init(&cs->mutex); ++} ++ ++static inline struct ovpn_crypto_key_slot * ++ovpn_crypto_key_id_to_slot(const struct ovpn_crypto_state *cs, u8 key_id) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ if (unlikely(!cs)) ++ return NULL; ++ ++ rcu_read_lock(); ++ ks = rcu_dereference(cs->primary); ++ if (ks && ks->key_id == key_id) { ++ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) ++ ks = NULL; ++ goto out; ++ } ++ ++ ks = rcu_dereference(cs->secondary); ++ if (ks && ks->key_id == key_id) { ++ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) ++ ks = NULL; ++ goto out; ++ } ++ ++ /* when both key slots are occupied but no matching key ID is found, ks has to be reset to ++ * NULL to avoid carrying a stale pointer ++ */ ++ ks = NULL; ++out: ++ rcu_read_unlock(); ++ ++ return ks; ++} ++ ++static inline struct ovpn_crypto_key_slot * ++ovpn_crypto_key_slot_primary(const struct ovpn_crypto_state *cs) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ rcu_read_lock(); ++ ks = rcu_dereference(cs->primary); ++ if (unlikely(ks && !ovpn_crypto_key_slot_hold(ks))) ++ ks = NULL; ++ rcu_read_unlock(); ++ ++ return ks; ++} ++ ++void ovpn_crypto_key_slot_release(struct kref *kref); ++ ++static inline void ovpn_crypto_key_slot_put(struct ovpn_crypto_key_slot *ks) ++{ ++ kref_put(&ks->refcount, ovpn_crypto_key_slot_release); ++} ++ ++int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, ++ const struct ovpn_peer_key_reset *pkr); ++ ++void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, ++ enum ovpn_key_slot slot); ++ ++void ovpn_crypto_state_release(struct ovpn_crypto_state *cs); ++ ++void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs); ++ ++void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs); ++ ++#endif /* _NET_OVPN_DCO_OVPNCRYPTO_H_ */ +diff --git a/drivers/net/ovpn-dco/crypto_aead.c b/drivers/net/ovpn-dco/crypto_aead.c +new file mode 100644 +index 000000000000..c21bff90d748 +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto_aead.c +@@ -0,0 +1,367 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "crypto_aead.h" ++#include "crypto.h" ++#include "pktid.h" ++#include "proto.h" ++#include "skb.h" ++ ++#include ++#include ++#include ++ ++#define AUTH_TAG_SIZE 16 ++ ++static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) ++{ ++ return OVPN_OP_SIZE_V2 + /* OP header size */ ++ 4 + /* Packet ID */ ++ crypto_aead_authsize(ks->encrypt); /* Auth Tag */ ++} ++ ++int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id) ++{ ++ const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); ++ const unsigned int head_size = ovpn_aead_encap_overhead(ks); ++ struct scatterlist sg[MAX_SKB_FRAGS + 2]; ++ DECLARE_CRYPTO_WAIT(wait); ++ struct aead_request *req; ++ struct sk_buff *trailer; ++ u8 iv[NONCE_SIZE]; ++ int nfrags, ret; ++ u32 pktid, op; ++ ++ /* Sample AEAD header format: ++ * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... ++ * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] ++ * [4-byte ++ * IV head] ++ */ ++ ++ /* check that there's enough headroom in the skb for packet ++ * encapsulation, after adding network header and encryption overhead ++ */ ++ if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM + head_size))) ++ return -ENOBUFS; ++ ++ /* get number of skb frags and ensure that packet data is writable */ ++ nfrags = skb_cow_data(skb, 0, &trailer); ++ if (unlikely(nfrags < 0)) ++ return nfrags; ++ ++ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) ++ return -ENOSPC; ++ ++ req = aead_request_alloc(ks->encrypt, GFP_KERNEL); ++ if (unlikely(!req)) ++ return -ENOMEM; ++ ++ /* sg table: ++ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), ++ * 1, 2, 3, ..., n: payload, ++ * n+1: auth_tag (len=tag_size) ++ */ ++ sg_init_table(sg, nfrags + 2); ++ ++ /* build scatterlist to encrypt packet payload */ ++ ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); ++ if (unlikely(nfrags != ret)) { ++ ret = -EINVAL; ++ goto free_req; ++ } ++ ++ /* append auth_tag onto scatterlist */ ++ __skb_push(skb, tag_size); ++ sg_set_buf(sg + nfrags + 1, skb->data, tag_size); ++ ++ /* obtain packet ID, which is used both as a first ++ * 4 bytes of nonce and last 4 bytes of associated data. ++ */ ++ ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); ++ if (unlikely(ret < 0)) ++ goto free_req; ++ ++ /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes nonce */ ++ ovpn_pktid_aead_write(pktid, &ks->nonce_tail_xmit, iv); ++ ++ /* make space for packet id and push it to the front */ ++ __skb_push(skb, NONCE_WIRE_SIZE); ++ memcpy(skb->data, iv, NONCE_WIRE_SIZE); ++ ++ /* add packet op as head of additional data */ ++ op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer_id); ++ __skb_push(skb, OVPN_OP_SIZE_V2); ++ BUILD_BUG_ON(sizeof(op) != OVPN_OP_SIZE_V2); ++ *((__force __be32 *)skb->data) = htonl(op); ++ ++ /* AEAD Additional data */ ++ sg_set_buf(sg, skb->data, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); ++ ++ /* setup async crypto operation */ ++ aead_request_set_tfm(req, ks->encrypt); ++ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | ++ CRYPTO_TFM_REQ_MAY_SLEEP, ++ crypto_req_done, &wait); ++ aead_request_set_crypt(req, sg, sg, skb->len - head_size, iv); ++ aead_request_set_ad(req, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); ++ ++ /* encrypt it */ ++ ret = crypto_wait_req(crypto_aead_encrypt(req), &wait); ++ if (ret < 0) ++ net_err_ratelimited("%s: encrypt failed: %d\n", __func__, ret); ++ ++free_req: ++ aead_request_free(req); ++ return ret; ++} ++ ++int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) ++{ ++ const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); ++ struct scatterlist sg[MAX_SKB_FRAGS + 2]; ++ int ret, payload_len, nfrags; ++ u8 *sg_data, iv[NONCE_SIZE]; ++ unsigned int payload_offset; ++ DECLARE_CRYPTO_WAIT(wait); ++ struct aead_request *req; ++ struct sk_buff *trailer; ++ unsigned int sg_len; ++ __be32 *pid; ++ ++ payload_offset = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE + tag_size; ++ payload_len = skb->len - payload_offset; ++ ++ /* sanity check on packet size, payload size must be >= 0 */ ++ if (unlikely(payload_len < 0)) ++ return -EINVAL; ++ ++ /* Prepare the skb data buffer to be accessed up until the auth tag. ++ * This is required because this area is directly mapped into the sg list. ++ */ ++ if (unlikely(!pskb_may_pull(skb, payload_offset))) ++ return -ENODATA; ++ ++ /* get number of skb frags and ensure that packet data is writable */ ++ nfrags = skb_cow_data(skb, 0, &trailer); ++ if (unlikely(nfrags < 0)) ++ return nfrags; ++ ++ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) ++ return -ENOSPC; ++ ++ req = aead_request_alloc(ks->decrypt, GFP_KERNEL); ++ if (unlikely(!req)) ++ return -ENOMEM; ++ ++ /* sg table: ++ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), ++ * 1, 2, 3, ..., n: payload, ++ * n+1: auth_tag (len=tag_size) ++ */ ++ sg_init_table(sg, nfrags + 2); ++ ++ /* packet op is head of additional data */ ++ sg_data = skb->data; ++ sg_len = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE; ++ sg_set_buf(sg, sg_data, sg_len); ++ ++ /* build scatterlist to decrypt packet payload */ ++ ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); ++ if (unlikely(nfrags != ret)) { ++ ret = -EINVAL; ++ goto free_req; ++ } ++ ++ /* append auth_tag onto scatterlist */ ++ sg_set_buf(sg + nfrags + 1, skb->data + sg_len, tag_size); ++ ++ /* copy nonce into IV buffer */ ++ memcpy(iv, skb->data + OVPN_OP_SIZE_V2, NONCE_WIRE_SIZE); ++ memcpy(iv + NONCE_WIRE_SIZE, ks->nonce_tail_recv.u8, ++ sizeof(struct ovpn_nonce_tail)); ++ ++ /* setup async crypto operation */ ++ aead_request_set_tfm(req, ks->decrypt); ++ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | ++ CRYPTO_TFM_REQ_MAY_SLEEP, ++ crypto_req_done, &wait); ++ aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); ++ ++ aead_request_set_ad(req, NONCE_WIRE_SIZE + OVPN_OP_SIZE_V2); ++ ++ /* decrypt it */ ++ ret = crypto_wait_req(crypto_aead_decrypt(req), &wait); ++ if (ret < 0) { ++ net_err_ratelimited("%s: decrypt failed: %d\n", __func__, ret); ++ goto free_req; ++ } ++ ++ /* PID sits after the op */ ++ pid = (__force __be32 *)(skb->data + OVPN_OP_SIZE_V2); ++ ret = ovpn_pktid_recv(&ks->pid_recv, ntohl(*pid), 0); ++ if (unlikely(ret < 0)) ++ goto free_req; ++ ++ /* point to encapsulated IP packet */ ++ __skb_pull(skb, payload_offset); ++ ++free_req: ++ aead_request_free(req); ++ return ret; ++} ++ ++/* Initialize a struct crypto_aead object */ ++struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, ++ const unsigned char *key, unsigned int keylen) ++{ ++ struct crypto_aead *aead; ++ int ret; ++ ++ aead = crypto_alloc_aead(alg_name, 0, 0); ++ if (IS_ERR(aead)) { ++ ret = PTR_ERR(aead); ++ pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); ++ aead = NULL; ++ goto error; ++ } ++ ++ ret = crypto_aead_setkey(aead, key, keylen); ++ if (ret) { ++ pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, keylen, ret); ++ goto error; ++ } ++ ++ ret = crypto_aead_setauthsize(aead, AUTH_TAG_SIZE); ++ if (ret) { ++ pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, ret); ++ goto error; ++ } ++ ++ /* basic AEAD assumption */ ++ if (crypto_aead_ivsize(aead) != NONCE_SIZE) { ++ pr_err("%s IV size must be %d\n", title, NONCE_SIZE); ++ ret = -EINVAL; ++ goto error; ++ } ++ ++ pr_debug("********* Cipher %s (%s)\n", alg_name, title); ++ pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); ++ pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); ++ pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); ++ pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); ++ pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); ++ ++ return aead; ++ ++error: ++ crypto_free_aead(aead); ++ return ERR_PTR(ret); ++} ++ ++void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks) ++{ ++ if (!ks) ++ return; ++ ++ crypto_free_aead(ks->encrypt); ++ crypto_free_aead(ks->decrypt); ++ kfree(ks); ++} ++ ++static struct ovpn_crypto_key_slot * ++ovpn_aead_crypto_key_slot_init(enum ovpn_cipher_alg alg, ++ const unsigned char *encrypt_key, ++ unsigned int encrypt_keylen, ++ const unsigned char *decrypt_key, ++ unsigned int decrypt_keylen, ++ const unsigned char *encrypt_nonce_tail, ++ unsigned int encrypt_nonce_tail_len, ++ const unsigned char *decrypt_nonce_tail, ++ unsigned int decrypt_nonce_tail_len, ++ u16 key_id) ++{ ++ struct ovpn_crypto_key_slot *ks = NULL; ++ const char *alg_name; ++ int ret; ++ ++ /* validate crypto alg */ ++ switch (alg) { ++ case OVPN_CIPHER_ALG_AES_GCM: ++ alg_name = "gcm(aes)"; ++ break; ++ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: ++ alg_name = "rfc7539(chacha20,poly1305)"; ++ break; ++ default: ++ return ERR_PTR(-EOPNOTSUPP); ++ } ++ ++ /* build the key slot */ ++ ks = kmalloc(sizeof(*ks), GFP_KERNEL); ++ if (!ks) ++ return ERR_PTR(-ENOMEM); ++ ++ ks->encrypt = NULL; ++ ks->decrypt = NULL; ++ kref_init(&ks->refcount); ++ ks->key_id = key_id; ++ ++ ks->encrypt = ovpn_aead_init("encrypt", alg_name, encrypt_key, ++ encrypt_keylen); ++ if (IS_ERR(ks->encrypt)) { ++ ret = PTR_ERR(ks->encrypt); ++ ks->encrypt = NULL; ++ goto destroy_ks; ++ } ++ ++ ks->decrypt = ovpn_aead_init("decrypt", alg_name, decrypt_key, ++ decrypt_keylen); ++ if (IS_ERR(ks->decrypt)) { ++ ret = PTR_ERR(ks->decrypt); ++ ks->decrypt = NULL; ++ goto destroy_ks; ++ } ++ ++ if (sizeof(struct ovpn_nonce_tail) != encrypt_nonce_tail_len || ++ sizeof(struct ovpn_nonce_tail) != decrypt_nonce_tail_len) { ++ ret = -EINVAL; ++ goto destroy_ks; ++ } ++ ++ memcpy(ks->nonce_tail_xmit.u8, encrypt_nonce_tail, ++ sizeof(struct ovpn_nonce_tail)); ++ memcpy(ks->nonce_tail_recv.u8, decrypt_nonce_tail, ++ sizeof(struct ovpn_nonce_tail)); ++ ++ /* init packet ID generation/validation */ ++ ovpn_pktid_xmit_init(&ks->pid_xmit); ++ ovpn_pktid_recv_init(&ks->pid_recv); ++ ++ return ks; ++ ++destroy_ks: ++ ovpn_aead_crypto_key_slot_destroy(ks); ++ return ERR_PTR(ret); ++} ++ ++struct ovpn_crypto_key_slot * ++ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) ++{ ++ return ovpn_aead_crypto_key_slot_init(kc->cipher_alg, ++ kc->encrypt.cipher_key, ++ kc->encrypt.cipher_key_size, ++ kc->decrypt.cipher_key, ++ kc->decrypt.cipher_key_size, ++ kc->encrypt.nonce_tail, ++ kc->encrypt.nonce_tail_size, ++ kc->decrypt.nonce_tail, ++ kc->decrypt.nonce_tail_size, ++ kc->key_id); ++} +diff --git a/drivers/net/ovpn-dco/crypto_aead.h b/drivers/net/ovpn-dco/crypto_aead.h +new file mode 100644 +index 000000000000..1e3054e7d5a4 +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto_aead.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNAEAD_H_ ++#define _NET_OVPN_DCO_OVPNAEAD_H_ ++ ++#include "crypto.h" ++ ++#include ++#include ++ ++struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, ++ const unsigned char *key, unsigned int keylen); ++ ++int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id); ++int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); ++ ++struct ovpn_crypto_key_slot *ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc); ++void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks); ++ ++#endif /* _NET_OVPN_DCO_OVPNAEAD_H_ */ +diff --git a/drivers/net/ovpn-dco/main.c b/drivers/net/ovpn-dco/main.c +new file mode 100644 +index 000000000000..4eb90ea7a500 +--- /dev/null ++++ b/drivers/net/ovpn-dco/main.c +@@ -0,0 +1,271 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ ++ ++#include "main.h" ++ ++#include "ovpn.h" ++#include "ovpnstruct.h" ++#include "netlink.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++/* Driver info */ ++#define DRV_NAME "ovpn-dco" ++#define DRV_VERSION OVPN_DCO_VERSION ++#define DRV_DESCRIPTION "OpenVPN data channel offload (ovpn-dco)" ++#define DRV_COPYRIGHT "(C) 2020-2022 OpenVPN, Inc." ++ ++static void ovpn_struct_free(struct net_device *net) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(net); ++ ++ security_tun_dev_free_security(ovpn->security); ++ free_percpu(net->tstats); ++ flush_workqueue(ovpn->crypto_wq); ++ flush_workqueue(ovpn->events_wq); ++ destroy_workqueue(ovpn->crypto_wq); ++ destroy_workqueue(ovpn->events_wq); ++ rcu_barrier(); ++} ++ ++/* Net device open */ ++static int ovpn_net_open(struct net_device *dev) ++{ ++ struct in_device *dev_v4 = __in_dev_get_rtnl(dev); ++ ++ if (dev_v4) { ++ /* disable redirects as Linux gets confused by ovpn-dco handling same-LAN routing */ ++ IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); ++ IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; ++ } ++ ++ netif_tx_start_all_queues(dev); ++ return 0; ++} ++ ++/* Net device stop -- called prior to device unload */ ++static int ovpn_net_stop(struct net_device *dev) ++{ ++ netif_tx_stop_all_queues(dev); ++ return 0; ++} ++ ++/******************************************* ++ * ovpn ethtool ops ++ *******************************************/ ++ ++static int ovpn_get_link_ksettings(struct net_device *dev, ++ struct ethtool_link_ksettings *cmd) ++{ ++ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 0); ++ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, 0); ++ cmd->base.speed = SPEED_1000; ++ cmd->base.duplex = DUPLEX_FULL; ++ cmd->base.port = PORT_TP; ++ cmd->base.phy_address = 0; ++ cmd->base.transceiver = XCVR_INTERNAL; ++ cmd->base.autoneg = AUTONEG_DISABLE; ++ ++ return 0; ++} ++ ++static void ovpn_get_drvinfo(struct net_device *dev, ++ struct ethtool_drvinfo *info) ++{ ++ strscpy(info->driver, DRV_NAME, sizeof(info->driver)); ++ strscpy(info->version, DRV_VERSION, sizeof(info->version)); ++ strscpy(info->bus_info, "ovpn", sizeof(info->bus_info)); ++} ++ ++bool ovpn_dev_is_valid(const struct net_device *dev) ++{ ++ return dev->netdev_ops->ndo_start_xmit == ovpn_net_xmit; ++} ++ ++/******************************************* ++ * ovpn exported methods ++ *******************************************/ ++ ++static const struct net_device_ops ovpn_netdev_ops = { ++ .ndo_open = ovpn_net_open, ++ .ndo_stop = ovpn_net_stop, ++ .ndo_start_xmit = ovpn_net_xmit, ++ .ndo_get_stats64 = dev_get_tstats64, ++}; ++ ++static const struct ethtool_ops ovpn_ethtool_ops = { ++ .get_link_ksettings = ovpn_get_link_ksettings, ++ .get_drvinfo = ovpn_get_drvinfo, ++ .get_link = ethtool_op_get_link, ++ .get_ts_info = ethtool_op_get_ts_info, ++}; ++ ++static void ovpn_setup(struct net_device *dev) ++{ ++ /* compute the overhead considering AEAD encryption */ ++ const int overhead = sizeof(u32) + NONCE_WIRE_SIZE + 16 + sizeof(struct udphdr) + ++ max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); ++ ++ netdev_features_t feat = NETIF_F_SG | NETIF_F_LLTX | ++ NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_GSO | ++ NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA; ++ ++ dev->ethtool_ops = &ovpn_ethtool_ops; ++ dev->needs_free_netdev = true; ++ ++ dev->netdev_ops = &ovpn_netdev_ops; ++ ++ dev->priv_destructor = ovpn_struct_free; ++ ++ /* Point-to-Point TUN Device */ ++ dev->hard_header_len = 0; ++ dev->addr_len = 0; ++ dev->mtu = ETH_DATA_LEN - overhead; ++ dev->min_mtu = IPV4_MIN_MTU; ++ dev->max_mtu = IP_MAX_MTU - overhead; ++ ++ /* Zero header length */ ++ dev->type = ARPHRD_NONE; ++ dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; ++ ++ dev->features |= feat; ++ dev->hw_features |= feat; ++ dev->hw_enc_features |= feat; ++ ++ dev->needed_headroom = OVPN_HEAD_ROOM; ++ dev->needed_tailroom = OVPN_MAX_PADDING; ++} ++ ++static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = { ++ [IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, __OVPN_MODE_FIRST, ++ __OVPN_MODE_AFTER_LAST - 1), ++}; ++ ++static int ovpn_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], ++ struct nlattr *data[], struct netlink_ext_ack *extack) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ int ret; ++ ++ ret = security_tun_dev_create(); ++ if (ret < 0) ++ return ret; ++ ++ ret = ovpn_struct_init(dev); ++ if (ret < 0) ++ return ret; ++ ++ ovpn->mode = OVPN_MODE_P2P; ++ if (data && data[IFLA_OVPN_MODE]) { ++ ovpn->mode = nla_get_u8(data[IFLA_OVPN_MODE]); ++ netdev_dbg(dev, "%s: setting device (%s) mode: %u\n", __func__, dev->name, ++ ovpn->mode); ++ } ++ ++ return register_netdevice(dev); ++} ++ ++static void ovpn_dellink(struct net_device *dev, struct list_head *head) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ ++ switch (ovpn->mode) { ++ case OVPN_MODE_P2P: ++ ovpn_peer_release_p2p(ovpn); ++ break; ++ default: ++ ovpn_peers_free(ovpn); ++ break; ++ } ++ ++ unregister_netdevice_queue(dev, head); ++} ++ ++/** ++ * ovpn_num_queues - define number of queues to allocate per device ++ * ++ * The value returned by this function is used to decide how many RX and TX ++ * queues to allocate when creating the netdev object ++ * ++ * Return the number of queues to allocate ++ */ ++static unsigned int ovpn_num_queues(void) ++{ ++ return num_online_cpus(); ++} ++ ++static struct rtnl_link_ops ovpn_link_ops __read_mostly = { ++ .kind = DRV_NAME, ++ .priv_size = sizeof(struct ovpn_struct), ++ .setup = ovpn_setup, ++ .policy = ovpn_policy, ++ .maxtype = IFLA_OVPN_MAX, ++ .newlink = ovpn_newlink, ++ .dellink = ovpn_dellink, ++ .get_num_tx_queues = ovpn_num_queues, ++ .get_num_rx_queues = ovpn_num_queues, ++}; ++ ++static int __init ovpn_init(void) ++{ ++ int err = 0; ++ ++ pr_info("%s %s -- %s\n", DRV_DESCRIPTION, DRV_VERSION, DRV_COPYRIGHT); ++ ++ /* init RTNL link ops */ ++ err = rtnl_link_register(&ovpn_link_ops); ++ if (err) { ++ pr_err("ovpn: can't register RTNL link ops\n"); ++ goto err; ++ } ++ ++ err = ovpn_netlink_register(); ++ if (err) { ++ pr_err("ovpn: can't register netlink family\n"); ++ goto err_rtnl_unregister; ++ } ++ ++ return 0; ++ ++err_rtnl_unregister: ++ rtnl_link_unregister(&ovpn_link_ops); ++err: ++ pr_err("ovpn: initialization failed, error status=%d\n", err); ++ return err; ++} ++ ++static __exit void ovpn_cleanup(void) ++{ ++ rtnl_link_unregister(&ovpn_link_ops); ++ ovpn_netlink_unregister(); ++ rcu_barrier(); /* because we use call_rcu */ ++} ++ ++module_init(ovpn_init); ++module_exit(ovpn_cleanup); ++ ++MODULE_DESCRIPTION(DRV_DESCRIPTION); ++MODULE_AUTHOR(DRV_COPYRIGHT); ++MODULE_LICENSE("GPL"); ++MODULE_VERSION(DRV_VERSION); ++MODULE_ALIAS_RTNL_LINK(DRV_NAME); ++MODULE_ALIAS_GENL_FAMILY(OVPN_NL_NAME); +diff --git a/drivers/net/ovpn-dco/main.h b/drivers/net/ovpn-dco/main.h +new file mode 100644 +index 000000000000..c4ef200b30f4 +--- /dev/null ++++ b/drivers/net/ovpn-dco/main.h +@@ -0,0 +1,32 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_MAIN_H_ ++#define _NET_OVPN_DCO_MAIN_H_ ++ ++#include ++#include ++#include ++#include ++ ++#define OVPN_DCO_VERSION "2.0.0" ++ ++struct net_device; ++bool ovpn_dev_is_valid(const struct net_device *dev); ++ ++#define SKB_HEADER_LEN \ ++ (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ ++ sizeof(struct udphdr) + NET_SKB_PAD) ++ ++#define OVPN_HEAD_ROOM ALIGN(16 + SKB_HEADER_LEN, 4) ++#define OVPN_MAX_PADDING 16 ++#define OVPN_QUEUE_LEN 1024 ++#define OVPN_MAX_TUN_QUEUE_LEN 0x10000 ++ ++#endif /* _NET_OVPN_DCO_OVPN_DCO_H_ */ +diff --git a/drivers/net/ovpn-dco/netlink.c b/drivers/net/ovpn-dco/netlink.c +new file mode 100644 +index 000000000000..ee5c943e7db4 +--- /dev/null ++++ b/drivers/net/ovpn-dco/netlink.c +@@ -0,0 +1,1143 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "ovpn.h" ++#include "peer.h" ++#include "proto.h" ++#include "netlink.h" ++#include "ovpnstruct.h" ++#include "udp.h" ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** The ovpn-dco netlink family */ ++static struct genl_family ovpn_netlink_family; ++ ++enum ovpn_netlink_multicast_groups { ++ OVPN_MCGRP_PEERS, ++}; ++ ++static const struct genl_multicast_group ovpn_netlink_mcgrps[] = { ++ [OVPN_MCGRP_PEERS] = { .name = OVPN_NL_MULTICAST_GROUP_PEERS }, ++}; ++ ++/** Key direction policy. Can be used for configuring an encryption and a decryption key */ ++static const struct nla_policy ovpn_netlink_policy_key_dir[OVPN_KEY_DIR_ATTR_MAX + 1] = { ++ [OVPN_KEY_DIR_ATTR_CIPHER_KEY] = NLA_POLICY_MAX_LEN(U8_MAX), ++ [OVPN_KEY_DIR_ATTR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(NONCE_TAIL_SIZE), ++}; ++ ++/** CMD_NEW_KEY policy */ ++static const struct nla_policy ovpn_netlink_policy_new_key[OVPN_NEW_KEY_ATTR_MAX + 1] = { ++ [OVPN_NEW_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_NEW_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, ++ __OVPN_KEY_SLOT_AFTER_LAST - 1), ++ [OVPN_NEW_KEY_ATTR_KEY_ID] = { .type = NLA_U8 }, ++ [OVPN_NEW_KEY_ATTR_CIPHER_ALG] = { .type = NLA_U16 }, ++ [OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), ++ [OVPN_NEW_KEY_ATTR_DECRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), ++}; ++ ++/** CMD_DEL_KEY policy */ ++static const struct nla_policy ovpn_netlink_policy_del_key[OVPN_DEL_KEY_ATTR_MAX + 1] = { ++ [OVPN_DEL_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_DEL_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, ++ __OVPN_KEY_SLOT_AFTER_LAST - 1), ++}; ++ ++/** CMD_SWAP_KEYS policy */ ++static const struct nla_policy ovpn_netlink_policy_swap_keys[OVPN_SWAP_KEYS_ATTR_MAX + 1] = { ++ [OVPN_SWAP_KEYS_ATTR_PEER_ID] = { .type = NLA_U32 }, ++}; ++ ++/** CMD_NEW_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_new_peer[OVPN_NEW_PEER_ATTR_MAX + 1] = { ++ [OVPN_NEW_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE] = NLA_POLICY_MIN_LEN(sizeof(struct sockaddr)), ++ [OVPN_NEW_PEER_ATTR_SOCKET] = { .type = NLA_U32 }, ++ [OVPN_NEW_PEER_ATTR_IPV4] = { .type = NLA_U32 }, ++ [OVPN_NEW_PEER_ATTR_IPV6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), ++ [OVPN_NEW_PEER_ATTR_LOCAL_IP] = NLA_POLICY_MAX_LEN(sizeof(struct in6_addr)), ++}; ++ ++/** CMD_SET_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_set_peer[OVPN_SET_PEER_ATTR_MAX + 1] = { ++ [OVPN_SET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, ++ [OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT] = { .type = NLA_U32 }, ++}; ++ ++/** CMD_DEL_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_del_peer[OVPN_DEL_PEER_ATTR_MAX + 1] = { ++ [OVPN_DEL_PEER_ATTR_REASON] = NLA_POLICY_RANGE(NLA_U8, __OVPN_DEL_PEER_REASON_FIRST, ++ __OVPN_DEL_PEER_REASON_AFTER_LAST - 1), ++ [OVPN_DEL_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++}; ++ ++/** CMD_GET_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_get_peer[OVPN_GET_PEER_ATTR_MAX + 1] = { ++ [OVPN_GET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++}; ++ ++/** CMD_PACKET polocy */ ++static const struct nla_policy ovpn_netlink_policy_packet[OVPN_PACKET_ATTR_MAX + 1] = { ++ [OVPN_PACKET_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_PACKET_ATTR_PACKET] = NLA_POLICY_MAX_LEN(U16_MAX), ++}; ++ ++/** Generic message container policy */ ++static const struct nla_policy ovpn_netlink_policy[OVPN_ATTR_MAX + 1] = { ++ [OVPN_ATTR_IFINDEX] = { .type = NLA_U32 }, ++ [OVPN_ATTR_NEW_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_peer), ++ [OVPN_ATTR_SET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_set_peer), ++ [OVPN_ATTR_DEL_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_peer), ++ [OVPN_ATTR_GET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_get_peer), ++ [OVPN_ATTR_NEW_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_key), ++ [OVPN_ATTR_SWAP_KEYS] = NLA_POLICY_NESTED(ovpn_netlink_policy_swap_keys), ++ [OVPN_ATTR_DEL_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_key), ++ [OVPN_ATTR_PACKET] = NLA_POLICY_NESTED(ovpn_netlink_policy_packet), ++}; ++ ++static struct net_device * ++ovpn_get_dev_from_attrs(struct net *net, struct nlattr **attrs) ++{ ++ struct net_device *dev; ++ int ifindex; ++ ++ if (!attrs[OVPN_ATTR_IFINDEX]) ++ return ERR_PTR(-EINVAL); ++ ++ ifindex = nla_get_u32(attrs[OVPN_ATTR_IFINDEX]); ++ ++ dev = dev_get_by_index(net, ifindex); ++ if (!dev) ++ return ERR_PTR(-ENODEV); ++ ++ if (!ovpn_dev_is_valid(dev)) ++ goto err_put_dev; ++ ++ return dev; ++ ++err_put_dev: ++ dev_put(dev); ++ ++ return ERR_PTR(-EINVAL); ++} ++ ++/** ++ * ovpn_pre_doit() - Prepare ovpn genl doit request ++ * @ops: requested netlink operation ++ * @skb: Netlink message with request data ++ * @info: receiver information ++ * ++ * Return: 0 on success or negative error number in case of failure ++ */ ++static int ovpn_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, ++ struct genl_info *info) ++{ ++ struct net *net = genl_info_net(info); ++ struct net_device *dev; ++ ++ dev = ovpn_get_dev_from_attrs(net, info->attrs); ++ if (IS_ERR(dev)) ++ return PTR_ERR(dev); ++ ++ info->user_ptr[0] = netdev_priv(dev); ++ ++ return 0; ++} ++ ++/** ++ * ovpn_post_doit() - complete ovpn genl doit request ++ * @ops: requested netlink operation ++ * @skb: Netlink message with request data ++ * @info: receiver information ++ */ ++static void ovpn_post_doit(const struct genl_ops *ops, struct sk_buff *skb, ++ struct genl_info *info) ++{ ++ struct ovpn_struct *ovpn; ++ ++ ovpn = info->user_ptr[0]; ++ dev_put(ovpn->dev); ++} ++ ++static int ovpn_netlink_get_key_dir(struct genl_info *info, struct nlattr *key, ++ enum ovpn_cipher_alg cipher, ++ struct ovpn_key_direction *dir) ++{ ++ struct nlattr *attr, *attrs[OVPN_KEY_DIR_ATTR_MAX + 1]; ++ int ret; ++ ++ ret = nla_parse_nested(attrs, OVPN_KEY_DIR_ATTR_MAX, key, NULL, info->extack); ++ if (ret) ++ return ret; ++ ++ switch (cipher) { ++ case OVPN_CIPHER_ALG_AES_GCM: ++ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: ++ attr = attrs[OVPN_KEY_DIR_ATTR_CIPHER_KEY]; ++ if (!attr) ++ return -EINVAL; ++ ++ dir->cipher_key = nla_data(attr); ++ dir->cipher_key_size = nla_len(attr); ++ ++ attr = attrs[OVPN_KEY_DIR_ATTR_NONCE_TAIL]; ++ /* These algorithms require a 96bit nonce, ++ * Construct it by combining 4-bytes packet id and ++ * 8-bytes nonce-tail from userspace ++ */ ++ if (!attr) ++ return -EINVAL; ++ ++ dir->nonce_tail = nla_data(attr); ++ dir->nonce_tail_size = nla_len(attr); ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int ovpn_netlink_new_key(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_NEW_KEY_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer_key_reset pkr; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_NEW_KEY]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_NEW_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_KEY], ++ NULL, info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_NEW_KEY_ATTR_PEER_ID] || ++ !attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT] || ++ !attrs[OVPN_NEW_KEY_ATTR_KEY_ID] || ++ !attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG] || ++ !attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] || ++ !attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_NEW_KEY_ATTR_PEER_ID]); ++ pkr.slot = nla_get_u8(attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT]); ++ pkr.key.key_id = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_KEY_ID]); ++ ++ pkr.key.cipher_alg = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG]); ++ ++ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY], ++ pkr.key.cipher_alg, &pkr.key.encrypt); ++ if (ret < 0) ++ return ret; ++ ++ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY], ++ pkr.key.cipher_alg, &pkr.key.decrypt); ++ if (ret < 0) ++ return ret; ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) { ++ netdev_dbg(ovpn->dev, "%s: no peer with id %u to set key for\n", __func__, peer_id); ++ return -ENOENT; ++ } ++ ++ mutex_lock(&peer->crypto.mutex); ++ ret = ovpn_crypto_state_reset(&peer->crypto, &pkr); ++ if (ret < 0) { ++ netdev_dbg(ovpn->dev, "%s: cannot install new key for peer %u\n", __func__, ++ peer_id); ++ goto unlock; ++ } ++ ++ netdev_dbg(ovpn->dev, "%s: new key installed (id=%u) for peer %u\n", __func__, ++ pkr.key.key_id, peer_id); ++unlock: ++ mutex_unlock(&peer->crypto.mutex); ++ ovpn_peer_put(peer); ++ return ret; ++} ++ ++static int ovpn_netlink_del_key(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_DEL_KEY_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ enum ovpn_key_slot slot; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_DEL_KEY]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_DEL_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_KEY], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_DEL_KEY_ATTR_PEER_ID] || !attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_DEL_KEY_ATTR_PEER_ID]); ++ slot = nla_get_u8(attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]); ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ ovpn_crypto_key_slot_delete(&peer->crypto, slot); ++ ovpn_peer_put(peer); ++ ++ return 0; ++} ++ ++static int ovpn_netlink_swap_keys(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_SWAP_KEYS_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_SWAP_KEYS]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_SWAP_KEYS_ATTR_MAX, info->attrs[OVPN_ATTR_SWAP_KEYS], ++ NULL, info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]); ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ ovpn_crypto_key_slots_swap(&peer->crypto); ++ ovpn_peer_put(peer); ++ ++ return 0; ++} ++ ++static int ovpn_netlink_new_peer(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_NEW_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct sockaddr_storage *ss = NULL; ++ struct sockaddr_in mapped; ++ struct sockaddr_in6 *in6; ++ struct ovpn_peer *peer; ++ size_t sa_len, ip_len; ++ struct socket *sock; ++ u8 *local_ip = NULL; ++ u32 sockfd, id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_NEW_PEER]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_NEW_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_NEW_PEER_ATTR_PEER_ID] || !attrs[OVPN_NEW_PEER_ATTR_SOCKET] || ++ (!attrs[OVPN_NEW_PEER_ATTR_IPV4] && !attrs[OVPN_NEW_PEER_ATTR_IPV6])) { ++ netdev_dbg(ovpn->dev, "%s: basic attributes missing\n", __func__); ++ return -EINVAL; ++ } ++ ++ /* lookup the fd in the kernel table and extract the socket object */ ++ sockfd = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_SOCKET]); ++ /* sockfd_lookup() increases sock's refcounter */ ++ sock = sockfd_lookup(sockfd, &ret); ++ if (!sock) { ++ netdev_dbg(ovpn->dev, "%s: cannot lookup peer socket (fd=%u): %d\n", __func__, ++ sockfd, ret); ++ return -ENOTSOCK; ++ } ++ ++ /* Only when using UDP as transport protocol the remote endpoint must be configured ++ * so that ovpn-dco knows where to send packets to. ++ * ++ * In case of TCP, the socket is connected to the peer and ovpn-dco will just send bytes ++ * over it, without the need to specify a destination. ++ */ ++ if (sock->sk->sk_protocol == IPPROTO_UDP) { ++ ret = -EINVAL; ++ ++ if (!attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]) { ++ netdev_err(ovpn->dev, "%s: cannot add UDP peer with no remote endpoint\n", ++ __func__); ++ goto sockfd_release; ++ } ++ ++ ss = nla_data(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); ++ sa_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); ++ switch (sa_len) { ++ case sizeof(struct sockaddr_in): ++ if (ss->ss_family == AF_INET) ++ /* valid sockaddr */ ++ break; ++ ++ netdev_err(ovpn->dev, "%s: remote sockaddr_in has invalid family\n", ++ __func__); ++ goto sockfd_release; ++ case sizeof(struct sockaddr_in6): ++ if (ss->ss_family == AF_INET6) ++ /* valid sockaddr */ ++ break; ++ ++ netdev_err(ovpn->dev, "%s: remote sockaddr_in6 has invalid family\n", ++ __func__); ++ goto sockfd_release; ++ default: ++ netdev_err(ovpn->dev, "%s: invalid size for sockaddr\n", __func__); ++ goto sockfd_release; ++ } ++ ++ if (ss->ss_family == AF_INET6) { ++ in6 = (struct sockaddr_in6 *)ss; ++ ++ if (ipv6_addr_type(&in6->sin6_addr) & IPV6_ADDR_MAPPED) { ++ mapped.sin_family = AF_INET; ++ mapped.sin_addr.s_addr = in6->sin6_addr.s6_addr32[3]; ++ mapped.sin_port = in6->sin6_port; ++ ss = (struct sockaddr_storage *)&mapped; ++ } ++ } ++ ++ /* When using UDP we may be talking over socket bound to 0.0.0.0/::. ++ * In this case, if the host has multiple IPs, we need to make sure ++ * that outgoing traffic has as source IP the same address that the ++ * peer is using to reach us. ++ * ++ * Since early control packets were all forwarded to userspace, we ++ * need the latter to tell us what IP has to be used. ++ */ ++ if (attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]) { ++ ip_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); ++ local_ip = nla_data(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); ++ ++ if (ip_len == sizeof(struct in_addr)) { ++ if (ss->ss_family != AF_INET) { ++ netdev_dbg(ovpn->dev, ++ "%s: the specified local IP is IPv4, but the peer endpoint is not\n", ++ __func__); ++ goto sockfd_release; ++ } ++ } else if (ip_len == sizeof(struct in6_addr)) { ++ bool is_mapped = ipv6_addr_type((struct in6_addr *)local_ip) & ++ IPV6_ADDR_MAPPED; ++ ++ if (ss->ss_family != AF_INET6 && !is_mapped) { ++ netdev_dbg(ovpn->dev, ++ "%s: the specified local IP is IPv6, but the peer endpoint is not\n", ++ __func__); ++ goto sockfd_release; ++ } ++ ++ if (is_mapped) ++ /* this is an IPv6-mapped IPv4 address, therefore extract ++ * the actual v4 address from the last 4 bytes ++ */ ++ local_ip += 12; ++ } else { ++ netdev_dbg(ovpn->dev, ++ "%s: invalid length %zu for local IP\n", __func__, ++ ip_len); ++ goto sockfd_release; ++ } ++ } ++ ++ /* sanity checks passed */ ++ ret = 0; ++ } ++ ++ id = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_PEER_ID]); ++ peer = ovpn_peer_new(ovpn, ss, sock, id, local_ip); ++ if (IS_ERR(peer)) { ++ netdev_err(ovpn->dev, "%s: cannot create new peer object for peer %u %pIScp\n", ++ __func__, id, ss); ++ ret = PTR_ERR(peer); ++ goto sockfd_release; ++ } ++ ++ if (attrs[OVPN_NEW_PEER_ATTR_IPV4]) { ++ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV4]) != sizeof(struct in_addr)) { ++ ret = -EINVAL; ++ goto peer_release; ++ } ++ ++ peer->vpn_addrs.ipv4.s_addr = nla_get_be32(attrs[OVPN_NEW_PEER_ATTR_IPV4]); ++ } ++ ++ if (attrs[OVPN_NEW_PEER_ATTR_IPV6]) { ++ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV6]) != sizeof(struct in6_addr)) { ++ ret = -EINVAL; ++ goto peer_release; ++ } ++ ++ memcpy(&peer->vpn_addrs.ipv6, nla_data(attrs[OVPN_NEW_PEER_ATTR_IPV6]), ++ sizeof(struct in6_addr)); ++ } ++ ++ netdev_dbg(ovpn->dev, ++ "%s: adding peer with endpoint=%pIScp/%s id=%u VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n", ++ __func__, ss, sock->sk->sk_prot_creator->name, peer->id, ++ &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6); ++ ++ ret = ovpn_peer_add(ovpn, peer); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot add new peer (id=%u) to hashtable: %d\n", ++ __func__, peer->id, ret); ++ goto peer_release; ++ } ++ ++ return 0; ++ ++peer_release: ++ /* release right away because peer is not really used in any context */ ++ ovpn_peer_release(peer); ++ return ret; ++ ++sockfd_release: ++ sockfd_put(sock); ++ return ret; ++} ++ ++static int ovpn_netlink_set_peer(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ u32 peer_id, interv, timeout; ++ bool keepalive_set = false; ++ struct ovpn_peer *peer; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_SET_PEER]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_SET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_SET_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_SET_PEER_ATTR_PEER_ID]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_PEER_ID]); ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ /* when setting the keepalive, both parameters have to be configured */ ++ if (attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] && ++ attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]) { ++ keepalive_set = true; ++ interv = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL]); ++ timeout = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]); ++ } ++ ++ if (keepalive_set) ++ ovpn_peer_keepalive_set(peer, interv, timeout); ++ ++ ovpn_peer_put(peer); ++ return 0; ++} ++ ++static int ovpn_netlink_send_peer(struct sk_buff *skb, const struct ovpn_peer *peer, u32 portid, ++ u32 seq, int flags) ++{ ++ const struct ovpn_bind *bind; ++ struct nlattr *attr; ++ void *hdr; ++ ++ hdr = genlmsg_put(skb, portid, seq, &ovpn_netlink_family, flags, OVPN_CMD_GET_PEER); ++ if (!hdr) { ++ netdev_dbg(peer->ovpn->dev, "%s: cannot create message header\n", __func__); ++ return -EMSGSIZE; ++ } ++ ++ attr = nla_nest_start(skb, OVPN_ATTR_GET_PEER); ++ if (!attr) { ++ netdev_dbg(peer->ovpn->dev, "%s: cannot create submessage\n", __func__); ++ goto err; ++ } ++ ++ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_PEER_ID, peer->id)) ++ goto err; ++ ++ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV4, sizeof(peer->vpn_addrs.ipv4), ++ &peer->vpn_addrs.ipv4)) ++ goto err; ++ ++ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV6, sizeof(peer->vpn_addrs.ipv6), ++ &peer->vpn_addrs.ipv6)) ++ goto err; ++ ++ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, ++ peer->keepalive_interval) || ++ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, ++ peer->keepalive_timeout)) ++ goto err; ++ ++ rcu_read_lock(); ++ bind = rcu_dereference(peer->bind); ++ if (bind) { ++ if (bind->sa.in4.sin_family == AF_INET) { ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, ++ sizeof(bind->sa.in4), &bind->sa.in4) || ++ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, ++ sizeof(bind->local.ipv4), &bind->local.ipv4)) ++ goto err_unlock; ++ } else if (bind->sa.in4.sin_family == AF_INET6) { ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, ++ sizeof(bind->sa.in6), &bind->sa.in6) || ++ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, ++ sizeof(bind->local.ipv6), &bind->local.ipv6)) ++ goto err_unlock; ++ } ++ } ++ rcu_read_unlock(); ++ ++ if (nla_put_net16(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, ++ inet_sk(peer->sock->sock->sk)->inet_sport) || ++ /* RX stats */ ++ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_RX_BYTES, ++ atomic64_read(&peer->stats.rx.bytes), ++ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || ++ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, ++ atomic_read(&peer->stats.rx.packets)) || ++ /* TX stats */ ++ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_TX_BYTES, ++ atomic64_read(&peer->stats.tx.bytes), ++ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || ++ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, ++ atomic_read(&peer->stats.tx.packets))) ++ goto err; ++ ++ nla_nest_end(skb, attr); ++ genlmsg_end(skb, hdr); ++ ++ return 0; ++err_unlock: ++ rcu_read_unlock(); ++err: ++ genlmsg_cancel(skb, hdr); ++ return -EMSGSIZE; ++} ++ ++static int ovpn_netlink_get_peer(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer *peer; ++ struct sk_buff *msg; ++ u32 peer_id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_GET_PEER]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_GET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_GET_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_GET_PEER_ATTR_PEER_ID]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_GET_PEER_ATTR_PEER_ID]); ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); ++ if (!msg) ++ return -ENOMEM; ++ ++ ret = ovpn_netlink_send_peer(msg, peer, info->snd_portid, info->snd_seq, 0); ++ if (ret < 0) { ++ nlmsg_free(msg); ++ goto err; ++ } ++ ++ ret = genlmsg_reply(msg, info); ++err: ++ ovpn_peer_put(peer); ++ return ret; ++} ++ ++static int ovpn_netlink_dump_done(struct netlink_callback *cb) ++{ ++ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; ++ ++ dev_put(ovpn->dev); ++ return 0; ++} ++ ++static int ovpn_netlink_dump_prepare(struct netlink_callback *cb) ++{ ++ struct net *netns = sock_net(cb->skb->sk); ++ struct nlattr **attrbuf; ++ struct net_device *dev; ++ int ret; ++ ++ attrbuf = kcalloc(OVPN_ATTR_MAX + 1, sizeof(*attrbuf), GFP_KERNEL); ++ if (!attrbuf) ++ return -ENOMEM; ++ ++ ret = nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrbuf, OVPN_ATTR_MAX, ++ ovpn_netlink_policy, NULL); ++ if (ret < 0) ++ goto err; ++ ++ dev = ovpn_get_dev_from_attrs(netns, attrbuf); ++ if (IS_ERR(dev)) { ++ ret = PTR_ERR(dev); ++ goto err; ++ } ++ ++ cb->args[0] = (long)netdev_priv(dev); ++ ret = 0; ++err: ++ kfree(attrbuf); ++ return ret; ++} ++ ++static int ovpn_netlink_dump_peers(struct sk_buff *skb, struct netlink_callback *cb) ++{ ++ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; ++ int ret, bkt, last_idx = cb->args[1], dumped = 0; ++ struct ovpn_peer *peer; ++ ++ if (!ovpn) { ++ ret = ovpn_netlink_dump_prepare(cb); ++ if (ret < 0) { ++ netdev_dbg(ovpn->dev, "%s: cannot prepare for dump: %d\n", __func__, ret); ++ return ret; ++ } ++ ++ ovpn = (struct ovpn_struct *)cb->args[0]; ++ } ++ ++ rcu_read_lock(); ++ hash_for_each_rcu(ovpn->peers.by_id, bkt, peer, hash_entry_id) { ++ /* skip already dumped peers that were dumped by previous invocations */ ++ if (last_idx > 0) { ++ last_idx--; ++ continue; ++ } ++ ++ if (ovpn_netlink_send_peer(skb, peer, NETLINK_CB(cb->skb).portid, ++ cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) ++ break; ++ ++ /* count peers being dumped during this invocation */ ++ dumped++; ++ } ++ rcu_read_unlock(); ++ ++ /* sum up peers dumped in this message, so that at the next invocation ++ * we can continue from where we left ++ */ ++ cb->args[1] += dumped; ++ ++ return skb->len; ++} ++ ++static int ovpn_netlink_del_peer(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_DEL_PEER]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_DEL_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_DEL_PEER_ATTR_PEER_ID]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_DEL_PEER_ATTR_PEER_ID]); ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ netdev_dbg(ovpn->dev, "%s: peer id=%u\n", __func__, peer->id); ++ ret = ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_USERSPACE); ++ ovpn_peer_put(peer); ++ ++ return ret; ++} ++ ++static int ovpn_netlink_register_packet(struct sk_buff *skb, ++ struct genl_info *info) ++{ ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ ++ /* only one registered process per interface is allowed for now */ ++ if (ovpn->registered_nl_portid_set) { ++ netdev_dbg(ovpn->dev, "%s: userspace listener already registered\n", __func__); ++ return -EBUSY; ++ } ++ ++ netdev_dbg(ovpn->dev, "%s: registering userspace at %u\n", __func__, info->snd_portid); ++ ++ ovpn->registered_nl_portid = info->snd_portid; ++ ovpn->registered_nl_portid_set = true; ++ ++ return 0; ++} ++ ++static int ovpn_netlink_packet(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_PACKET_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ const u8 *packet; ++ u32 peer_id; ++ size_t len; ++ u8 opcode; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_PACKET]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_PACKET_ATTR_MAX, info->attrs[OVPN_ATTR_PACKET], ++ NULL, info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_PACKET_ATTR_PACKET] || !attrs[OVPN_PACKET_ATTR_PEER_ID]) { ++ netdev_dbg(ovpn->dev, "received netlink packet with no payload\n"); ++ return -EINVAL; ++ } ++ ++ peer_id = nla_get_u32(attrs[OVPN_PACKET_ATTR_PEER_ID]); ++ ++ len = nla_len(attrs[OVPN_PACKET_ATTR_PACKET]); ++ ++ if (len < 4 || len > ovpn->dev->mtu) { ++ netdev_dbg(ovpn->dev, "%s: invalid packet size %zu (min is 4, max is MTU: %u)\n", ++ __func__, len, ovpn->dev->mtu); ++ return -EINVAL; ++ } ++ ++ packet = nla_data(attrs[OVPN_PACKET_ATTR_PACKET]); ++ opcode = ovpn_opcode_from_byte(packet[0]); ++ ++ /* reject data packets from userspace as they could lead to IV reuse */ ++ if (opcode == OVPN_DATA_V1 || opcode == OVPN_DATA_V2) { ++ netdev_dbg(ovpn->dev, "%s: rejecting data packet from userspace (opcode=%u)\n", ++ __func__, opcode); ++ return -EINVAL; ++ } ++ ++ netdev_dbg(ovpn->dev, "%s: sending userspace packet to peer %u...\n", __func__, peer_id); ++ ++ return ovpn_send_data(ovpn, peer_id, packet, len); ++} ++ ++static const struct genl_ops ovpn_netlink_ops[] = { ++ { ++ .cmd = OVPN_CMD_NEW_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_new_peer, ++ }, ++ { ++ .cmd = OVPN_CMD_SET_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_set_peer, ++ }, ++ { ++ .cmd = OVPN_CMD_DEL_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_del_peer, ++ }, ++ { ++ .cmd = OVPN_CMD_GET_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_get_peer, ++ .dumpit = ovpn_netlink_dump_peers, ++ .done = ovpn_netlink_dump_done, ++ }, ++ { ++ .cmd = OVPN_CMD_NEW_KEY, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_new_key, ++ }, ++ { ++ .cmd = OVPN_CMD_DEL_KEY, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_del_key, ++ }, ++ { ++ .cmd = OVPN_CMD_SWAP_KEYS, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_swap_keys, ++ }, ++ { ++ .cmd = OVPN_CMD_REGISTER_PACKET, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_register_packet, ++ }, ++ { ++ .cmd = OVPN_CMD_PACKET, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_packet, ++ }, ++}; ++ ++static struct genl_family ovpn_netlink_family __ro_after_init = { ++ .hdrsize = 0, ++ .name = OVPN_NL_NAME, ++ .version = 1, ++ .maxattr = OVPN_ATTR_MAX, ++ .policy = ovpn_netlink_policy, ++ .netnsok = true, ++ .pre_doit = ovpn_pre_doit, ++ .post_doit = ovpn_post_doit, ++ .module = THIS_MODULE, ++ .ops = ovpn_netlink_ops, ++ .n_ops = ARRAY_SIZE(ovpn_netlink_ops), ++ .mcgrps = ovpn_netlink_mcgrps, ++ .n_mcgrps = ARRAY_SIZE(ovpn_netlink_mcgrps), ++}; ++ ++int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer) ++{ ++ struct sk_buff *msg; ++ struct nlattr *attr; ++ void *hdr; ++ int ret; ++ ++ netdev_info(peer->ovpn->dev, "%s: deleting peer with id %u, reason %d\n", ++ peer->ovpn->dev->name, peer->id, peer->delete_reason); ++ ++ msg = nlmsg_new(100, GFP_KERNEL); ++ if (!msg) ++ return -ENOMEM; ++ ++ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, ++ OVPN_CMD_DEL_PEER); ++ if (!hdr) { ++ ret = -ENOBUFS; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, peer->ovpn->dev->ifindex)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ attr = nla_nest_start(msg, OVPN_ATTR_DEL_PEER); ++ if (!attr) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u8(msg, OVPN_DEL_PEER_ATTR_REASON, peer->delete_reason)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u32(msg, OVPN_DEL_PEER_ATTR_PEER_ID, peer->id)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ nla_nest_end(msg, attr); ++ ++ genlmsg_end(msg, hdr); ++ ++ genlmsg_multicast_netns(&ovpn_netlink_family, dev_net(peer->ovpn->dev), ++ msg, 0, OVPN_MCGRP_PEERS, GFP_KERNEL); ++ ++ return 0; ++ ++err_free_msg: ++ nlmsg_free(msg); ++ return ret; ++} ++ ++int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, ++ const u8 *buf, size_t len) ++{ ++ struct nlattr *attr; ++ struct sk_buff *msg; ++ void *hdr; ++ int ret; ++ ++ if (!ovpn->registered_nl_portid_set) { ++ net_warn_ratelimited("%s: no userspace listener\n", __func__); ++ return 0; ++ } ++ ++ netdev_dbg(ovpn->dev, "%s: sending packet to userspace, len: %zd\n", __func__, len); ++ ++ msg = nlmsg_new(100 + len, GFP_ATOMIC); ++ if (!msg) ++ return -ENOMEM; ++ ++ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, ++ OVPN_CMD_PACKET); ++ if (!hdr) { ++ ret = -ENOBUFS; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, ovpn->dev->ifindex)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ attr = nla_nest_start(msg, OVPN_ATTR_PACKET); ++ if (!attr) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put(msg, OVPN_PACKET_ATTR_PACKET, len, buf)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u32(msg, OVPN_PACKET_ATTR_PEER_ID, peer->id)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ nla_nest_end(msg, attr); ++ ++ genlmsg_end(msg, hdr); ++ ++ return genlmsg_unicast(dev_net(ovpn->dev), msg, ++ ovpn->registered_nl_portid); ++ ++err_free_msg: ++ nlmsg_free(msg); ++ return ret; ++} ++ ++static int ovpn_netlink_notify(struct notifier_block *nb, unsigned long state, ++ void *_notify) ++{ ++ struct netlink_notify *notify = _notify; ++ struct ovpn_struct *ovpn; ++ struct net_device *dev; ++ struct net *netns; ++ bool found = false; ++ ++ if (state != NETLINK_URELEASE || notify->protocol != NETLINK_GENERIC) ++ return NOTIFY_DONE; ++ ++ rcu_read_lock(); ++ for_each_net_rcu(netns) { ++ for_each_netdev_rcu(netns, dev) { ++ if (!ovpn_dev_is_valid(dev)) ++ continue; ++ ++ ovpn = netdev_priv(dev); ++ if (notify->portid != ovpn->registered_nl_portid) ++ continue; ++ ++ found = true; ++ netdev_dbg(ovpn->dev, "%s: deregistering userspace listener\n", __func__); ++ ovpn->registered_nl_portid_set = false; ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ ++ /* if no interface matched our purposes, pass the notification along */ ++ if (!found) ++ return NOTIFY_DONE; ++ ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block ovpn_netlink_notifier = { ++ .notifier_call = ovpn_netlink_notify, ++}; ++ ++int ovpn_netlink_init(struct ovpn_struct *ovpn) ++{ ++ ovpn->registered_nl_portid_set = false; ++ ++ return 0; ++} ++ ++/** ++ * ovpn_netlink_register() - register the ovpn genl netlink family ++ */ ++int __init ovpn_netlink_register(void) ++{ ++ int ret; ++ ++ ret = genl_register_family(&ovpn_netlink_family); ++ if (ret) ++ return ret; ++ ++ ret = netlink_register_notifier(&ovpn_netlink_notifier); ++ if (ret) ++ goto err; ++ ++ return 0; ++err: ++ genl_unregister_family(&ovpn_netlink_family); ++ return ret; ++} ++ ++/** ++ * ovpn_netlink_unregister() - unregister the ovpn genl netlink family ++ */ ++void __exit ovpn_netlink_unregister(void) ++{ ++ netlink_unregister_notifier(&ovpn_netlink_notifier); ++ genl_unregister_family(&ovpn_netlink_family); ++} +diff --git a/drivers/net/ovpn-dco/netlink.h b/drivers/net/ovpn-dco/netlink.h +new file mode 100644 +index 000000000000..843daf052c03 +--- /dev/null ++++ b/drivers/net/ovpn-dco/netlink.h +@@ -0,0 +1,22 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_NETLINK_H_ ++#define _NET_OVPN_DCO_NETLINK_H_ ++ ++struct ovpn_struct; ++struct ovpn_peer; ++ ++int ovpn_netlink_init(struct ovpn_struct *ovpn); ++int ovpn_netlink_register(void); ++void ovpn_netlink_unregister(void); ++int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, ++ const u8 *buf, size_t len); ++int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer); ++ ++#endif /* _NET_OVPN_DCO_NETLINK_H_ */ +diff --git a/drivers/net/ovpn-dco/ovpn.c b/drivers/net/ovpn-dco/ovpn.c +new file mode 100644 +index 000000000000..66c019174f5e +--- /dev/null ++++ b/drivers/net/ovpn-dco/ovpn.c +@@ -0,0 +1,600 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "bind.h" ++#include "netlink.h" ++#include "ovpn.h" ++#include "sock.h" ++#include "peer.h" ++#include "stats.h" ++#include "proto.h" ++#include "crypto.h" ++#include "crypto_aead.h" ++#include "skb.h" ++#include "tcp.h" ++#include "udp.h" ++ ++#include ++#include ++ ++static const unsigned char ovpn_keepalive_message[] = { ++ 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, ++ 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 ++}; ++ ++static const unsigned char ovpn_explicit_exit_notify_message[] = { ++ 0x28, 0x7f, 0x34, 0x6b, 0xd4, 0xef, 0x7a, 0x81, ++ 0x2d, 0x56, 0xb8, 0xd3, 0xaf, 0xc5, 0x45, 0x9c, ++ 6 // OCC_EXIT ++}; ++ ++/* Is keepalive message? ++ * Assumes that single byte at skb->data is defined. ++ */ ++static bool ovpn_is_keepalive(struct sk_buff *skb) ++{ ++ if (*skb->data != OVPN_KEEPALIVE_FIRST_BYTE) ++ return false; ++ ++ if (!pskb_may_pull(skb, sizeof(ovpn_keepalive_message))) ++ return false; ++ ++ return !memcmp(skb->data, ovpn_keepalive_message, ++ sizeof(ovpn_keepalive_message)); ++} ++ ++int ovpn_struct_init(struct net_device *dev) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ int err; ++ ++ memset(ovpn, 0, sizeof(*ovpn)); ++ ++ ovpn->dev = dev; ++ ++ err = ovpn_netlink_init(ovpn); ++ if (err < 0) ++ return err; ++ ++ spin_lock_init(&ovpn->lock); ++ spin_lock_init(&ovpn->peers.lock); ++ ++ ovpn->crypto_wq = alloc_workqueue("ovpn-crypto-wq-%s", ++ WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, ++ dev->name); ++ if (!ovpn->crypto_wq) ++ return -ENOMEM; ++ ++ ovpn->events_wq = alloc_workqueue("ovpn-event-wq-%s", WQ_MEM_RECLAIM, 0, dev->name); ++ if (!ovpn->events_wq) ++ return -ENOMEM; ++ ++ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); ++ if (!dev->tstats) ++ return -ENOMEM; ++ ++ err = security_tun_dev_alloc_security(&ovpn->security); ++ if (err < 0) ++ return err; ++ ++ /* kernel -> userspace tun queue length */ ++ ovpn->max_tun_queue_len = OVPN_MAX_TUN_QUEUE_LEN; ++ ++ return 0; ++} ++ ++/* Called after decrypt to write IP packet to tun netdev. ++ * This method is expected to manage/free skb. ++ */ ++static void tun_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ /* packet integrity was verified on the VPN layer - no need to perform ++ * any additional check along the stack ++ */ ++ skb->ip_summed = CHECKSUM_UNNECESSARY; ++ skb->csum_level = ~0; ++ ++ /* skb hash for transport packet no longer valid after decapsulation */ ++ skb_clear_hash(skb); ++ ++ /* post-decrypt scrub -- prepare to inject encapsulated packet onto tun ++ * interface, based on __skb_tunnel_rx() in dst.h ++ */ ++ skb->dev = peer->ovpn->dev; ++ skb_set_queue_mapping(skb, 0); ++ skb_scrub_packet(skb, true); ++ ++ skb_reset_network_header(skb); ++ skb_reset_transport_header(skb); ++ skb_probe_transport_header(skb); ++ skb_reset_inner_headers(skb); ++ ++ /* update per-cpu RX stats with the stored size of encrypted packet */ ++ ++ /* we are in softirq context - hence no locking nor disable preemption needed */ ++ dev_sw_netstats_rx_add(peer->ovpn->dev, OVPN_SKB_CB(skb)->rx_stats_size); ++ ++ /* cause packet to be "received" by tun interface */ ++ napi_gro_receive(&peer->napi, skb); ++} ++ ++int ovpn_napi_poll(struct napi_struct *napi, int budget) ++{ ++ struct ovpn_peer *peer = container_of(napi, struct ovpn_peer, napi); ++ struct sk_buff *skb; ++ int work_done = 0; ++ ++ if (unlikely(budget <= 0)) ++ return 0; ++ /* this function should schedule at most 'budget' number of ++ * packets for delivery to the tun interface. ++ * If in the queue we have more packets than what allowed by the ++ * budget, the next polling will take care of those ++ */ ++ while ((work_done < budget) && ++ (skb = ptr_ring_consume_bh(&peer->netif_rx_ring))) { ++ tun_netdev_write(peer, skb); ++ work_done++; ++ } ++ ++ if (work_done < budget) ++ napi_complete_done(napi, work_done); ++ ++ return work_done; ++} ++ ++static int ovpn_transport_to_userspace(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, ++ struct sk_buff *skb) ++{ ++ int ret; ++ ++ ret = skb_linearize(skb); ++ if (ret < 0) ++ return ret; ++ ++ ret = ovpn_netlink_send_packet(ovpn, peer, skb->data, skb->len); ++ if (ret < 0) ++ return ret; ++ ++ consume_skb(skb); ++ return 0; ++} ++ ++/* Entry point for processing an incoming packet (in skb form) ++ * ++ * Enqueue the packet and schedule RX consumer. ++ * Reference to peer is dropped only in case of success. ++ * ++ * Return 0 if the packet was handled (and consumed) ++ * Return <0 in case of error (return value is error code) ++ */ ++int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ int ret; ++ ++ /* At this point we know the packet is from a configured peer. ++ * DATA_V2 packets are handled in kernel space, the rest goes to user space. ++ * ++ * Packets are sent to userspace via netlink API in order to be consistenbt across ++ * UDP and TCP. ++ */ ++ if (unlikely(ovpn_opcode_from_skb(skb, 0) != OVPN_DATA_V2)) { ++ ret = ovpn_transport_to_userspace(ovpn, peer, skb); ++ if (ret < 0) ++ return ret; ++ ++ ovpn_peer_put(peer); ++ return 0; ++ } ++ ++ ret = ptr_ring_produce_bh(&peer->rx_ring, skb); ++ if (unlikely(ret < 0)) ++ return -ENOSPC; ++ ++ if (!queue_work(ovpn->crypto_wq, &peer->decrypt_work)) ++ ovpn_peer_put(peer); ++ ++ return 0; ++} ++ ++static int ovpn_decrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct ovpn_peer *allowed_peer = NULL; ++ struct ovpn_crypto_key_slot *ks; ++ unsigned int rx_stats_size; ++ __be16 proto; ++ int ret = -1; ++ u8 key_id; ++ ++ /* save original packet size for stats accounting */ ++ OVPN_SKB_CB(skb)->rx_stats_size = skb->len; ++ ++ /* get the key slot matching the key Id in the received packet */ ++ key_id = ovpn_key_id_from_skb(skb); ++ ks = ovpn_crypto_key_id_to_slot(&peer->crypto, key_id); ++ if (unlikely(!ks)) { ++ net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n", __func__, ++ peer->id, key_id); ++ goto drop; ++ } ++ ++ /* decrypt */ ++ ret = ovpn_aead_decrypt(ks, skb); ++ ++ ovpn_crypto_key_slot_put(ks); ++ ++ if (unlikely(ret < 0)) { ++ net_err_ratelimited("%s: error during decryption for peer %u, key-id %u: %d\n", ++ __func__, peer->id, key_id, ret); ++ goto drop; ++ } ++ ++ /* note event of authenticated packet received for keepalive */ ++ ovpn_peer_keepalive_recv_reset(peer); ++ ++ /* update source and destination endpoint for this peer */ ++ if (peer->sock->sock->sk->sk_protocol == IPPROTO_UDP) ++ ovpn_peer_update_local_endpoint(peer, skb); ++ ++ /* increment RX stats */ ++ rx_stats_size = OVPN_SKB_CB(skb)->rx_stats_size; ++ ovpn_peer_stats_increment_rx(&peer->stats, rx_stats_size); ++ ++ /* check if this is a valid datapacket that has to be delivered to the ++ * tun interface ++ */ ++ skb_reset_network_header(skb); ++ proto = ovpn_ip_check_protocol(skb); ++ if (unlikely(!proto)) { ++ /* check if null packet */ ++ if (unlikely(!pskb_may_pull(skb, 1))) { ++ ret = -EINVAL; ++ goto drop; ++ } ++ ++ /* check if special OpenVPN message */ ++ if (ovpn_is_keepalive(skb)) { ++ netdev_dbg(peer->ovpn->dev, "%s: ping received from peer with id %u\n", ++ __func__, peer->id); ++ /* not an error */ ++ consume_skb(skb); ++ /* inform the caller that NAPI should not be scheduled ++ * for this packet ++ */ ++ return -1; ++ } ++ ++ ret = -EPROTONOSUPPORT; ++ goto drop; ++ } ++ skb->protocol = proto; ++ ++ /* perform Reverse Path Filtering (RPF) */ ++ allowed_peer = ovpn_peer_lookup_vpn_addr(peer->ovpn, skb, true); ++ if (unlikely(allowed_peer != peer)) { ++ ret = -EPERM; ++ goto drop; ++ } ++ ++ ret = ptr_ring_produce_bh(&peer->netif_rx_ring, skb); ++drop: ++ if (likely(allowed_peer)) ++ ovpn_peer_put(allowed_peer); ++ ++ if (unlikely(ret < 0)) ++ kfree_skb(skb); ++ ++ return ret; ++} ++ ++/* pick packet from RX queue, decrypt and forward it to the tun device */ ++void ovpn_decrypt_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer; ++ struct sk_buff *skb; ++ ++ peer = container_of(work, struct ovpn_peer, decrypt_work); ++ while ((skb = ptr_ring_consume_bh(&peer->rx_ring))) { ++ if (likely(ovpn_decrypt_one(peer, skb) == 0)) { ++ /* if a packet has been enqueued for NAPI, signal ++ * availability to the networking stack ++ */ ++ local_bh_disable(); ++ napi_schedule(&peer->napi); ++ local_bh_enable(); ++ } ++ ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); ++ } ++ ovpn_peer_put(peer); ++} ++ ++static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ bool success = false; ++ int ret; ++ ++ /* get primary key to be used for encrypting data */ ++ ks = ovpn_crypto_key_slot_primary(&peer->crypto); ++ if (unlikely(!ks)) { ++ net_info_ratelimited("%s: error while retrieving primary key slot\n", __func__); ++ return false; ++ } ++ ++ if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && ++ skb_checksum_help(skb))) { ++ net_err_ratelimited("%s: cannot compute checksum for outgoing packet\n", __func__); ++ goto err; ++ } ++ ++ ovpn_peer_stats_increment_tx(&peer->stats, skb->len); ++ ++ /* encrypt */ ++ ret = ovpn_aead_encrypt(ks, skb, peer->id); ++ if (unlikely(ret < 0)) { ++ /* if we ran out of IVs we must kill the key as it can't be used anymore */ ++ if (ret == -ERANGE) { ++ netdev_warn(peer->ovpn->dev, ++ "%s: killing primary key as we ran out of IVs\n", __func__); ++ ovpn_crypto_kill_primary(&peer->crypto); ++ goto err; ++ } ++ net_err_ratelimited("%s: error during encryption for peer %u, key-id %u: %d\n", ++ __func__, peer->id, ks->key_id, ret); ++ goto err; ++ } ++ ++ success = true; ++err: ++ ovpn_crypto_key_slot_put(ks); ++ return success; ++} ++ ++/* Process packets in TX queue in a transport-specific way. ++ * ++ * UDP transport - encrypt and send across the tunnel. ++ * TCP transport - encrypt and put into TCP TX queue. ++ */ ++void ovpn_encrypt_work(struct work_struct *work) ++{ ++ struct sk_buff *skb, *curr, *next; ++ struct ovpn_peer *peer; ++ ++ peer = container_of(work, struct ovpn_peer, encrypt_work); ++ while ((skb = ptr_ring_consume_bh(&peer->tx_ring))) { ++ /* this might be a GSO-segmented skb list: process each skb ++ * independently ++ */ ++ skb_list_walk_safe(skb, curr, next) { ++ /* if one segment fails encryption, we drop the entire ++ * packet, because it does not really make sense to send ++ * only part of it at this point ++ */ ++ if (unlikely(!ovpn_encrypt_one(peer, curr))) { ++ kfree_skb_list(skb); ++ skb = NULL; ++ break; ++ } ++ } ++ ++ /* successful encryption */ ++ if (skb) { ++ skb_list_walk_safe(skb, curr, next) { ++ skb_mark_not_on_list(curr); ++ ++ switch (peer->sock->sock->sk->sk_protocol) { ++ case IPPROTO_UDP: ++ ovpn_udp_send_skb(peer->ovpn, peer, curr); ++ break; ++ case IPPROTO_TCP: ++ ovpn_tcp_send_skb(peer, curr); ++ break; ++ default: ++ /* no transport configured yet */ ++ consume_skb(skb); ++ break; ++ } ++ } ++ ++ /* note event of authenticated packet xmit for keepalive */ ++ ovpn_peer_keepalive_xmit_reset(peer); ++ } ++ ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); ++ } ++ ovpn_peer_put(peer); ++} ++ ++/* Put skb into TX queue and schedule a consumer */ ++static void ovpn_queue_skb(struct ovpn_struct *ovpn, struct sk_buff *skb, struct ovpn_peer *peer) ++{ ++ int ret; ++ ++ if (likely(!peer)) ++ peer = ovpn_peer_lookup_vpn_addr(ovpn, skb, false); ++ if (unlikely(!peer)) { ++ net_dbg_ratelimited("%s: no peer to send data to\n", ovpn->dev->name); ++ goto drop; ++ } ++ ++ ret = ptr_ring_produce_bh(&peer->tx_ring, skb); ++ if (unlikely(ret < 0)) { ++ net_err_ratelimited("%s: cannot queue packet to TX ring\n", __func__); ++ goto drop; ++ } ++ ++ if (!queue_work(ovpn->crypto_wq, &peer->encrypt_work)) ++ ovpn_peer_put(peer); ++ ++ return; ++drop: ++ if (peer) ++ ovpn_peer_put(peer); ++ kfree_skb_list(skb); ++} ++ ++/* Net device start xmit ++ */ ++netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ struct sk_buff *segments, *tmp, *curr, *next; ++ struct sk_buff_head skb_list; ++ __be16 proto; ++ int ret; ++ ++ /* reset netfilter state */ ++ nf_reset_ct(skb); ++ ++ /* verify IP header size in network packet */ ++ proto = ovpn_ip_check_protocol(skb); ++ if (unlikely(!proto || skb->protocol != proto)) { ++ net_dbg_ratelimited("%s: dropping malformed payload packet\n", ++ dev->name); ++ goto drop; ++ } ++ ++ if (skb_is_gso(skb)) { ++ segments = skb_gso_segment(skb, 0); ++ if (IS_ERR(segments)) { ++ ret = PTR_ERR(segments); ++ net_dbg_ratelimited("%s: cannot segment packet: %d\n", dev->name, ret); ++ goto drop; ++ } ++ ++ consume_skb(skb); ++ skb = segments; ++ } ++ ++ /* from this moment on, "skb" might be a list */ ++ ++ __skb_queue_head_init(&skb_list); ++ skb_list_walk_safe(skb, curr, next) { ++ skb_mark_not_on_list(curr); ++ ++ tmp = skb_share_check(curr, GFP_ATOMIC); ++ if (unlikely(!tmp)) { ++ kfree_skb_list(next); ++ net_dbg_ratelimited("%s: skb_share_check failed\n", dev->name); ++ goto drop_list; ++ } ++ ++ __skb_queue_tail(&skb_list, tmp); ++ } ++ skb_list.prev->next = NULL; ++ ++ ovpn_queue_skb(ovpn, skb_list.next, NULL); ++ ++ return NETDEV_TX_OK; ++ ++drop_list: ++ skb_queue_walk_safe(&skb_list, curr, next) ++ kfree_skb(curr); ++drop: ++ skb_tx_error(skb); ++ kfree_skb_list(skb); ++ return NET_XMIT_DROP; ++} ++ ++/* Encrypt and transmit a special message to peer, such as keepalive ++ * or explicit-exit-notify. Called from softirq context. ++ * Assumes that caller holds a reference to peer. ++ */ ++static void ovpn_xmit_special(struct ovpn_peer *peer, const void *data, ++ const unsigned int len) ++{ ++ struct ovpn_struct *ovpn; ++ struct sk_buff *skb; ++ ++ ovpn = peer->ovpn; ++ if (unlikely(!ovpn)) ++ return; ++ ++ skb = alloc_skb(256 + len, GFP_ATOMIC); ++ if (unlikely(!skb)) ++ return; ++ ++ skb_reserve(skb, 128); ++ skb->priority = TC_PRIO_BESTEFFORT; ++ memcpy(__skb_put(skb, len), data, len); ++ ++ /* increase reference counter when passing peer to sending queue */ ++ if (!ovpn_peer_hold(peer)) { ++ netdev_dbg(ovpn->dev, "%s: cannot hold peer reference for sending special packet\n", ++ __func__); ++ kfree_skb(skb); ++ return; ++ } ++ ++ ovpn_queue_skb(ovpn, skb, peer); ++} ++ ++void ovpn_keepalive_xmit(struct ovpn_peer *peer) ++{ ++ ovpn_xmit_special(peer, ovpn_keepalive_message, ++ sizeof(ovpn_keepalive_message)); ++} ++ ++/* Transmit explicit exit notification. ++ * Called from process context. ++ */ ++void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer) ++{ ++ ovpn_xmit_special(peer, ovpn_explicit_exit_notify_message, ++ sizeof(ovpn_explicit_exit_notify_message)); ++} ++ ++/* Copy buffer into skb and send it across the tunnel. ++ * ++ * For UDP transport: just sent the skb to peer ++ * For TCP transport: put skb into TX queue ++ */ ++int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len) ++{ ++ u16 skb_len = SKB_HEADER_LEN + len; ++ struct ovpn_peer *peer; ++ struct sk_buff *skb; ++ bool tcp = false; ++ int ret = 0; ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (unlikely(!peer)) { ++ netdev_dbg(ovpn->dev, "no peer to send data to\n"); ++ return -EHOSTUNREACH; ++ } ++ ++ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) { ++ skb_len += sizeof(u16); ++ tcp = true; ++ } ++ ++ skb = alloc_skb(skb_len, GFP_ATOMIC); ++ if (unlikely(!skb)) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ skb_reserve(skb, SKB_HEADER_LEN); ++ skb_put_data(skb, data, len); ++ ++ /* prepend TCP packet with size, as required by OpenVPN protocol */ ++ if (tcp) { ++ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); ++ ovpn_queue_tcp_skb(peer, skb); ++ } else { ++ ovpn_udp_send_skb(ovpn, peer, skb); ++ } ++out: ++ ovpn_peer_put(peer); ++ return ret; ++} +diff --git a/drivers/net/ovpn-dco/ovpn.h b/drivers/net/ovpn-dco/ovpn.h +new file mode 100644 +index 000000000000..9364fd5dd309 +--- /dev/null ++++ b/drivers/net/ovpn-dco/ovpn.h +@@ -0,0 +1,43 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPN_H_ ++#define _NET_OVPN_DCO_OVPN_H_ ++ ++#include "main.h" ++#include "peer.h" ++#include "sock.h" ++#include "ovpnstruct.h" ++ ++#include ++#include ++#include ++ ++struct ovpn_struct; ++struct net_device; ++ ++int ovpn_struct_init(struct net_device *dev); ++ ++u16 ovpn_select_queue(struct net_device *dev, struct sk_buff *skb, ++ struct net_device *sb_dev); ++ ++void ovpn_keepalive_xmit(struct ovpn_peer *peer); ++void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer); ++ ++netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev); ++ ++int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb); ++ ++void ovpn_encrypt_work(struct work_struct *work); ++void ovpn_decrypt_work(struct work_struct *work); ++int ovpn_napi_poll(struct napi_struct *napi, int budget); ++ ++int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len); ++ ++#endif /* _NET_OVPN_DCO_OVPN_H_ */ +diff --git a/drivers/net/ovpn-dco/ovpnstruct.h b/drivers/net/ovpn-dco/ovpnstruct.h +new file mode 100644 +index 000000000000..f9bc559609cd +--- /dev/null ++++ b/drivers/net/ovpn-dco/ovpnstruct.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNSTRUCT_H_ ++#define _NET_OVPN_DCO_OVPNSTRUCT_H_ ++ ++#include "peer.h" ++ ++#include ++#include ++#include ++ ++/* Our state per ovpn interface */ ++struct ovpn_struct { ++ /* read-mostly objects in this section */ ++ struct net_device *dev; ++ ++ /* device operation mode (i.e. P2P, MP) */ ++ enum ovpn_mode mode; ++ ++ /* protect writing to the ovpn_struct object */ ++ spinlock_t lock; ++ ++ /* workqueue used to schedule crypto work that may sleep */ ++ struct workqueue_struct *crypto_wq; ++ /* workqueue used to schedule generic event that may sleep or that need ++ * to be performed out of softirq context ++ */ ++ struct workqueue_struct *events_wq; ++ ++ /* list of known peers */ ++ struct { ++ DECLARE_HASHTABLE(by_id, 12); ++ DECLARE_HASHTABLE(by_transp_addr, 12); ++ DECLARE_HASHTABLE(by_vpn_addr, 12); ++ /* protects write access to any of the hashtables above */ ++ spinlock_t lock; ++ } peers; ++ ++ /* for p2p mode */ ++ struct ovpn_peer __rcu *peer; ++ ++ unsigned int max_tun_queue_len; ++ ++ netdev_features_t set_features; ++ ++ void *security; ++ ++ u32 registered_nl_portid; ++ bool registered_nl_portid_set; ++}; ++ ++#endif /* _NET_OVPN_DCO_OVPNSTRUCT_H_ */ +diff --git a/drivers/net/ovpn-dco/peer.c b/drivers/net/ovpn-dco/peer.c +new file mode 100644 +index 000000000000..87d3f1b34c4d +--- /dev/null ++++ b/drivers/net/ovpn-dco/peer.c +@@ -0,0 +1,906 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "ovpn.h" ++#include "bind.h" ++#include "crypto.h" ++#include "peer.h" ++#include "netlink.h" ++#include "tcp.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++static void ovpn_peer_ping(struct timer_list *t) ++{ ++ struct ovpn_peer *peer = from_timer(peer, t, keepalive_xmit); ++ ++ netdev_dbg(peer->ovpn->dev, "%s: sending ping to peer %u\n", __func__, peer->id); ++ ovpn_keepalive_xmit(peer); ++} ++ ++static void ovpn_peer_expire(struct timer_list *t) ++{ ++ struct ovpn_peer *peer = from_timer(peer, t, keepalive_recv); ++ ++ netdev_dbg(peer->ovpn->dev, "%s: peer %u expired\n", __func__, peer->id); ++ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_EXPIRED); ++} ++ ++/* Construct a new peer */ ++static struct ovpn_peer *ovpn_peer_create(struct ovpn_struct *ovpn, u32 id) ++{ ++ struct ovpn_peer *peer; ++ int ret; ++ ++ /* alloc and init peer object */ ++ peer = kzalloc(sizeof(*peer), GFP_KERNEL); ++ if (!peer) ++ return ERR_PTR(-ENOMEM); ++ ++ peer->id = id; ++ peer->halt = false; ++ peer->ovpn = ovpn; ++ ++ peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY); ++ peer->vpn_addrs.ipv6 = in6addr_any; ++ ++ RCU_INIT_POINTER(peer->bind, NULL); ++ ovpn_crypto_state_init(&peer->crypto); ++ spin_lock_init(&peer->lock); ++ kref_init(&peer->refcount); ++ ovpn_peer_stats_init(&peer->stats); ++ ++ INIT_WORK(&peer->encrypt_work, ovpn_encrypt_work); ++ INIT_WORK(&peer->decrypt_work, ovpn_decrypt_work); ++ ++ ret = dst_cache_init(&peer->dst_cache, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot initialize dst cache\n", __func__); ++ goto err; ++ } ++ ++ ret = ptr_ring_init(&peer->tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot allocate TX ring\n", __func__); ++ goto err_dst_cache; ++ } ++ ++ ret = ptr_ring_init(&peer->rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot allocate RX ring\n", __func__); ++ goto err_tx_ring; ++ } ++ ++ ret = ptr_ring_init(&peer->netif_rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot allocate NETIF RX ring\n", __func__); ++ goto err_rx_ring; ++ } ++ ++ /* configure and start NAPI */ ++ netif_tx_napi_add(ovpn->dev, &peer->napi, ovpn_napi_poll, ++ NAPI_POLL_WEIGHT); ++ napi_enable(&peer->napi); ++ ++ dev_hold(ovpn->dev); ++ ++ timer_setup(&peer->keepalive_xmit, ovpn_peer_ping, 0); ++ timer_setup(&peer->keepalive_recv, ovpn_peer_expire, 0); ++ ++ return peer; ++err_rx_ring: ++ ptr_ring_cleanup(&peer->rx_ring, NULL); ++err_tx_ring: ++ ptr_ring_cleanup(&peer->tx_ring, NULL); ++err_dst_cache: ++ dst_cache_destroy(&peer->dst_cache); ++err: ++ kfree(peer); ++ return ERR_PTR(ret); ++} ++ ++/* Reset the ovpn_sockaddr associated with a peer */ ++static int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, const struct sockaddr_storage *ss, ++ const u8 *local_ip) ++{ ++ struct ovpn_bind *bind; ++ size_t ip_len; ++ ++ /* create new ovpn_bind object */ ++ bind = ovpn_bind_from_sockaddr(ss); ++ if (IS_ERR(bind)) ++ return PTR_ERR(bind); ++ ++ if (local_ip) { ++ if (ss->ss_family == AF_INET) { ++ ip_len = sizeof(struct in_addr); ++ } else if (ss->ss_family == AF_INET6) { ++ ip_len = sizeof(struct in6_addr); ++ } else { ++ netdev_dbg(peer->ovpn->dev, "%s: invalid family for remote endpoint\n", ++ __func__); ++ kfree(bind); ++ return -EINVAL; ++ } ++ ++ memcpy(&bind->local, local_ip, ip_len); ++ } ++ ++ /* set binding */ ++ ovpn_bind_reset(peer, bind); ++ ++ return 0; ++} ++ ++void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct sockaddr_storage ss; ++ const u8 *local_ip = NULL; ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa; ++ struct ovpn_bind *bind; ++ sa_family_t family; ++ ++ rcu_read_lock(); ++ bind = rcu_dereference(peer->bind); ++ if (unlikely(!bind)) ++ goto unlock; ++ ++ if (likely(ovpn_bind_skb_src_match(bind, skb))) ++ goto unlock; ++ ++ family = skb_protocol_to_family(skb); ++ ++ if (bind->sa.in4.sin_family == family) ++ local_ip = (u8 *)&bind->local; ++ ++ switch (family) { ++ case AF_INET: ++ sa = (struct sockaddr_in *)&ss; ++ sa->sin_family = AF_INET; ++ sa->sin_addr.s_addr = ip_hdr(skb)->saddr; ++ sa->sin_port = udp_hdr(skb)->source; ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)&ss; ++ sa6->sin6_family = AF_INET6; ++ sa6->sin6_addr = ipv6_hdr(skb)->saddr; ++ sa6->sin6_port = udp_hdr(skb)->source; ++ sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); ++ break; ++ default: ++ goto unlock; ++ } ++ ++ netdev_dbg(peer->ovpn->dev, "%s: peer %d floated to %pIScp", __func__, peer->id, &ss); ++ ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip); ++unlock: ++ rcu_read_unlock(); ++} ++ ++static void ovpn_peer_timer_delete_all(struct ovpn_peer *peer) ++{ ++ del_timer_sync(&peer->keepalive_xmit); ++ del_timer_sync(&peer->keepalive_recv); ++} ++ ++static void ovpn_peer_free(struct ovpn_peer *peer) ++{ ++ ovpn_bind_reset(peer, NULL); ++ ovpn_peer_timer_delete_all(peer); ++ ++ WARN_ON(!__ptr_ring_empty(&peer->tx_ring)); ++ ptr_ring_cleanup(&peer->tx_ring, NULL); ++ WARN_ON(!__ptr_ring_empty(&peer->rx_ring)); ++ ptr_ring_cleanup(&peer->rx_ring, NULL); ++ WARN_ON(!__ptr_ring_empty(&peer->netif_rx_ring)); ++ ptr_ring_cleanup(&peer->netif_rx_ring, NULL); ++ ++ dst_cache_destroy(&peer->dst_cache); ++ ++ dev_put(peer->ovpn->dev); ++ ++ kfree(peer); ++} ++ ++static void ovpn_peer_release_rcu(struct rcu_head *head) ++{ ++ struct ovpn_peer *peer = container_of(head, struct ovpn_peer, rcu); ++ ++ ovpn_crypto_state_release(&peer->crypto); ++ ovpn_peer_free(peer); ++} ++ ++void ovpn_peer_release(struct ovpn_peer *peer) ++{ ++ napi_disable(&peer->napi); ++ netif_napi_del(&peer->napi); ++ ++ if (peer->sock) ++ ovpn_socket_put(peer->sock); ++ ++ call_rcu(&peer->rcu, ovpn_peer_release_rcu); ++} ++ ++static void ovpn_peer_delete_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, ++ delete_work); ++ ovpn_peer_release(peer); ++ ovpn_netlink_notify_del_peer(peer); ++} ++ ++/* Use with kref_put calls, when releasing refcount ++ * on ovpn_peer objects. This method should only ++ * be called from process context with config_mutex held. ++ */ ++void ovpn_peer_release_kref(struct kref *kref) ++{ ++ struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount); ++ ++ INIT_WORK(&peer->delete_work, ovpn_peer_delete_work); ++ queue_work(peer->ovpn->events_wq, &peer->delete_work); ++} ++ ++struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, ++ struct socket *sock, u32 id, uint8_t *local_ip) ++{ ++ struct ovpn_peer *peer; ++ int ret; ++ ++ /* create new peer */ ++ peer = ovpn_peer_create(ovpn, id); ++ if (IS_ERR(peer)) ++ return peer; ++ ++ if (sock->sk->sk_protocol == IPPROTO_UDP) { ++ /* a UDP peer must have a remote endpoint */ ++ if (!sa) { ++ ovpn_peer_release(peer); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ /* set peer sockaddr */ ++ ret = ovpn_peer_reset_sockaddr(peer, sa, local_ip); ++ if (ret < 0) { ++ ovpn_peer_release(peer); ++ return ERR_PTR(ret); ++ } ++ } ++ ++ peer->sock = ovpn_socket_new(sock, peer); ++ if (IS_ERR(peer->sock)) { ++ peer->sock = NULL; ++ ovpn_peer_release(peer); ++ return ERR_PTR(-ENOTSOCK); ++ } ++ ++ /* schedule initial TCP RX work only after having assigned peer->sock */ ++ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) ++ queue_work(peer->ovpn->events_wq, &peer->tcp.rx_work); ++ ++ return peer; ++} ++ ++/* Configure keepalive parameters */ ++void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) ++{ ++ u32 delta; ++ ++ netdev_dbg(peer->ovpn->dev, ++ "%s: scheduling keepalive for peer %u: interval=%u timeout=%u\n", __func__, ++ peer->id, interval, timeout); ++ ++ peer->keepalive_interval = interval; ++ if (interval > 0) { ++ delta = msecs_to_jiffies(interval * MSEC_PER_SEC); ++ mod_timer(&peer->keepalive_xmit, jiffies + delta); ++ } else { ++ del_timer(&peer->keepalive_xmit); ++ } ++ ++ peer->keepalive_timeout = timeout; ++ if (timeout) { ++ delta = msecs_to_jiffies(timeout * MSEC_PER_SEC); ++ mod_timer(&peer->keepalive_recv, jiffies + delta); ++ } else { ++ del_timer(&peer->keepalive_recv); ++ } ++} ++ ++#define ovpn_peer_index(_tbl, _key, _key_len) \ ++ (jhash(_key, _key_len, 0) % HASH_SIZE(_tbl)) \ ++ ++static struct ovpn_peer *ovpn_peer_lookup_vpn_addr4(struct hlist_head *head, __be32 *addr) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr4) { ++ if (*addr != tmp->vpn_addrs.ipv4.s_addr) ++ continue; ++ ++ if (!ovpn_peer_hold(tmp)) ++ continue; ++ ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++static struct ovpn_peer *ovpn_peer_lookup_vpn_addr6(struct hlist_head *head, struct in6_addr *addr) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ int i; ++ ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr6) { ++ for (i = 0; i < 4; i++) { ++ if (addr->s6_addr32[i] != tmp->vpn_addrs.ipv6.s6_addr32[i]) ++ continue; ++ } ++ ++ if (!ovpn_peer_hold(tmp)) ++ continue; ++ ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++/** ++ * ovpn_nexthop4() - looks up the IP of the nexthop for the given destination ++ * ++ * Looks up in the IPv4 system routing table the IO of the nexthop to be used ++ * to reach the destination passed as argument. IF no nexthop can be found, the ++ * destination itself is returned as it probably has to be used as nexthop. ++ * ++ * @ovpn: the private data representing the current VPN session ++ * @dst: the destination to be looked up ++ * ++ * Return the IP of the next hop if found or the dst itself otherwise ++ */ ++static __be32 ovpn_nexthop4(struct ovpn_struct *ovpn, __be32 dst) ++{ ++ struct rtable *rt; ++ struct flowi4 fl = { ++ .daddr = dst ++ }; ++ ++ rt = ip_route_output_flow(dev_net(ovpn->dev), &fl, NULL); ++ if (IS_ERR(rt)) { ++ net_dbg_ratelimited("%s: no route to host %pI4\n", __func__, &dst); ++ /* if we end up here this packet is probably going to be thrown away later */ ++ return dst; ++ } ++ ++ if (!rt->rt_uses_gateway) ++ goto out; ++ ++ dst = rt->rt_gw4; ++out: ++ ip_rt_put(rt); ++ return dst; ++} ++ ++/** ++ * ovpn_nexthop6() - looks up the IPv6 of the nexthop for the given destination ++ * ++ * Looks up in the IPv6 system routing table the IO of the nexthop to be used ++ * to reach the destination passed as argument. IF no nexthop can be found, the ++ * destination itself is returned as it probably has to be used as nexthop. ++ * ++ * @ovpn: the private data representing the current VPN session ++ * @dst: the destination to be looked up ++ * ++ * Return the IP of the next hop if found or the dst itself otherwise ++ */ ++static struct in6_addr ovpn_nexthop6(struct ovpn_struct *ovpn, struct in6_addr dst) ++{ ++#if IS_ENABLED(CONFIG_IPV6) ++ struct rt6_info *rt; ++ struct flowi6 fl = { ++ .daddr = dst, ++ }; ++ ++ rt = (struct rt6_info *)ipv6_stub->ipv6_dst_lookup_flow(dev_net(ovpn->dev), NULL, &fl, ++ NULL); ++ if (IS_ERR(rt)) { ++ net_dbg_ratelimited("%s: no route to host %pI6\n", __func__, &dst); ++ /* if we end up here this packet is probably going to be thrown away later */ ++ return dst; ++ } ++ ++ if (!(rt->rt6i_flags & RTF_GATEWAY)) ++ goto out; ++ ++ dst = rt->rt6i_gateway; ++out: ++ dst_release((struct dst_entry *)rt); ++#endif ++ return dst; ++} ++ ++/** ++ * ovpn_peer_lookup_vpn_addr() - Lookup peer to send skb to ++ * ++ * This function takes a tunnel packet and looks up the peer to send it to ++ * after encapsulation. The skb is expected to be the in-tunnel packet, without ++ * any OpenVPN related header. ++ * ++ * Assume that the IP header is accessible in the skb data. ++ * ++ * @ovpn: the private data representing the current VPN session ++ * @skb: the skb to extract the destination address from ++ * ++ * Return the peer if found or NULL otherwise. ++ */ ++struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, ++ bool use_src) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ struct hlist_head *head; ++ struct rt6_info *rt6i = NULL; ++ struct rtable *rt = NULL; ++ sa_family_t sa_fam; ++ struct in6_addr addr6; ++ __be32 addr4; ++ u32 index; ++ ++ /* in P2P mode, no matter the destination, packets are always sent to the single peer ++ * listening on the other side ++ */ ++ if (ovpn->mode == OVPN_MODE_P2P) { ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (likely(tmp && ovpn_peer_hold(tmp))) ++ peer = tmp; ++ rcu_read_unlock(); ++ return peer; ++ } ++ ++ sa_fam = skb_protocol_to_family(skb); ++ ++ switch (sa_fam) { ++ case AF_INET: ++ if (use_src) ++ addr4 = ip_hdr(skb)->saddr; ++ else ++ addr4 = ip_hdr(skb)->daddr; ++ addr4 = ovpn_nexthop4(ovpn, addr4); ++ ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr4, sizeof(addr4)); ++ head = &ovpn->peers.by_vpn_addr[index]; ++ ++ peer = ovpn_peer_lookup_vpn_addr4(head, &addr4); ++ break; ++ case AF_INET6: ++ if (use_src) ++ addr6 = ipv6_hdr(skb)->saddr; ++ else ++ addr6 = ipv6_hdr(skb)->daddr; ++ addr6 = ovpn_nexthop6(ovpn, addr6); ++ ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr6, sizeof(addr6)); ++ head = &ovpn->peers.by_vpn_addr[index]; ++ ++ peer = ovpn_peer_lookup_vpn_addr6(head, &addr6); ++ break; ++ } ++ ++ if (rt) ++ ip_rt_put(rt); ++ if (rt6i) ++ dst_release((struct dst_entry *)rt6i); ++ ++ return peer; ++} ++ ++static bool ovpn_peer_transp_match(struct ovpn_peer *peer, struct sockaddr_storage *ss) ++{ ++ struct ovpn_bind *bind = rcu_dereference(peer->bind); ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa4; ++ ++ if (unlikely(!bind)) ++ return false; ++ ++ if (ss->ss_family != bind->sa.in4.sin_family) ++ return false; ++ ++ switch (ss->ss_family) { ++ case AF_INET: ++ sa4 = (struct sockaddr_in *)ss; ++ if (sa4->sin_addr.s_addr != bind->sa.in4.sin_addr.s_addr) ++ return false; ++ if (sa4->sin_port != bind->sa.in4.sin_port) ++ return false; ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)ss; ++ if (memcmp(&sa6->sin6_addr, &bind->sa.in6.sin6_addr, sizeof(struct in6_addr))) ++ return false; ++ if (sa6->sin6_port != bind->sa.in6.sin6_port) ++ return false; ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++} ++ ++static bool ovpn_peer_skb_to_sockaddr(struct sk_buff *skb, struct sockaddr_storage *ss) ++{ ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa4; ++ ++ ss->ss_family = skb_protocol_to_family(skb); ++ switch (ss->ss_family) { ++ case AF_INET: ++ sa4 = (struct sockaddr_in *)ss; ++ sa4->sin_family = AF_INET; ++ sa4->sin_addr.s_addr = ip_hdr(skb)->saddr; ++ sa4->sin_port = udp_hdr(skb)->source; ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)ss; ++ sa6->sin6_family = AF_INET6; ++ sa6->sin6_addr = ipv6_hdr(skb)->saddr; ++ sa6->sin6_port = udp_hdr(skb)->source; ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++} ++ ++static struct ovpn_peer *ovpn_peer_lookup_transp_addr_p2p(struct ovpn_struct *ovpn, ++ struct sockaddr_storage *ss) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (likely(tmp && ovpn_peer_transp_match(tmp, ss) && ovpn_peer_hold(tmp))) ++ peer = tmp; ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb) ++{ ++ struct ovpn_peer *peer = NULL, *tmp; ++ struct sockaddr_storage ss = { 0 }; ++ struct hlist_head *head; ++ size_t sa_len; ++ bool found; ++ u32 index; ++ ++ if (unlikely(!ovpn_peer_skb_to_sockaddr(skb, &ss))) ++ return NULL; ++ ++ if (ovpn->mode == OVPN_MODE_P2P) ++ return ovpn_peer_lookup_transp_addr_p2p(ovpn, &ss); ++ ++ switch (ss.ss_family) { ++ case AF_INET: ++ sa_len = sizeof(struct sockaddr_in); ++ break; ++ case AF_INET6: ++ sa_len = sizeof(struct sockaddr_in6); ++ break; ++ default: ++ return NULL; ++ } ++ ++ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &ss, sa_len); ++ head = &ovpn->peers.by_transp_addr[index]; ++ ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_transp_addr) { ++ found = ovpn_peer_transp_match(tmp, &ss); ++ if (!found) ++ continue; ++ ++ if (!ovpn_peer_hold(tmp)) ++ continue; ++ ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++static struct ovpn_peer *ovpn_peer_lookup_id_p2p(struct ovpn_struct *ovpn, u32 peer_id) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp))) ++ peer = tmp; ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ struct hlist_head *head; ++ u32 index; ++ ++ if (ovpn->mode == OVPN_MODE_P2P) ++ return ovpn_peer_lookup_id_p2p(ovpn, peer_id); ++ ++ index = ovpn_peer_index(ovpn->peers.by_id, &peer_id, sizeof(peer_id)); ++ head = &ovpn->peers.by_id[index]; ++ ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_id) { ++ if (tmp->id != peer_id) ++ continue; ++ ++ if (!ovpn_peer_hold(tmp)) ++ continue; ++ ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct ovpn_bind *bind; ++ ++ rcu_read_lock(); ++ bind = rcu_dereference(peer->bind); ++ if (unlikely(!bind)) ++ goto unlock; ++ ++ switch (skb_protocol_to_family(skb)) { ++ case AF_INET: ++ if (unlikely(bind->local.ipv4.s_addr != ip_hdr(skb)->daddr)) { ++ netdev_dbg(peer->ovpn->dev, ++ "%s: learning local IPv4 for peer %d (%pI4 -> %pI4)\n", __func__, ++ peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); ++ bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; ++ } ++ break; ++ case AF_INET6: ++ if (unlikely(memcmp(&bind->local.ipv6, &ipv6_hdr(skb)->daddr, ++ sizeof(bind->local.ipv6)))) { ++ netdev_dbg(peer->ovpn->dev, ++ "%s: learning local IPv6 for peer %d (%pI6c -> %pI6c\n", ++ __func__, peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); ++ bind->local.ipv6 = ipv6_hdr(skb)->daddr; ++ } ++ break; ++ default: ++ break; ++ } ++unlock: ++ rcu_read_unlock(); ++} ++ ++static int ovpn_peer_add_mp(struct ovpn_struct *ovpn, struct ovpn_peer *peer) ++{ ++ struct sockaddr_storage sa = { 0 }; ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa4; ++ struct ovpn_bind *bind; ++ struct ovpn_peer *tmp; ++ size_t salen; ++ int ret = 0; ++ u32 index; ++ ++ spin_lock_bh(&ovpn->peers.lock); ++ /* do not add duplicates */ ++ tmp = ovpn_peer_lookup_id(ovpn, peer->id); ++ if (tmp) { ++ ovpn_peer_put(tmp); ++ ret = -EEXIST; ++ goto unlock; ++ } ++ ++ hlist_del_init_rcu(&peer->hash_entry_transp_addr); ++ bind = rcu_dereference_protected(peer->bind, true); ++ /* peers connected via UDP have bind == NULL */ ++ if (bind) { ++ switch (bind->sa.in4.sin_family) { ++ case AF_INET: ++ sa4 = (struct sockaddr_in *)&sa; ++ ++ sa4->sin_family = AF_INET; ++ sa4->sin_addr.s_addr = bind->sa.in4.sin_addr.s_addr; ++ sa4->sin_port = bind->sa.in4.sin_port; ++ salen = sizeof(*sa4); ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)&sa; ++ ++ sa6->sin6_family = AF_INET6; ++ sa6->sin6_addr = bind->sa.in6.sin6_addr; ++ sa6->sin6_port = bind->sa.in6.sin6_port; ++ salen = sizeof(*sa6); ++ break; ++ default: ++ ret = -EPROTONOSUPPORT; ++ goto unlock; ++ } ++ ++ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &sa, salen); ++ hlist_add_head_rcu(&peer->hash_entry_transp_addr, ++ &ovpn->peers.by_transp_addr[index]); ++ } ++ ++ index = ovpn_peer_index(ovpn->peers.by_id, &peer->id, sizeof(peer->id)); ++ hlist_add_head_rcu(&peer->hash_entry_id, &ovpn->peers.by_id[index]); ++ ++ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv4, ++ sizeof(peer->vpn_addrs.ipv4)); ++ hlist_add_head_rcu(&peer->hash_entry_addr4, &ovpn->peers.by_vpn_addr[index]); ++ } ++ ++ hlist_del_init_rcu(&peer->hash_entry_addr6); ++ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) { ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv6, ++ sizeof(peer->vpn_addrs.ipv6)); ++ hlist_add_head_rcu(&peer->hash_entry_addr6, &ovpn->peers.by_vpn_addr[index]); ++ } ++ ++unlock: ++ spin_unlock_bh(&ovpn->peers.lock); ++ ++ return ret; ++} ++ ++static int ovpn_peer_add_p2p(struct ovpn_struct *ovpn, struct ovpn_peer *peer) ++{ ++ struct ovpn_peer *tmp; ++ ++ spin_lock_bh(&ovpn->lock); ++ /* in p2p mode it is possible to have a single peer only, therefore the ++ * old one is released and substituted by the new one ++ */ ++ tmp = rcu_dereference(ovpn->peer); ++ if (tmp) { ++ tmp->delete_reason = OVPN_DEL_PEER_REASON_TEARDOWN; ++ ovpn_peer_put(tmp); ++ } ++ ++ rcu_assign_pointer(ovpn->peer, peer); ++ spin_unlock_bh(&ovpn->lock); ++ ++ return 0; ++} ++ ++/* assume refcounter was increased by caller */ ++int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer) ++{ ++ switch (ovpn->mode) { ++ case OVPN_MODE_MP: ++ return ovpn_peer_add_mp(ovpn, peer); ++ case OVPN_MODE_P2P: ++ return ovpn_peer_add_p2p(ovpn, peer); ++ default: ++ return -EOPNOTSUPP; ++ } ++} ++ ++static void ovpn_peer_unhash(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++{ ++ hlist_del_init_rcu(&peer->hash_entry_id); ++ hlist_del_init_rcu(&peer->hash_entry_addr4); ++ hlist_del_init_rcu(&peer->hash_entry_addr6); ++ hlist_del_init_rcu(&peer->hash_entry_transp_addr); ++ ++ ovpn_peer_put(peer); ++ peer->delete_reason = reason; ++} ++ ++static int ovpn_peer_del_mp(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++{ ++ struct ovpn_peer *tmp; ++ int ret = 0; ++ ++ spin_lock_bh(&peer->ovpn->peers.lock); ++ tmp = ovpn_peer_lookup_id(peer->ovpn, peer->id); ++ if (tmp != peer) { ++ ret = -ENOENT; ++ goto unlock; ++ } ++ ovpn_peer_unhash(peer, reason); ++ ++unlock: ++ spin_unlock_bh(&peer->ovpn->peers.lock); ++ ++ if (tmp) ++ ovpn_peer_put(tmp); ++ ++ return ret; ++} ++ ++static int ovpn_peer_del_p2p(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++{ ++ struct ovpn_peer *tmp; ++ int ret = -ENOENT; ++ ++ spin_lock_bh(&peer->ovpn->lock); ++ tmp = rcu_dereference(peer->ovpn->peer); ++ if (tmp != peer) ++ goto unlock; ++ ++ ovpn_peer_put(tmp); ++ tmp->delete_reason = reason; ++ RCU_INIT_POINTER(peer->ovpn->peer, NULL); ++ ret = 0; ++ ++unlock: ++ spin_unlock_bh(&peer->ovpn->lock); ++ ++ return ret; ++} ++ ++void ovpn_peer_release_p2p(struct ovpn_struct *ovpn) ++{ ++ struct ovpn_peer *tmp; ++ ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (!tmp) ++ goto unlock; ++ ++ ovpn_peer_del_p2p(tmp, OVPN_DEL_PEER_REASON_TEARDOWN); ++unlock: ++ rcu_read_unlock(); ++} ++ ++int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++{ ++ switch (peer->ovpn->mode) { ++ case OVPN_MODE_MP: ++ return ovpn_peer_del_mp(peer, reason); ++ case OVPN_MODE_P2P: ++ return ovpn_peer_del_p2p(peer, reason); ++ default: ++ return -EOPNOTSUPP; ++ } ++} ++ ++void ovpn_peers_free(struct ovpn_struct *ovpn) ++{ ++ struct hlist_node *tmp; ++ struct ovpn_peer *peer; ++ int bkt; ++ ++ spin_lock_bh(&ovpn->peers.lock); ++ hash_for_each_safe(ovpn->peers.by_id, bkt, tmp, peer, hash_entry_id) ++ ovpn_peer_unhash(peer, OVPN_DEL_PEER_REASON_TEARDOWN); ++ spin_unlock_bh(&ovpn->peers.lock); ++} +diff --git a/drivers/net/ovpn-dco/peer.h b/drivers/net/ovpn-dco/peer.h +new file mode 100644 +index 000000000000..b759c2c9da48 +--- /dev/null ++++ b/drivers/net/ovpn-dco/peer.h +@@ -0,0 +1,168 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNPEER_H_ ++#define _NET_OVPN_DCO_OVPNPEER_H_ ++ ++#include "addr.h" ++#include "bind.h" ++#include "sock.h" ++#include "stats.h" ++ ++#include ++#include ++#include ++ ++struct ovpn_peer { ++ struct ovpn_struct *ovpn; ++ ++ u32 id; ++ ++ struct { ++ struct in_addr ipv4; ++ struct in6_addr ipv6; ++ } vpn_addrs; ++ ++ struct hlist_node hash_entry_id; ++ struct hlist_node hash_entry_addr4; ++ struct hlist_node hash_entry_addr6; ++ struct hlist_node hash_entry_transp_addr; ++ ++ /* work objects to handle encryption/decryption of packets. ++ * these works are queued on the ovpn->crypt_wq workqueue. ++ */ ++ struct work_struct encrypt_work; ++ struct work_struct decrypt_work; ++ ++ struct ptr_ring tx_ring; ++ struct ptr_ring rx_ring; ++ struct ptr_ring netif_rx_ring; ++ ++ struct napi_struct napi; ++ ++ struct ovpn_socket *sock; ++ ++ /* state of the TCP reading. Needed to keep track of how much of a single packet has already ++ * been read from the stream and how much is missing ++ */ ++ struct { ++ struct ptr_ring tx_ring; ++ struct work_struct tx_work; ++ struct work_struct rx_work; ++ ++ u8 raw_len[sizeof(u16)]; ++ struct sk_buff *skb; ++ u16 offset; ++ u16 data_len; ++ struct { ++ void (*sk_state_change)(struct sock *sk); ++ void (*sk_data_ready)(struct sock *sk); ++ void (*sk_write_space)(struct sock *sk); ++ } sk_cb; ++ } tcp; ++ ++ struct dst_cache dst_cache; ++ ++ /* our crypto state */ ++ struct ovpn_crypto_state crypto; ++ ++ /* our binding to peer, protected by spinlock */ ++ struct ovpn_bind __rcu *bind; ++ ++ /* timer used to send periodic ping messages to the other peer, if no ++ * other data was sent within the past keepalive_interval seconds ++ */ ++ struct timer_list keepalive_xmit; ++ /* keepalive interval in seconds */ ++ unsigned long keepalive_interval; ++ ++ /* timer used to mark a peer as expired when no data is received for ++ * keepalive_timeout seconds ++ */ ++ struct timer_list keepalive_recv; ++ /* keepalive timeout in seconds */ ++ unsigned long keepalive_timeout; ++ ++ /* true if ovpn_peer_mark_delete was called */ ++ bool halt; ++ ++ /* per-peer rx/tx stats */ ++ struct ovpn_peer_stats stats; ++ ++ /* why peer was deleted - keepalive timeout, module removed etc */ ++ enum ovpn_del_peer_reason delete_reason; ++ ++ /* protects binding to peer (bind) and timers ++ * (keepalive_xmit, keepalive_expire) ++ */ ++ spinlock_t lock; ++ ++ /* needed because crypto methods can go async */ ++ struct kref refcount; ++ ++ /* needed to free a peer in an RCU safe way */ ++ struct rcu_head rcu; ++ ++ /* needed to notify userspace about deletion */ ++ struct work_struct delete_work; ++}; ++ ++void ovpn_peer_release_kref(struct kref *kref); ++void ovpn_peer_release(struct ovpn_peer *peer); ++ ++static inline bool ovpn_peer_hold(struct ovpn_peer *peer) ++{ ++ return kref_get_unless_zero(&peer->refcount); ++} ++ ++static inline void ovpn_peer_put(struct ovpn_peer *peer) ++{ ++ kref_put(&peer->refcount, ovpn_peer_release_kref); ++} ++ ++static inline void ovpn_peer_keepalive_recv_reset(struct ovpn_peer *peer) ++{ ++ u32 delta = msecs_to_jiffies(peer->keepalive_timeout * MSEC_PER_SEC); ++ ++ if (unlikely(!delta)) ++ return; ++ ++ mod_timer(&peer->keepalive_recv, jiffies + delta); ++} ++ ++static inline void ovpn_peer_keepalive_xmit_reset(struct ovpn_peer *peer) ++{ ++ u32 delta = msecs_to_jiffies(peer->keepalive_interval * MSEC_PER_SEC); ++ ++ if (unlikely(!delta)) ++ return; ++ ++ mod_timer(&peer->keepalive_xmit, jiffies + delta); ++} ++ ++struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, ++ struct socket *sock, u32 id, uint8_t *local_ip); ++ ++void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout); ++ ++int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer); ++int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason); ++struct ovpn_peer *ovpn_peer_find(struct ovpn_struct *ovpn, u32 peer_id); ++void ovpn_peer_release_p2p(struct ovpn_struct *ovpn); ++void ovpn_peers_free(struct ovpn_struct *ovpn); ++ ++struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb); ++struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, ++ bool use_src); ++struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id); ++ ++void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb); ++void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb); ++ ++#endif /* _NET_OVPN_DCO_OVPNPEER_H_ */ +diff --git a/drivers/net/ovpn-dco/pktid.c b/drivers/net/ovpn-dco/pktid.c +new file mode 100644 +index 000000000000..fcde8fba5156 +--- /dev/null ++++ b/drivers/net/ovpn-dco/pktid.c +@@ -0,0 +1,127 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ ++ ++#include "pktid.h" ++ ++#include ++#include ++ ++void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid) ++{ ++ atomic64_set(&pid->seq_num, 1); ++ pid->tcp_linear = NULL; ++} ++ ++void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr) ++{ ++ memset(pr, 0, sizeof(*pr)); ++ spin_lock_init(&pr->lock); ++} ++ ++/* Packet replay detection. ++ * Allows ID backtrack of up to REPLAY_WINDOW_SIZE - 1. ++ */ ++int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time) ++{ ++ const unsigned long now = jiffies; ++ int ret; ++ ++ spin_lock(&pr->lock); ++ ++ /* expire backtracks at or below pr->id after PKTID_RECV_EXPIRE time */ ++ if (unlikely(time_after_eq(now, pr->expire))) ++ pr->id_floor = pr->id; ++ ++ /* ID must not be zero */ ++ if (unlikely(pkt_id == 0)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ /* time changed? */ ++ if (unlikely(pkt_time != pr->time)) { ++ if (pkt_time > pr->time) { ++ /* time moved forward, accept */ ++ pr->base = 0; ++ pr->extent = 0; ++ pr->id = 0; ++ pr->time = pkt_time; ++ pr->id_floor = 0; ++ } else { ++ /* time moved backward, reject */ ++ ret = -ETIME; ++ goto out; ++ } ++ } ++ ++ if (likely(pkt_id == pr->id + 1)) { ++ /* well-formed ID sequence (incremented by 1) */ ++ pr->base = REPLAY_INDEX(pr->base, -1); ++ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); ++ if (pr->extent < REPLAY_WINDOW_SIZE) ++ ++pr->extent; ++ pr->id = pkt_id; ++ } else if (pkt_id > pr->id) { ++ /* ID jumped forward by more than one */ ++ const unsigned int delta = pkt_id - pr->id; ++ ++ if (delta < REPLAY_WINDOW_SIZE) { ++ unsigned int i; ++ ++ pr->base = REPLAY_INDEX(pr->base, -delta); ++ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); ++ pr->extent += delta; ++ if (pr->extent > REPLAY_WINDOW_SIZE) ++ pr->extent = REPLAY_WINDOW_SIZE; ++ for (i = 1; i < delta; ++i) { ++ unsigned int newb = REPLAY_INDEX(pr->base, i); ++ ++ pr->history[newb / 8] &= ~BIT(newb % 8); ++ } ++ } else { ++ pr->base = 0; ++ pr->extent = REPLAY_WINDOW_SIZE; ++ memset(pr->history, 0, sizeof(pr->history)); ++ pr->history[0] = 1; ++ } ++ pr->id = pkt_id; ++ } else { ++ /* ID backtrack */ ++ const unsigned int delta = pr->id - pkt_id; ++ ++ if (delta > pr->max_backtrack) ++ pr->max_backtrack = delta; ++ if (delta < pr->extent) { ++ if (pkt_id > pr->id_floor) { ++ const unsigned int ri = REPLAY_INDEX(pr->base, ++ delta); ++ u8 *p = &pr->history[ri / 8]; ++ const u8 mask = (1 << (ri % 8)); ++ ++ if (*p & mask) { ++ ret = -EINVAL; ++ goto out; ++ } ++ *p |= mask; ++ } else { ++ ret = -EINVAL; ++ goto out; ++ } ++ } else { ++ ret = -EINVAL; ++ goto out; ++ } ++ } ++ ++ pr->expire = now + PKTID_RECV_EXPIRE; ++ ret = 0; ++out: ++ spin_unlock(&pr->lock); ++ return ret; ++} +diff --git a/drivers/net/ovpn-dco/pktid.h b/drivers/net/ovpn-dco/pktid.h +new file mode 100644 +index 000000000000..2447bb37ba55 +--- /dev/null ++++ b/drivers/net/ovpn-dco/pktid.h +@@ -0,0 +1,116 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNPKTID_H_ ++#define _NET_OVPN_DCO_OVPNPKTID_H_ ++ ++#include "main.h" ++ ++/* When the OpenVPN protocol is run in AEAD mode, use ++ * the OpenVPN packet ID as the AEAD nonce: ++ * ++ * 00000005 521c3b01 4308c041 ++ * [seq # ] [ nonce_tail ] ++ * [ 12-byte full IV ] -> NONCE_SIZE ++ * [4-bytes -> NONCE_WIRE_SIZE ++ * on wire] ++ */ ++ ++/* OpenVPN nonce size */ ++#define NONCE_SIZE 12 ++/* amount of bytes of the nonce received from user space */ ++#define NONCE_TAIL_SIZE 8 ++ ++/* OpenVPN nonce size reduced by 8-byte nonce tail -- this is the ++ * size of the AEAD Associated Data (AD) sent over the wire ++ * and is normally the head of the IV ++ */ ++#define NONCE_WIRE_SIZE (NONCE_SIZE - sizeof(struct ovpn_nonce_tail)) ++ ++/* If no packets received for this length of time, set a backtrack floor ++ * at highest received packet ID thus far. ++ */ ++#define PKTID_RECV_EXPIRE (30 * HZ) ++ ++/* Last 8 bytes of AEAD nonce ++ * Provided by userspace and usually derived from ++ * key material generated during TLS handshake ++ */ ++struct ovpn_nonce_tail { ++ u8 u8[NONCE_TAIL_SIZE]; ++}; ++ ++/* Packet-ID state for transmitter */ ++struct ovpn_pktid_xmit { ++ atomic64_t seq_num; ++ struct ovpn_tcp_linear *tcp_linear; ++}; ++ ++/* replay window sizing in bytes = 2^REPLAY_WINDOW_ORDER */ ++#define REPLAY_WINDOW_ORDER 8 ++ ++#define REPLAY_WINDOW_BYTES BIT(REPLAY_WINDOW_ORDER) ++#define REPLAY_WINDOW_SIZE (REPLAY_WINDOW_BYTES * 8) ++#define REPLAY_INDEX(base, i) (((base) + (i)) & (REPLAY_WINDOW_SIZE - 1)) ++ ++/* Packet-ID state for receiver. ++ * Other than lock member, can be zeroed to initialize. ++ */ ++struct ovpn_pktid_recv { ++ /* "sliding window" bitmask of recent packet IDs received */ ++ u8 history[REPLAY_WINDOW_BYTES]; ++ /* bit position of deque base in history */ ++ unsigned int base; ++ /* extent (in bits) of deque in history */ ++ unsigned int extent; ++ /* expiration of history in jiffies */ ++ unsigned long expire; ++ /* highest sequence number received */ ++ u32 id; ++ /* highest time stamp received */ ++ u32 time; ++ /* we will only accept backtrack IDs > id_floor */ ++ u32 id_floor; ++ unsigned int max_backtrack; ++ /* protects entire pktd ID state */ ++ spinlock_t lock; ++}; ++ ++/* Get the next packet ID for xmit */ ++static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) ++{ ++ const s64 seq_num = atomic64_fetch_add_unless(&pid->seq_num, 1, ++ 0x100000000LL); ++ /* when the 32bit space is over, we return an error because the packet ID is used to create ++ * the cipher IV and we do not want to re-use the same value more than once ++ */ ++ if (unlikely(seq_num == 0x100000000LL)) ++ return -ERANGE; ++ ++ *pktid = (u32)seq_num; ++ ++ return 0; ++} ++ ++/* Write 12-byte AEAD IV to dest */ ++static inline void ovpn_pktid_aead_write(const u32 pktid, ++ const struct ovpn_nonce_tail *nt, ++ unsigned char *dest) ++{ ++ *(__force __be32 *)(dest) = htonl(pktid); ++ BUILD_BUG_ON(4 + sizeof(struct ovpn_nonce_tail) != NONCE_SIZE); ++ memcpy(dest + 4, nt->u8, sizeof(struct ovpn_nonce_tail)); ++} ++ ++void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid); ++void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr); ++ ++int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time); ++ ++#endif /* _NET_OVPN_DCO_OVPNPKTID_H_ */ +diff --git a/drivers/net/ovpn-dco/proto.h b/drivers/net/ovpn-dco/proto.h +new file mode 100644 +index 000000000000..875529021c1b +--- /dev/null ++++ b/drivers/net/ovpn-dco/proto.h +@@ -0,0 +1,101 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNPROTO_H_ ++#define _NET_OVPN_DCO_OVPNPROTO_H_ ++ ++#include "main.h" ++ ++#include ++ ++/* Methods for operating on the initial command ++ * byte of the OpenVPN protocol. ++ */ ++ ++/* packet opcode (high 5 bits) and key-id (low 3 bits) are combined in ++ * one byte ++ */ ++#define OVPN_KEY_ID_MASK 0x07 ++#define OVPN_OPCODE_SHIFT 3 ++#define OVPN_OPCODE_MASK 0x1F ++/* upper bounds on opcode and key ID */ ++#define OVPN_KEY_ID_MAX (OVPN_KEY_ID_MASK + 1) ++#define OVPN_OPCODE_MAX (OVPN_OPCODE_MASK + 1) ++/* packet opcodes of interest to us */ ++#define OVPN_DATA_V1 6 /* data channel V1 packet */ ++#define OVPN_DATA_V2 9 /* data channel V2 packet */ ++/* size of initial packet opcode */ ++#define OVPN_OP_SIZE_V1 1 ++#define OVPN_OP_SIZE_V2 4 ++#define OVPN_PEER_ID_MASK 0x00FFFFFF ++#define OVPN_PEER_ID_UNDEF 0x00FFFFFF ++/* first byte of keepalive message */ ++#define OVPN_KEEPALIVE_FIRST_BYTE 0x2a ++/* first byte of exit message */ ++#define OVPN_EXPLICIT_EXIT_NOTIFY_FIRST_BYTE 0x28 ++ ++/** ++ * Extract the OP code from the specified byte ++ * ++ * Return the OP code ++ */ ++static inline u8 ovpn_opcode_from_byte(u8 byte) ++{ ++ return byte >> OVPN_OPCODE_SHIFT; ++} ++ ++/** ++ * Extract the OP code from the skb head. ++ * ++ * Note: this function assumes that the skb head was pulled enough ++ * to access the first byte. ++ * ++ * Return the OP code ++ */ ++static inline u8 ovpn_opcode_from_skb(const struct sk_buff *skb, u16 offset) ++{ ++ return ovpn_opcode_from_byte(*(skb->data + offset)); ++} ++ ++/** ++ * Extract the key ID from the skb head. ++ * ++ * Note: this function assumes that the skb head was pulled enough ++ * to access the first byte. ++ * ++ * Return the key ID ++ */ ++ ++static inline u8 ovpn_key_id_from_skb(const struct sk_buff *skb) ++{ ++ return *skb->data & OVPN_KEY_ID_MASK; ++} ++ ++/** ++ * Extract the peer ID from the skb head. ++ * ++ * Note: this function assumes that the skb head was pulled enough ++ * to access the first 4 bytes. ++ * ++ * Return the peer ID. ++ */ ++ ++static inline u32 ovpn_peer_id_from_skb(const struct sk_buff *skb, u16 offset) ++{ ++ return ntohl(*(__be32 *)(skb->data + offset)) & OVPN_PEER_ID_MASK; ++} ++ ++static inline u32 ovpn_opcode_compose(u8 opcode, u8 key_id, u32 peer_id) ++{ ++ const u8 op = (opcode << OVPN_OPCODE_SHIFT) | (key_id & OVPN_KEY_ID_MASK); ++ ++ return (op << 24) | (peer_id & OVPN_PEER_ID_MASK); ++} ++ ++#endif /* _NET_OVPN_DCO_OVPNPROTO_H_ */ +diff --git a/drivers/net/ovpn-dco/rcu.h b/drivers/net/ovpn-dco/rcu.h +new file mode 100644 +index 000000000000..02a50f49ba2e +--- /dev/null ++++ b/drivers/net/ovpn-dco/rcu.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNRCU_H_ ++#define _NET_OVPN_DCO_OVPNRCU_H_ ++ ++static inline void ovpn_rcu_lockdep_assert_held(void) ++{ ++#ifdef CONFIG_PROVE_RCU ++ RCU_LOCKDEP_WARN(!rcu_read_lock_held(), ++ "ovpn-dco RCU read lock not held"); ++#endif ++} ++ ++#endif /* _NET_OVPN_DCO_OVPNRCU_H_ */ +diff --git a/drivers/net/ovpn-dco/skb.h b/drivers/net/ovpn-dco/skb.h +new file mode 100644 +index 000000000000..d38dc2da01df +--- /dev/null ++++ b/drivers/net/ovpn-dco/skb.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ ++ ++#ifndef _NET_OVPN_DCO_SKB_H_ ++#define _NET_OVPN_DCO_SKB_H_ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define OVPN_SKB_CB(skb) ((struct ovpn_skb_cb *)&((skb)->cb)) ++ ++struct ovpn_skb_cb { ++ /* original recv packet size for stats accounting */ ++ unsigned int rx_stats_size; ++ ++ union { ++ struct in_addr ipv4; ++ struct in6_addr ipv6; ++ } local; ++ sa_family_t sa_fam; ++}; ++ ++/* Return IP protocol version from skb header. ++ * Return 0 if protocol is not IPv4/IPv6 or cannot be read. ++ */ ++static inline __be16 ovpn_ip_check_protocol(struct sk_buff *skb) ++{ ++ __be16 proto = 0; ++ ++ /* skb could be non-linear, ++ * make sure IP header is in non-fragmented part ++ */ ++ if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) ++ return 0; ++ ++ if (ip_hdr(skb)->version == 4) ++ proto = htons(ETH_P_IP); ++ else if (ip_hdr(skb)->version == 6) ++ proto = htons(ETH_P_IPV6); ++ ++ return proto; ++} ++ ++#endif /* _NET_OVPN_DCO_SKB_H_ */ +diff --git a/drivers/net/ovpn-dco/sock.c b/drivers/net/ovpn-dco/sock.c +new file mode 100644 +index 000000000000..e92a4a9b952e +--- /dev/null ++++ b/drivers/net/ovpn-dco/sock.c +@@ -0,0 +1,134 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "ovpn.h" ++#include "peer.h" ++#include "sock.h" ++#include "rcu.h" ++#include "tcp.h" ++#include "udp.h" ++ ++#include ++#include ++ ++/* Finalize release of socket, called after RCU grace period */ ++static void ovpn_socket_detach(struct socket *sock) ++{ ++ if (!sock) ++ return; ++ ++ if (sock->sk->sk_protocol == IPPROTO_UDP) ++ ovpn_udp_socket_detach(sock); ++ else if (sock->sk->sk_protocol == IPPROTO_TCP) ++ ovpn_tcp_socket_detach(sock); ++ ++ sockfd_put(sock); ++} ++ ++void ovpn_socket_release_kref(struct kref *kref) ++{ ++ struct ovpn_socket *sock = container_of(kref, struct ovpn_socket, refcount); ++ ++ ovpn_socket_detach(sock->sock); ++ kfree_rcu(sock, rcu); ++} ++ ++static bool ovpn_socket_hold(struct ovpn_socket *sock) ++{ ++ return kref_get_unless_zero(&sock->refcount); ++} ++ ++static struct ovpn_socket *ovpn_socket_get(struct socket *sock) ++{ ++ struct ovpn_socket *ovpn_sock; ++ ++ rcu_read_lock(); ++ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); ++ if (!ovpn_socket_hold(ovpn_sock)) { ++ pr_warn("%s: found ovpn_socket with ref = 0\n", __func__); ++ ovpn_sock = NULL; ++ } ++ rcu_read_unlock(); ++ ++ return ovpn_sock; ++} ++ ++/* Finalize release of socket, called after RCU grace period */ ++static int ovpn_socket_attach(struct socket *sock, struct ovpn_peer *peer) ++{ ++ int ret = -EOPNOTSUPP; ++ ++ if (!sock || !peer) ++ return -EINVAL; ++ ++ if (sock->sk->sk_protocol == IPPROTO_UDP) ++ ret = ovpn_udp_socket_attach(sock, peer->ovpn); ++ else if (sock->sk->sk_protocol == IPPROTO_TCP) ++ ret = ovpn_tcp_socket_attach(sock, peer); ++ ++ return ret; ++} ++ ++struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk) ++{ ++ struct ovpn_socket *ovpn_sock; ++ ++ ovpn_rcu_lockdep_assert_held(); ++ ++ if (unlikely(READ_ONCE(udp_sk(sk)->encap_type) != UDP_ENCAP_OVPNINUDP)) ++ return NULL; ++ ++ ovpn_sock = rcu_dereference_sk_user_data(sk); ++ if (unlikely(!ovpn_sock)) ++ return NULL; ++ ++ /* make sure that sk matches our stored transport socket */ ++ if (unlikely(!ovpn_sock->sock || sk != ovpn_sock->sock->sk)) ++ return NULL; ++ ++ return ovpn_sock->ovpn; ++} ++ ++struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ++{ ++ struct ovpn_socket *ovpn_sock; ++ int ret; ++ ++ ret = ovpn_socket_attach(sock, peer); ++ if (ret < 0 && ret != -EALREADY) ++ return ERR_PTR(ret); ++ ++ /* if this socket is already owned by this interface, just increase the refcounter */ ++ if (ret == -EALREADY) { ++ /* caller is expected to increase the sock refcounter before passing it to this ++ * function. For this reason we drop it if not needed, like when this socket is ++ * already owned. ++ */ ++ ovpn_sock = ovpn_socket_get(sock); ++ sockfd_put(sock); ++ return ovpn_sock; ++ } ++ ++ ovpn_sock = kzalloc(sizeof(*ovpn_sock), GFP_KERNEL); ++ if (!ovpn_sock) ++ return ERR_PTR(-ENOMEM); ++ ++ ovpn_sock->ovpn = peer->ovpn; ++ ovpn_sock->sock = sock; ++ kref_init(&ovpn_sock->refcount); ++ ++ /* TCP sockets are per-peer, therefore they are linked to their unique peer */ ++ if (sock->sk->sk_protocol == IPPROTO_TCP) ++ ovpn_sock->peer = peer; ++ ++ rcu_assign_sk_user_data(sock->sk, ovpn_sock); ++ ++ return ovpn_sock; ++} +diff --git a/drivers/net/ovpn-dco/sock.h b/drivers/net/ovpn-dco/sock.h +new file mode 100644 +index 000000000000..9e79c1b5fe04 +--- /dev/null ++++ b/drivers/net/ovpn-dco/sock.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_SOCK_H_ ++#define _NET_OVPN_DCO_SOCK_H_ ++ ++#include ++#include ++#include ++ ++#include "peer.h" ++ ++struct ovpn_struct; ++ ++/** ++ * struct ovpn_socket - a kernel socket referenced in the ovpn-dco code ++ */ ++struct ovpn_socket { ++ union { ++ /** @ovpn: the VPN session object owning this socket (UDP only) */ ++ struct ovpn_struct *ovpn; ++ ++ /** @peer: the unique peer transmitting over this socket (TCP only) */ ++ struct ovpn_peer *peer; ++ }; ++ ++ /** @sock: the kernel socket */ ++ struct socket *sock; ++ ++ /** @refcount: amount of contexts currently referencing this object */ ++ struct kref refcount; ++ ++ /** @rcu: member used to schedule RCU destructor callback */ ++ struct rcu_head rcu; ++}; ++ ++struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk); ++ ++void ovpn_socket_release_kref(struct kref *kref); ++ ++static inline void ovpn_socket_put(struct ovpn_socket *sock) ++{ ++ kref_put(&sock->refcount, ovpn_socket_release_kref); ++} ++ ++struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer); ++ ++#endif /* _NET_OVPN_DCO_SOCK_H_ */ +diff --git a/drivers/net/ovpn-dco/stats.c b/drivers/net/ovpn-dco/stats.c +new file mode 100644 +index 000000000000..ee000b2a2177 +--- /dev/null ++++ b/drivers/net/ovpn-dco/stats.c +@@ -0,0 +1,20 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "stats.h" ++ ++void ovpn_peer_stats_init(struct ovpn_peer_stats *ps) ++{ ++ atomic64_set(&ps->rx.bytes, 0); ++ atomic_set(&ps->rx.packets, 0); ++ ++ atomic64_set(&ps->tx.bytes, 0); ++ atomic_set(&ps->tx.packets, 0); ++} +diff --git a/drivers/net/ovpn-dco/stats.h b/drivers/net/ovpn-dco/stats.h +new file mode 100644 +index 000000000000..3aa6bdc049c6 +--- /dev/null ++++ b/drivers/net/ovpn-dco/stats.h +@@ -0,0 +1,67 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ * Lev Stipakov ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNSTATS_H_ ++#define _NET_OVPN_DCO_OVPNSTATS_H_ ++ ++#include ++#include ++ ++struct ovpn_struct; ++ ++/* per-peer stats, measured on transport layer */ ++ ++/* one stat */ ++struct ovpn_peer_stat { ++ atomic64_t bytes; ++ atomic_t packets; ++}; ++ ++/* rx and tx stats, enabled by notify_per != 0 or period != 0 */ ++struct ovpn_peer_stats { ++ struct ovpn_peer_stat rx; ++ struct ovpn_peer_stat tx; ++}; ++ ++/* struct for OVPN_ERR_STATS */ ++ ++struct ovpn_err_stat { ++ unsigned int category; ++ int errcode; ++ u64 count; ++}; ++ ++struct ovpn_err_stats { ++ /* total stats, returned by kovpn */ ++ unsigned int total_stats; ++ /* number of stats dimensioned below */ ++ unsigned int n_stats; ++ struct ovpn_err_stat stats[]; ++}; ++ ++void ovpn_peer_stats_init(struct ovpn_peer_stats *ps); ++ ++static inline void ovpn_peer_stats_increment(struct ovpn_peer_stat *stat, const unsigned int n) ++{ ++ atomic64_add(n, &stat->bytes); ++ atomic_inc(&stat->packets); ++} ++ ++static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats, const unsigned int n) ++{ ++ ovpn_peer_stats_increment(&stats->rx, n); ++} ++ ++static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats, const unsigned int n) ++{ ++ ovpn_peer_stats_increment(&stats->tx, n); ++} ++ ++#endif /* _NET_OVPN_DCO_OVPNSTATS_H_ */ +diff --git a/drivers/net/ovpn-dco/tcp.c b/drivers/net/ovpn-dco/tcp.c +new file mode 100644 +index 000000000000..7e6690fee6e7 +--- /dev/null ++++ b/drivers/net/ovpn-dco/tcp.c +@@ -0,0 +1,326 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "ovpnstruct.h" ++#include "ovpn.h" ++#include "peer.h" ++#include "skb.h" ++#include "tcp.h" ++ ++#include ++#include ++#include ++ ++static void ovpn_tcp_state_change(struct sock *sk) ++{ ++} ++ ++static void ovpn_tcp_data_ready(struct sock *sk) ++{ ++ struct ovpn_socket *sock; ++ ++ rcu_read_lock(); ++ sock = rcu_dereference_sk_user_data(sk); ++ rcu_read_unlock(); ++ ++ if (!sock || !sock->peer) ++ return; ++ ++ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.rx_work); ++} ++ ++static void ovpn_tcp_write_space(struct sock *sk) ++{ ++ struct ovpn_socket *sock; ++ ++ rcu_read_lock(); ++ sock = rcu_dereference_sk_user_data(sk); ++ rcu_read_unlock(); ++ ++ if (!sock || !sock->peer) ++ return; ++ ++ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.tx_work); ++} ++ ++static void ovpn_destroy_skb(void *skb) ++{ ++ consume_skb(skb); ++} ++ ++void ovpn_tcp_socket_detach(struct socket *sock) ++{ ++ struct ovpn_socket *ovpn_sock; ++ struct ovpn_peer *peer; ++ ++ if (!sock) ++ return; ++ ++ rcu_read_lock(); ++ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); ++ rcu_read_unlock(); ++ ++ if (!ovpn_sock->peer) ++ return; ++ ++ peer = ovpn_sock->peer; ++ ++ /* restore CBs that were saved in ovpn_sock_set_tcp_cb() */ ++ write_lock_bh(&sock->sk->sk_callback_lock); ++ sock->sk->sk_state_change = peer->tcp.sk_cb.sk_state_change; ++ sock->sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; ++ sock->sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; ++ rcu_assign_sk_user_data(sock->sk, NULL); ++ write_unlock_bh(&sock->sk->sk_callback_lock); ++ ++ /* cancel any ongoing work. Done after removing the CBs so that these workers cannot be ++ * re-armed ++ */ ++ cancel_work_sync(&peer->tcp.tx_work); ++ cancel_work_sync(&peer->tcp.rx_work); ++ ++ ptr_ring_cleanup(&peer->tcp.tx_ring, ovpn_destroy_skb); ++} ++ ++/* Try to send one skb (or part of it) over the TCP stream. ++ * ++ * Return 0 on success or a negative error code otherwise. ++ * ++ * Note that the skb is modified by putting away the data being sent, therefore ++ * the caller should check if skb->len is zero to understand if the full skb was ++ * sent or not. ++ */ ++static int ovpn_tcp_send_one(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; ++ struct kvec iv = { 0 }; ++ int ret; ++ ++ if (skb_linearize(skb) < 0) { ++ net_err_ratelimited("%s: can't linearize packet\n", __func__); ++ return -ENOMEM; ++ } ++ ++ /* initialize iv structure now as skb_linearize() may have changed skb->data */ ++ iv.iov_base = skb->data; ++ iv.iov_len = skb->len; ++ ++ ret = kernel_sendmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len); ++ if (ret > 0) { ++ __skb_pull(skb, ret); ++ ++ /* since we update per-cpu stats in process context, ++ * we need to disable softirqs ++ */ ++ local_bh_disable(); ++ dev_sw_netstats_tx_add(peer->ovpn->dev, 1, ret); ++ local_bh_enable(); ++ ++ return 0; ++ } ++ ++ return ret; ++} ++ ++/* Process packets in TCP TX queue */ ++static void ovpn_tcp_tx_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer; ++ struct sk_buff *skb; ++ int ret; ++ ++ peer = container_of(work, struct ovpn_peer, tcp.tx_work); ++ while ((skb = __ptr_ring_peek(&peer->tcp.tx_ring))) { ++ ret = ovpn_tcp_send_one(peer, skb); ++ if (ret < 0 && ret != -EAGAIN) { ++ net_warn_ratelimited("%s: cannot send TCP packet to peer %u: %d\n", __func__, ++ peer->id, ret); ++ /* in case of TCP error stop sending loop and delete peer */ ++ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_ERROR); ++ break; ++ } else if (!skb->len) { ++ /* skb was entirely consumed and can now be removed from the ring */ ++ __ptr_ring_discard_one(&peer->tcp.tx_ring); ++ consume_skb(skb); ++ } ++ ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); ++ } ++} ++ ++static int ovpn_tcp_rx_one(struct ovpn_peer *peer) ++{ ++ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; ++ struct ovpn_skb_cb *cb; ++ int status, ret; ++ ++ /* no skb allocated means that we have to read (or finish reading) the 2 bytes prefix ++ * containing the actual packet size. ++ */ ++ if (!peer->tcp.skb) { ++ struct kvec iv = { ++ .iov_base = peer->tcp.raw_len + peer->tcp.offset, ++ .iov_len = sizeof(u16) - peer->tcp.offset, ++ }; ++ ++ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); ++ if (ret <= 0) ++ return ret; ++ ++ peer->tcp.offset += ret; ++ /* the entire packet size was read, prepare skb for reading data */ ++ if (peer->tcp.offset == sizeof(u16)) { ++ u16 len = ntohs(*(__be16 *)peer->tcp.raw_len); ++ /* invalid packet length: this is a fatal TCP error */ ++ if (!len) { ++ netdev_err(peer->ovpn->dev, "%s: received invalid packet length\n", ++ __func__); ++ return -EINVAL; ++ } ++ ++ peer->tcp.skb = netdev_alloc_skb_ip_align(peer->ovpn->dev, len); ++ peer->tcp.offset = 0; ++ peer->tcp.data_len = len; ++ } ++ } else { ++ struct kvec iv = { ++ .iov_base = peer->tcp.skb->data + peer->tcp.offset, ++ .iov_len = peer->tcp.data_len - peer->tcp.offset, ++ }; ++ ++ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); ++ if (ret <= 0) ++ return ret; ++ ++ peer->tcp.offset += ret; ++ /* full packet received, send it up for processing */ ++ if (peer->tcp.offset == peer->tcp.data_len) { ++ /* update the skb data structure with the amount of data written by ++ * kernel_recvmsg() ++ */ ++ skb_put(peer->tcp.skb, peer->tcp.data_len); ++ ++ /* do not perform IP caching for TCP connections */ ++ cb = OVPN_SKB_CB(peer->tcp.skb); ++ cb->sa_fam = AF_UNSPEC; ++ ++ /* hold reference to peer as requird by ovpn_recv() */ ++ ovpn_peer_hold(peer); ++ status = ovpn_recv(peer->ovpn, peer, peer->tcp.skb); ++ /* skb not consumed - free it now */ ++ if (unlikely(status < 0)) ++ kfree_skb(peer->tcp.skb); ++ ++ peer->tcp.skb = NULL; ++ peer->tcp.offset = 0; ++ peer->tcp.data_len = 0; ++ } ++ } ++ ++ return ret; ++} ++ ++static void ovpn_tcp_rx_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, tcp.rx_work); ++ int ret; ++ ++ while (true) { ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); ++ ++ ret = ovpn_tcp_rx_one(peer); ++ if (ret <= 0) ++ break; ++ } ++ ++ if (ret < 0 && ret != -EAGAIN) ++ netdev_err(peer->ovpn->dev, "%s: TCP socket error: %d\n", __func__, ret); ++} ++ ++/* Put packet into TCP TX queue and schedule a consumer */ ++void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ int ret; ++ ++ ret = ptr_ring_produce_bh(&peer->tcp.tx_ring, skb); ++ if (ret < 0) { ++ kfree_skb_list(skb); ++ return; ++ } ++ ++ queue_work(peer->ovpn->events_wq, &peer->tcp.tx_work); ++} ++ ++/* Set TCP encapsulation callbacks */ ++int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer) ++{ ++ void *old_data; ++ int ret; ++ ++ INIT_WORK(&peer->tcp.tx_work, ovpn_tcp_tx_work); ++ INIT_WORK(&peer->tcp.rx_work, ovpn_tcp_rx_work); ++ ++ ret = ptr_ring_init(&peer->tcp.tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(peer->ovpn->dev, "cannot allocate TCP TX ring\n"); ++ return ret; ++ } ++ ++ peer->tcp.skb = NULL; ++ peer->tcp.offset = 0; ++ peer->tcp.data_len = 0; ++ ++ write_lock_bh(&sock->sk->sk_callback_lock); ++ ++ /* make sure no pre-existing encapsulation handler exists */ ++ rcu_read_lock(); ++ old_data = rcu_dereference_sk_user_data(sock->sk); ++ rcu_read_unlock(); ++ if (old_data) { ++ netdev_err(peer->ovpn->dev, "provided socket already taken by other user\n"); ++ ret = -EBUSY; ++ goto err; ++ } ++ ++ /* sanity check */ ++ if (sock->sk->sk_protocol != IPPROTO_TCP) { ++ netdev_err(peer->ovpn->dev, "expected TCP socket\n"); ++ ret = -EINVAL; ++ goto err; ++ } ++ ++ /* only a fully connected socket are expected. Connection should be handled in userspace */ ++ if (sock->sk->sk_state != TCP_ESTABLISHED) { ++ netdev_err(peer->ovpn->dev, "unexpected state for TCP socket: %d\n", ++ sock->sk->sk_state); ++ ret = -EINVAL; ++ goto err; ++ } ++ ++ /* save current CBs so that they can be restored upon socket release */ ++ peer->tcp.sk_cb.sk_state_change = sock->sk->sk_state_change; ++ peer->tcp.sk_cb.sk_data_ready = sock->sk->sk_data_ready; ++ peer->tcp.sk_cb.sk_write_space = sock->sk->sk_write_space; ++ ++ /* assign our static CBs */ ++ sock->sk->sk_state_change = ovpn_tcp_state_change; ++ sock->sk->sk_data_ready = ovpn_tcp_data_ready; ++ sock->sk->sk_write_space = ovpn_tcp_write_space; ++ ++ write_unlock_bh(&sock->sk->sk_callback_lock); ++ ++ return 0; ++err: ++ write_unlock_bh(&sock->sk->sk_callback_lock); ++ ptr_ring_cleanup(&peer->tcp.tx_ring, NULL); ++ ++ return ret; ++} +diff --git a/drivers/net/ovpn-dco/tcp.h b/drivers/net/ovpn-dco/tcp.h +new file mode 100644 +index 000000000000..d243a8e1c34e +--- /dev/null ++++ b/drivers/net/ovpn-dco/tcp.h +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_TCP_H_ ++#define _NET_OVPN_DCO_TCP_H_ ++ ++#include "peer.h" ++ ++#include ++#include ++#include ++#include ++ ++void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb); ++ ++int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer); ++void ovpn_tcp_socket_detach(struct socket *sock); ++ ++/* Prepare skb and enqueue it for sending to peer. ++ * ++ * Preparation consist in prepending the skb payload with its size. ++ * Required by the OpenVPN protocol in order to extract packets from ++ * the TCP stream on the receiver side. ++ */ ++static inline void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ u16 len = skb->len; ++ ++ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); ++ ovpn_queue_tcp_skb(peer, skb); ++} ++ ++#endif /* _NET_OVPN_DCO_TCP_H_ */ +diff --git a/drivers/net/ovpn-dco/udp.c b/drivers/net/ovpn-dco/udp.c +new file mode 100644 +index 000000000000..afa236d1f15c +--- /dev/null ++++ b/drivers/net/ovpn-dco/udp.c +@@ -0,0 +1,343 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ ++ ++#include "main.h" ++#include "bind.h" ++#include "ovpn.h" ++#include "ovpnstruct.h" ++#include "peer.h" ++#include "proto.h" ++#include "skb.h" ++#include "udp.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * ovpn_udp_encap_recv() - Start processing a received UDP packet. ++ * If the first byte of the payload is DATA_V2, the packet is further processed, ++ * otherwise it is forwarded to the UDP stack for delivery to user space. ++ * ++ * @sk: the socket the packet was received on ++ * @skb: the sk_buff containing the actual packet ++ * ++ * Return codes: ++ * 0 : we consumed or dropped packet ++ * >0 : skb should be passed up to userspace as UDP (packet not consumed) ++ * <0 : skb should be resubmitted as proto -N (packet not consumed) ++ */ ++static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) ++{ ++ struct ovpn_peer *peer = NULL; ++ struct ovpn_struct *ovpn; ++ u32 peer_id; ++ u8 opcode; ++ int ret; ++ ++ ovpn = ovpn_from_udp_sock(sk); ++ if (unlikely(!ovpn)) { ++ net_err_ratelimited("%s: cannot obtain ovpn object from UDP socket\n", __func__); ++ goto drop; ++ } ++ ++ /* Make sure the first 4 bytes of the skb data buffer after the UDP header are accessible. ++ * They are required to fetch the OP code, the key ID and the peer ID. ++ */ ++ if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) + 4))) { ++ net_dbg_ratelimited("%s: packet too small\n", __func__); ++ goto drop; ++ } ++ ++ opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr)); ++ if (likely(opcode == OVPN_DATA_V2)) { ++ peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr)); ++ /* some OpenVPN server implementations send data packets with the peer-id set to ++ * undef. In this case we skip the peer lookup by peer-id and we try with the ++ * transport address ++ */ ++ if (peer_id != OVPN_PEER_ID_UNDEF) { ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) { ++ net_err_ratelimited("%s: received data from unknown peer (id: %d)\n", ++ __func__, peer_id); ++ goto drop; ++ } ++ ++ /* check if this peer changed it's IP address and update state */ ++ ovpn_peer_float(peer, skb); ++ } ++ } ++ ++ if (!peer) { ++ /* might be a control packet or a data packet with undef peer-id */ ++ peer = ovpn_peer_lookup_transp_addr(ovpn, skb); ++ if (unlikely(!peer)) { ++ if (opcode != OVPN_DATA_V2) { ++ netdev_dbg(ovpn->dev, ++ "%s: control packet from unknown peer, sending to userspace", ++ __func__); ++ return 1; ++ } ++ ++ netdev_dbg(ovpn->dev, ++ "%s: received data with undef peer-id from unknown source\n", ++ __func__); ++ goto drop; ++ } ++ } ++ ++ /* pop off outer UDP header */ ++ __skb_pull(skb, sizeof(struct udphdr)); ++ ++ ret = ovpn_recv(ovpn, peer, skb); ++ if (unlikely(ret < 0)) { ++ net_err_ratelimited("%s: cannot handle incoming packet: %d\n", __func__, ret); ++ goto drop; ++ } ++ ++ /* should this be a non DATA_V2 packet, ret will be >0 and this will instruct the UDP ++ * stack to continue processing this packet as usual (i.e. deliver to user space) ++ */ ++ return ret; ++ ++drop: ++ if (peer) ++ ovpn_peer_put(peer); ++ kfree_skb(skb); ++ return 0; ++} ++ ++static int ovpn_udp4_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, ++ struct dst_cache *cache, struct sock *sk, ++ struct sk_buff *skb) ++{ ++ struct rtable *rt; ++ struct flowi4 fl = { ++ .saddr = bind->local.ipv4.s_addr, ++ .daddr = bind->sa.in4.sin_addr.s_addr, ++ .fl4_sport = inet_sk(sk)->inet_sport, ++ .fl4_dport = bind->sa.in4.sin_port, ++ .flowi4_proto = sk->sk_protocol, ++ .flowi4_mark = sk->sk_mark, ++ }; ++ int ret; ++ ++ local_bh_disable(); ++ rt = dst_cache_get_ip4(cache, &fl.saddr); ++ if (rt) ++ goto transmit; ++ ++ if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { ++ /* we may end up here when the cached address is not usable anymore. ++ * In this case we reset address/cache and perform a new look up ++ */ ++ fl.saddr = 0; ++ bind->local.ipv4.s_addr = 0; ++ dst_cache_reset(cache); ++ } ++ ++ rt = ip_route_output_flow(sock_net(sk), &fl, sk); ++ if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { ++ fl.saddr = 0; ++ bind->local.ipv4.s_addr = 0; ++ dst_cache_reset(cache); ++ ++ rt = ip_route_output_flow(sock_net(sk), &fl, sk); ++ } ++ ++ if (IS_ERR(rt)) { ++ ret = PTR_ERR(rt); ++ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, ++ &bind->sa.in4, ret); ++ goto err; ++ } ++ dst_cache_set_ip4(cache, &rt->dst, fl.saddr); ++ ++transmit: ++ udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, ++ ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, ++ fl.fl4_dport, false, sk->sk_no_check_tx); ++ ret = 0; ++err: ++ local_bh_enable(); ++ return ret; ++} ++ ++#if IS_ENABLED(CONFIG_IPV6) ++static int ovpn_udp6_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, ++ struct dst_cache *cache, struct sock *sk, ++ struct sk_buff *skb) ++{ ++ struct dst_entry *dst; ++ int ret; ++ ++ struct flowi6 fl = { ++ .saddr = bind->local.ipv6, ++ .daddr = bind->sa.in6.sin6_addr, ++ .fl6_sport = inet_sk(sk)->inet_sport, ++ .fl6_dport = bind->sa.in6.sin6_port, ++ .flowi6_proto = sk->sk_protocol, ++ .flowi6_mark = sk->sk_mark, ++ .flowi6_oif = bind->sa.in6.sin6_scope_id, ++ }; ++ ++ local_bh_disable(); ++ dst = dst_cache_get_ip6(cache, &fl.saddr); ++ if (dst) ++ goto transmit; ++ ++ if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { ++ /* we may end up here when the cached address is not usable anymore. ++ * In this case we reset address/cache and perform a new look up ++ */ ++ fl.saddr = in6addr_any; ++ bind->local.ipv6 = in6addr_any; ++ dst_cache_reset(cache); ++ } ++ ++ dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); ++ if (IS_ERR(dst)) { ++ ret = PTR_ERR(dst); ++ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, ++ &bind->sa.in6, ret); ++ goto err; ++ } ++ dst_cache_set_ip6(cache, dst, &fl.saddr); ++ ++transmit: ++ udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0, ++ ip6_dst_hoplimit(dst), 0, fl.fl6_sport, ++ fl.fl6_dport, udp_get_no_check6_tx(sk)); ++ ret = 0; ++err: ++ local_bh_enable(); ++ return ret; ++} ++#endif ++ ++/* Transmit skb utilizing kernel-provided UDP tunneling framework. ++ * ++ * rcu_read_lock should be held on entry. ++ * On return, the skb is consumed. ++ */ ++static int ovpn_udp_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, ++ struct dst_cache *cache, struct sock *sk, ++ struct sk_buff *skb) ++{ ++ int ret; ++ ++ ovpn_rcu_lockdep_assert_held(); ++ ++ /* set sk to null if skb is already orphaned */ ++ if (!skb->destructor) ++ skb->sk = NULL; ++ ++ switch (bind->sa.in4.sin_family) { ++ case AF_INET: ++ ret = ovpn_udp4_output(ovpn, bind, cache, sk, skb); ++ break; ++#if IS_ENABLED(CONFIG_IPV6) ++ case AF_INET6: ++ ret = ovpn_udp6_output(ovpn, bind, cache, sk, skb); ++ break; ++#endif ++ default: ++ ret = -EAFNOSUPPORT; ++ break; ++ } ++ ++ return ret; ++} ++ ++void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, ++ struct sk_buff *skb) ++{ ++ struct ovpn_bind *bind; ++ struct socket *sock; ++ int ret = -1; ++ ++ skb->dev = ovpn->dev; ++ /* no checksum performed at this layer */ ++ skb->ip_summed = CHECKSUM_NONE; ++ ++ /* get socket info */ ++ sock = peer->sock->sock; ++ if (unlikely(!sock)) { ++ net_dbg_ratelimited("%s: no sock for remote peer\n", __func__); ++ goto out; ++ } ++ ++ rcu_read_lock(); ++ /* get binding */ ++ bind = rcu_dereference(peer->bind); ++ if (unlikely(!bind)) { ++ net_dbg_ratelimited("%s: no bind for remote peer\n", __func__); ++ goto out_unlock; ++ } ++ ++ /* crypto layer -> transport (UDP) */ ++ ret = ovpn_udp_output(ovpn, bind, &peer->dst_cache, sock->sk, skb); ++ ++out_unlock: ++ rcu_read_unlock(); ++out: ++ if (ret < 0) ++ kfree_skb(skb); ++} ++ ++/* Set UDP encapsulation callbacks */ ++int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn) ++{ ++ struct udp_tunnel_sock_cfg cfg = { ++ .sk_user_data = ovpn, ++ .encap_type = UDP_ENCAP_OVPNINUDP, ++ .encap_rcv = ovpn_udp_encap_recv, ++ }; ++ struct ovpn_socket *old_data; ++ ++ /* sanity check */ ++ if (sock->sk->sk_protocol != IPPROTO_UDP) { ++ netdev_err(ovpn->dev, "%s: expected UDP socket\n", __func__); ++ return -EINVAL; ++ } ++ ++ /* make sure no pre-existing encapsulation handler exists */ ++ rcu_read_lock(); ++ old_data = rcu_dereference_sk_user_data(sock->sk); ++ rcu_read_unlock(); ++ if (old_data) { ++ if (old_data->ovpn == ovpn) { ++ netdev_dbg(ovpn->dev, ++ "%s: provided socket already owned by this interface\n", ++ __func__); ++ return -EALREADY; ++ } ++ ++ netdev_err(ovpn->dev, "%s: provided socket already taken by other user\n", ++ __func__); ++ return -EBUSY; ++ } ++ ++ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); ++ ++ return 0; ++} ++ ++/* Detach socket from encapsulation handler and/or other callbacks */ ++void ovpn_udp_socket_detach(struct socket *sock) ++{ ++ struct udp_tunnel_sock_cfg cfg = { }; ++ ++ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); ++} +diff --git a/drivers/net/ovpn-dco/udp.h b/drivers/net/ovpn-dco/udp.h +new file mode 100644 +index 000000000000..be94fb74669b +--- /dev/null ++++ b/drivers/net/ovpn-dco/udp.h +@@ -0,0 +1,25 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ ++ ++#ifndef _NET_OVPN_DCO_UDP_H_ ++#define _NET_OVPN_DCO_UDP_H_ ++ ++#include "peer.h" ++#include "ovpnstruct.h" ++ ++#include ++#include ++#include ++#include ++ ++int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn); ++void ovpn_udp_socket_detach(struct socket *sock); ++void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, ++ struct sk_buff *skb); ++ ++#endif /* _NET_OVPN_DCO_UDP_H_ */ +diff --git a/include/net/netlink.h b/include/net/netlink.h +index 7a2a9d3144ba..335f44871529 100644 +--- a/include/net/netlink.h ++++ b/include/net/netlink.h +@@ -441,6 +441,7 @@ struct nla_policy { + .max = _len \ + } + #define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len) ++#define NLA_POLICY_MAX_LEN(_len) NLA_POLICY_MAX(NLA_BINARY, _len) + + /** + * struct nl_info - netlink source information +diff --git a/include/uapi/linux/ovpn_dco.h b/include/uapi/linux/ovpn_dco.h +new file mode 100644 +index 000000000000..6afee8b3fedd +--- /dev/null ++++ b/include/uapi/linux/ovpn_dco.h +@@ -0,0 +1,265 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* ++ * OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#ifndef _UAPI_LINUX_OVPN_DCO_H_ ++#define _UAPI_LINUX_OVPN_DCO_H_ ++ ++#define OVPN_NL_NAME "ovpn-dco" ++ ++#define OVPN_NL_MULTICAST_GROUP_PEERS "peers" ++ ++/** ++ * enum ovpn_nl_commands - supported netlink commands ++ */ ++enum ovpn_nl_commands { ++ /** ++ * @OVPN_CMD_UNSPEC: unspecified command to catch errors ++ */ ++ OVPN_CMD_UNSPEC = 0, ++ ++ /** ++ * @OVPN_CMD_NEW_PEER: Configure peer with its crypto keys ++ */ ++ OVPN_CMD_NEW_PEER, ++ ++ /** ++ * @OVPN_CMD_SET_PEER: Tweak parameters for an existing peer ++ */ ++ OVPN_CMD_SET_PEER, ++ ++ /** ++ * @OVPN_CMD_DEL_PEER: Remove peer from internal table ++ */ ++ OVPN_CMD_DEL_PEER, ++ ++ OVPN_CMD_NEW_KEY, ++ ++ OVPN_CMD_SWAP_KEYS, ++ ++ OVPN_CMD_DEL_KEY, ++ ++ /** ++ * @OVPN_CMD_REGISTER_PACKET: Register for specific packet types to be ++ * forwarded to userspace ++ */ ++ OVPN_CMD_REGISTER_PACKET, ++ ++ /** ++ * @OVPN_CMD_PACKET: Send a packet from userspace to kernelspace. Also ++ * used to send to userspace packets for which a process had registered ++ * with OVPN_CMD_REGISTER_PACKET ++ */ ++ OVPN_CMD_PACKET, ++ ++ /** ++ * @OVPN_CMD_GET_PEER: Retrieve the status of a peer or all peers ++ */ ++ OVPN_CMD_GET_PEER, ++}; ++ ++enum ovpn_cipher_alg { ++ /** ++ * @OVPN_CIPHER_ALG_NONE: No encryption - reserved for debugging only ++ */ ++ OVPN_CIPHER_ALG_NONE = 0, ++ /** ++ * @OVPN_CIPHER_ALG_AES_GCM: AES-GCM AEAD cipher with any allowed key size ++ */ ++ OVPN_CIPHER_ALG_AES_GCM, ++ /** ++ * @OVPN_CIPHER_ALG_CHACHA20_POLY1305: ChaCha20Poly1305 AEAD cipher ++ */ ++ OVPN_CIPHER_ALG_CHACHA20_POLY1305, ++}; ++ ++enum ovpn_del_peer_reason { ++ __OVPN_DEL_PEER_REASON_FIRST, ++ OVPN_DEL_PEER_REASON_TEARDOWN = __OVPN_DEL_PEER_REASON_FIRST, ++ OVPN_DEL_PEER_REASON_USERSPACE, ++ OVPN_DEL_PEER_REASON_EXPIRED, ++ OVPN_DEL_PEER_REASON_TRANSPORT_ERROR, ++ __OVPN_DEL_PEER_REASON_AFTER_LAST ++}; ++ ++enum ovpn_key_slot { ++ __OVPN_KEY_SLOT_FIRST, ++ OVPN_KEY_SLOT_PRIMARY = __OVPN_KEY_SLOT_FIRST, ++ OVPN_KEY_SLOT_SECONDARY, ++ __OVPN_KEY_SLOT_AFTER_LAST, ++}; ++ ++enum ovpn_netlink_attrs { ++ OVPN_ATTR_UNSPEC = 0, ++ OVPN_ATTR_IFINDEX, ++ OVPN_ATTR_NEW_PEER, ++ OVPN_ATTR_SET_PEER, ++ OVPN_ATTR_DEL_PEER, ++ OVPN_ATTR_NEW_KEY, ++ OVPN_ATTR_SWAP_KEYS, ++ OVPN_ATTR_DEL_KEY, ++ OVPN_ATTR_PACKET, ++ OVPN_ATTR_GET_PEER, ++ ++ __OVPN_ATTR_AFTER_LAST, ++ OVPN_ATTR_MAX = __OVPN_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_key_dir_attrs { ++ OVPN_KEY_DIR_ATTR_UNSPEC = 0, ++ OVPN_KEY_DIR_ATTR_CIPHER_KEY, ++ OVPN_KEY_DIR_ATTR_NONCE_TAIL, ++ ++ __OVPN_KEY_DIR_ATTR_AFTER_LAST, ++ OVPN_KEY_DIR_ATTR_MAX = __OVPN_KEY_DIR_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_new_key_attrs { ++ OVPN_NEW_KEY_ATTR_UNSPEC = 0, ++ OVPN_NEW_KEY_ATTR_PEER_ID, ++ OVPN_NEW_KEY_ATTR_KEY_SLOT, ++ OVPN_NEW_KEY_ATTR_KEY_ID, ++ OVPN_NEW_KEY_ATTR_CIPHER_ALG, ++ OVPN_NEW_KEY_ATTR_ENCRYPT_KEY, ++ OVPN_NEW_KEY_ATTR_DECRYPT_KEY, ++ ++ __OVPN_NEW_KEY_ATTR_AFTER_LAST, ++ OVPN_NEW_KEY_ATTR_MAX = __OVPN_NEW_KEY_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_del_key_attrs { ++ OVPN_DEL_KEY_ATTR_UNSPEC = 0, ++ OVPN_DEL_KEY_ATTR_PEER_ID, ++ OVPN_DEL_KEY_ATTR_KEY_SLOT, ++ ++ __OVPN_DEL_KEY_ATTR_AFTER_LAST, ++ OVPN_DEL_KEY_ATTR_MAX = __OVPN_DEL_KEY_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_swap_keys_attrs { ++ OVPN_SWAP_KEYS_ATTR_UNSPEC = 0, ++ OVPN_SWAP_KEYS_ATTR_PEER_ID, ++ ++ __OVPN_SWAP_KEYS_ATTR_AFTER_LAST, ++ OVPN_SWAP_KEYS_ATTR_MAX = __OVPN_SWAP_KEYS_ATTR_AFTER_LAST - 1, ++ ++}; ++ ++enum ovpn_netlink_new_peer_attrs { ++ OVPN_NEW_PEER_ATTR_UNSPEC = 0, ++ OVPN_NEW_PEER_ATTR_PEER_ID, ++ OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE, ++ OVPN_NEW_PEER_ATTR_SOCKET, ++ OVPN_NEW_PEER_ATTR_IPV4, ++ OVPN_NEW_PEER_ATTR_IPV6, ++ OVPN_NEW_PEER_ATTR_LOCAL_IP, ++ ++ __OVPN_NEW_PEER_ATTR_AFTER_LAST, ++ OVPN_NEW_PEER_ATTR_MAX = __OVPN_NEW_PEER_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_set_peer_attrs { ++ OVPN_SET_PEER_ATTR_UNSPEC = 0, ++ OVPN_SET_PEER_ATTR_PEER_ID, ++ OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL, ++ OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT, ++ ++ __OVPN_SET_PEER_ATTR_AFTER_LAST, ++ OVPN_SET_PEER_ATTR_MAX = __OVPN_SET_PEER_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_del_peer_attrs { ++ OVPN_DEL_PEER_ATTR_UNSPEC = 0, ++ OVPN_DEL_PEER_ATTR_REASON, ++ OVPN_DEL_PEER_ATTR_PEER_ID, ++ ++ __OVPN_DEL_PEER_ATTR_AFTER_LAST, ++ OVPN_DEL_PEER_ATTR_MAX = __OVPN_DEL_PEER_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_get_peer_attrs { ++ OVPN_GET_PEER_ATTR_UNSPEC = 0, ++ OVPN_GET_PEER_ATTR_PEER_ID, ++ ++ __OVPN_GET_PEER_ATTR_AFTER_LAST, ++ OVPN_GET_PEER_ATTR_MAX = __OVPN_GET_PEER_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_get_peer_response_attrs { ++ OVPN_GET_PEER_RESP_ATTR_UNSPEC = 0, ++ OVPN_GET_PEER_RESP_ATTR_PEER_ID, ++ OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, ++ OVPN_GET_PEER_RESP_ATTR_IPV4, ++ OVPN_GET_PEER_RESP_ATTR_IPV6, ++ OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, ++ OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, ++ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, ++ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, ++ OVPN_GET_PEER_RESP_ATTR_RX_BYTES, ++ OVPN_GET_PEER_RESP_ATTR_TX_BYTES, ++ OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, ++ OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, ++ ++ __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST, ++ OVPN_GET_PEER_RESP_ATTR_MAX = __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_peer_stats_attrs { ++ OVPN_PEER_STATS_ATTR_UNSPEC = 0, ++ OVPN_PEER_STATS_BYTES, ++ OVPN_PEER_STATS_PACKETS, ++ ++ __OVPN_PEER_STATS_ATTR_AFTER_LAST, ++ OVPN_PEER_STATS_ATTR_MAX = __OVPN_PEER_STATS_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_peer_attrs { ++ OVPN_PEER_ATTR_UNSPEC = 0, ++ OVPN_PEER_ATTR_PEER_ID, ++ OVPN_PEER_ATTR_SOCKADDR_REMOTE, ++ OVPN_PEER_ATTR_IPV4, ++ OVPN_PEER_ATTR_IPV6, ++ OVPN_PEER_ATTR_LOCAL_IP, ++ OVPN_PEER_ATTR_KEEPALIVE_INTERVAL, ++ OVPN_PEER_ATTR_KEEPALIVE_TIMEOUT, ++ OVPN_PEER_ATTR_ENCRYPT_KEY, ++ OVPN_PEER_ATTR_DECRYPT_KEY, ++ OVPN_PEER_ATTR_RX_STATS, ++ OVPN_PEER_ATTR_TX_STATS, ++ ++ __OVPN_PEER_ATTR_AFTER_LAST, ++ OVPN_PEER_ATTR_MAX = __OVPN_PEER_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_netlink_packet_attrs { ++ OVPN_PACKET_ATTR_UNSPEC = 0, ++ OVPN_PACKET_ATTR_PACKET, ++ OVPN_PACKET_ATTR_PEER_ID, ++ ++ __OVPN_PACKET_ATTR_AFTER_LAST, ++ OVPN_PACKET_ATTR_MAX = __OVPN_PACKET_ATTR_AFTER_LAST - 1, ++}; ++ ++enum ovpn_ifla_attrs { ++ IFLA_OVPN_UNSPEC = 0, ++ IFLA_OVPN_MODE, ++ ++ __IFLA_OVPN_AFTER_LAST, ++ IFLA_OVPN_MAX = __IFLA_OVPN_AFTER_LAST - 1, ++}; ++ ++enum ovpn_mode { ++ __OVPN_MODE_FIRST = 0, ++ OVPN_MODE_P2P = __OVPN_MODE_FIRST, ++ OVPN_MODE_MP, ++ ++ __OVPN_MODE_AFTER_LAST, ++}; ++ ++#endif /* _UAPI_LINUX_OVPN_DCO_H_ */ +diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h +index 4828794efcf8..8008c762e6b8 100644 +--- a/include/uapi/linux/udp.h ++++ b/include/uapi/linux/udp.h +@@ -43,5 +43,6 @@ struct udphdr { + #define UDP_ENCAP_GTP1U 5 /* 3GPP TS 29.060 */ + #define UDP_ENCAP_RXRPC 6 + #define TCP_ENCAP_ESPINTCP 7 /* Yikes, this is really xfrm encap types. */ ++#define UDP_ENCAP_OVPNINUDP 8 /* OpenVPN over UDP connection */ + + #endif /* _UAPI_LINUX_UDP_H */ +-- +2.38.0 + +From 0f8d23628b7a867dabd253cf6d5e21fa8a3a9c08 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Sat, 8 Oct 2022 15:02:07 +0200 +Subject: [PATCH 05/17] mm: multi-gen LRU + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/mm/index.rst | 1 + + Documentation/admin-guide/mm/multigen_lru.rst | 162 + + Documentation/mm/index.rst | 1 + + Documentation/mm/multigen_lru.rst | 159 + + arch/Kconfig | 8 + + arch/arm64/include/asm/pgtable.h | 15 +- + arch/x86/Kconfig | 1 + + arch/x86/include/asm/pgtable.h | 9 +- + arch/x86/mm/pgtable.c | 5 +- + fs/exec.c | 2 + + fs/fuse/dev.c | 3 +- + include/linux/cgroup.h | 15 +- + include/linux/memcontrol.h | 36 + + include/linux/mm.h | 5 + + include/linux/mm_inline.h | 231 +- + include/linux/mm_types.h | 76 + + include/linux/mmzone.h | 216 ++ + include/linux/nodemask.h | 1 + + include/linux/page-flags-layout.h | 16 +- + include/linux/page-flags.h | 4 +- + include/linux/pgtable.h | 17 +- + include/linux/sched.h | 4 + + include/linux/swap.h | 4 + + kernel/bounds.c | 7 + + kernel/cgroup/cgroup-internal.h | 1 - + kernel/exit.c | 1 + + kernel/fork.c | 9 + + kernel/sched/core.c | 1 + + mm/Kconfig | 26 + + mm/huge_memory.c | 3 +- + mm/internal.h | 1 + + mm/memcontrol.c | 28 + + mm/memory.c | 39 +- + mm/mm_init.c | 6 +- + mm/mmzone.c | 2 + + mm/rmap.c | 6 + + mm/swap.c | 54 +- + mm/vmscan.c | 3250 +++++++++++++++-- + mm/workingset.c | 110 +- + 39 files changed, 4249 insertions(+), 286 deletions(-) + create mode 100644 Documentation/admin-guide/mm/multigen_lru.rst + create mode 100644 Documentation/mm/multigen_lru.rst + +diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst +index 1bd11118dfb1..d1064e0ba34a 100644 +--- a/Documentation/admin-guide/mm/index.rst ++++ b/Documentation/admin-guide/mm/index.rst +@@ -32,6 +32,7 @@ the Linux memory management. + idle_page_tracking + ksm + memory-hotplug ++ multigen_lru + nommu-mmap + numa_memory_policy + numaperf +diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst +new file mode 100644 +index 000000000000..33e068830497 +--- /dev/null ++++ b/Documentation/admin-guide/mm/multigen_lru.rst +@@ -0,0 +1,162 @@ ++.. SPDX-License-Identifier: GPL-2.0 ++ ++============= ++Multi-Gen LRU ++============= ++The multi-gen LRU is an alternative LRU implementation that optimizes ++page reclaim and improves performance under memory pressure. Page ++reclaim decides the kernel's caching policy and ability to overcommit ++memory. It directly impacts the kswapd CPU usage and RAM efficiency. ++ ++Quick start ++=========== ++Build the kernel with the following configurations. ++ ++* ``CONFIG_LRU_GEN=y`` ++* ``CONFIG_LRU_GEN_ENABLED=y`` ++ ++All set! ++ ++Runtime options ++=============== ++``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the ++following subsections. ++ ++Kill switch ++----------- ++``enabled`` accepts different values to enable or disable the ++following components. Its default value depends on ++``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled ++unless some of them have unforeseen side effects. Writing to ++``enabled`` has no effect when a component is not supported by the ++hardware, and valid values will be accepted even when the main switch ++is off. ++ ++====== =============================================================== ++Values Components ++====== =============================================================== ++0x0001 The main switch for the multi-gen LRU. ++0x0002 Clearing the accessed bit in leaf page table entries in large ++ batches, when MMU sets it (e.g., on x86). This behavior can ++ theoretically worsen lock contention (mmap_lock). If it is ++ disabled, the multi-gen LRU will suffer a minor performance ++ degradation for workloads that contiguously map hot pages, ++ whose accessed bits can be otherwise cleared by fewer larger ++ batches. ++0x0004 Clearing the accessed bit in non-leaf page table entries as ++ well, when MMU sets it (e.g., on x86). This behavior was not ++ verified on x86 varieties other than Intel and AMD. If it is ++ disabled, the multi-gen LRU will suffer a negligible ++ performance degradation. ++[yYnN] Apply to all the components above. ++====== =============================================================== ++ ++E.g., ++:: ++ ++ echo y >/sys/kernel/mm/lru_gen/enabled ++ cat /sys/kernel/mm/lru_gen/enabled ++ 0x0007 ++ echo 5 >/sys/kernel/mm/lru_gen/enabled ++ cat /sys/kernel/mm/lru_gen/enabled ++ 0x0005 ++ ++Thrashing prevention ++-------------------- ++Personal computers are more sensitive to thrashing because it can ++cause janks (lags when rendering UI) and negatively impact user ++experience. The multi-gen LRU offers thrashing prevention to the ++majority of laptop and desktop users who do not have ``oomd``. ++ ++Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of ++``N`` milliseconds from getting evicted. The OOM killer is triggered ++if this working set cannot be kept in memory. In other words, this ++option works as an adjustable pressure relief valve, and when open, it ++terminates applications that are hopefully not being used. ++ ++Based on the average human detectable lag (~100ms), ``N=1000`` usually ++eliminates intolerable janks due to thrashing. Larger values like ++``N=3000`` make janks less noticeable at the risk of premature OOM ++kills. ++ ++The default value ``0`` means disabled. ++ ++Experimental features ++===================== ++``/sys/kernel/debug/lru_gen`` accepts commands described in the ++following subsections. Multiple command lines are supported, so does ++concatenation with delimiters ``,`` and ``;``. ++ ++``/sys/kernel/debug/lru_gen_full`` provides additional stats for ++debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from ++evicted generations in this file. ++ ++Working set estimation ++---------------------- ++Working set estimation measures how much memory an application needs ++in a given time interval, and it is usually done with little impact on ++the performance of the application. E.g., data centers want to ++optimize job scheduling (bin packing) to improve memory utilizations. ++When a new job comes in, the job scheduler needs to find out whether ++each server it manages can allocate a certain amount of memory for ++this new job before it can pick a candidate. To do so, the job ++scheduler needs to estimate the working sets of the existing jobs. ++ ++When it is read, ``lru_gen`` returns a histogram of numbers of pages ++accessed over different time intervals for each memcg and node. ++``MAX_NR_GENS`` decides the number of bins for each histogram. The ++histograms are noncumulative. ++:: ++ ++ memcg memcg_id memcg_path ++ node node_id ++ min_gen_nr age_in_ms nr_anon_pages nr_file_pages ++ ... ++ max_gen_nr age_in_ms nr_anon_pages nr_file_pages ++ ++Each bin contains an estimated number of pages that have been accessed ++within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages ++and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of ++the former is the largest and that of the latter is the smallest. ++ ++Users can write the following command to ``lru_gen`` to create a new ++generation ``max_gen_nr+1``: ++ ++ ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` ++ ++``can_swap`` defaults to the swap setting and, if it is set to ``1``, ++it forces the scan of anon pages when swap is off, and vice versa. ++``force_scan`` defaults to ``1`` and, if it is set to ``0``, it ++employs heuristics to reduce the overhead, which is likely to reduce ++the coverage as well. ++ ++A typical use case is that a job scheduler runs this command at a ++certain time interval to create new generations, and it ranks the ++servers it manages based on the sizes of their cold pages defined by ++this time interval. ++ ++Proactive reclaim ++----------------- ++Proactive reclaim induces page reclaim when there is no memory ++pressure. It usually targets cold pages only. E.g., when a new job ++comes in, the job scheduler wants to proactively reclaim cold pages on ++the server it selected, to improve the chance of successfully landing ++this new job. ++ ++Users can write the following command to ``lru_gen`` to evict ++generations less than or equal to ``min_gen_nr``. ++ ++ ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` ++ ++``min_gen_nr`` should be less than ``max_gen_nr-1``, since ++``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to ++the active list) and therefore cannot be evicted. ``swappiness`` ++overrides the default value in ``/proc/sys/vm/swappiness``. ++``nr_to_reclaim`` limits the number of pages to evict. ++ ++A typical use case is that a job scheduler runs this command before it ++tries to land a new job on a server. If it fails to materialize enough ++cold pages because of the overestimation, it retries on the next ++server according to the ranking result obtained from the working set ++estimation step. This less forceful approach limits the impacts on the ++existing jobs. +diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst +index 575ccd40e30c..4aa12b8be278 100644 +--- a/Documentation/mm/index.rst ++++ b/Documentation/mm/index.rst +@@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. + ksm + memory-model + mmu_notifier ++ multigen_lru + numa + overcommit-accounting + page_migration +diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst +new file mode 100644 +index 000000000000..d7062c6a8946 +--- /dev/null ++++ b/Documentation/mm/multigen_lru.rst +@@ -0,0 +1,159 @@ ++.. SPDX-License-Identifier: GPL-2.0 ++ ++============= ++Multi-Gen LRU ++============= ++The multi-gen LRU is an alternative LRU implementation that optimizes ++page reclaim and improves performance under memory pressure. Page ++reclaim decides the kernel's caching policy and ability to overcommit ++memory. It directly impacts the kswapd CPU usage and RAM efficiency. ++ ++Design overview ++=============== ++Objectives ++---------- ++The design objectives are: ++ ++* Good representation of access recency ++* Try to profit from spatial locality ++* Fast paths to make obvious choices ++* Simple self-correcting heuristics ++ ++The representation of access recency is at the core of all LRU ++implementations. In the multi-gen LRU, each generation represents a ++group of pages with similar access recency. Generations establish a ++(time-based) common frame of reference and therefore help make better ++choices, e.g., between different memcgs on a computer or different ++computers in a data center (for job scheduling). ++ ++Exploiting spatial locality improves efficiency when gathering the ++accessed bit. A rmap walk targets a single page and does not try to ++profit from discovering a young PTE. A page table walk can sweep all ++the young PTEs in an address space, but the address space can be too ++sparse to make a profit. The key is to optimize both methods and use ++them in combination. ++ ++Fast paths reduce code complexity and runtime overhead. Unmapped pages ++do not require TLB flushes; clean pages do not require writeback. ++These facts are only helpful when other conditions, e.g., access ++recency, are similar. With generations as a common frame of reference, ++additional factors stand out. But obvious choices might not be good ++choices; thus self-correction is necessary. ++ ++The benefits of simple self-correcting heuristics are self-evident. ++Again, with generations as a common frame of reference, this becomes ++attainable. Specifically, pages in the same generation can be ++categorized based on additional factors, and a feedback loop can ++statistically compare the refault percentages across those categories ++and infer which of them are better choices. ++ ++Assumptions ++----------- ++The protection of hot pages and the selection of cold pages are based ++on page access channels and patterns. There are two access channels: ++ ++* Accesses through page tables ++* Accesses through file descriptors ++ ++The protection of the former channel is by design stronger because: ++ ++1. The uncertainty in determining the access patterns of the former ++ channel is higher due to the approximation of the accessed bit. ++2. The cost of evicting the former channel is higher due to the TLB ++ flushes required and the likelihood of encountering the dirty bit. ++3. The penalty of underprotecting the former channel is higher because ++ applications usually do not prepare themselves for major page ++ faults like they do for blocked I/O. E.g., GUI applications ++ commonly use dedicated I/O threads to avoid blocking rendering ++ threads. ++ ++There are also two access patterns: ++ ++* Accesses exhibiting temporal locality ++* Accesses not exhibiting temporal locality ++ ++For the reasons listed above, the former channel is assumed to follow ++the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is ++present, and the latter channel is assumed to follow the latter ++pattern unless outlying refaults have been observed. ++ ++Workflow overview ++================= ++Evictable pages are divided into multiple generations for each ++``lruvec``. The youngest generation number is stored in ++``lrugen->max_seq`` for both anon and file types as they are aged on ++an equal footing. The oldest generation numbers are stored in ++``lrugen->min_seq[]`` separately for anon and file types as clean file ++pages can be evicted regardless of swap constraints. These three ++variables are monotonically increasing. ++ ++Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` ++bits in order to fit into the gen counter in ``folio->flags``. Each ++truncated generation number is an index to ``lrugen->lists[]``. The ++sliding window technique is used to track at least ``MIN_NR_GENS`` and ++at most ``MAX_NR_GENS`` generations. The gen counter stores a value ++within ``[1, MAX_NR_GENS]`` while a page is on one of ++``lrugen->lists[]``; otherwise it stores zero. ++ ++Each generation is divided into multiple tiers. A page accessed ``N`` ++times through file descriptors is in tier ``order_base_2(N)``. Unlike ++generations, tiers do not have dedicated ``lrugen->lists[]``. In ++contrast to moving across generations, which requires the LRU lock, ++moving across tiers only involves atomic operations on ++``folio->flags`` and therefore has a negligible cost. A feedback loop ++modeled after the PID controller monitors refaults over all the tiers ++from anon and file types and decides which tiers from which types to ++evict or protect. ++ ++There are two conceptually independent procedures: the aging and the ++eviction. They form a closed-loop system, i.e., the page reclaim. ++ ++Aging ++----- ++The aging produces young generations. Given an ``lruvec``, it ++increments ``max_seq`` when ``max_seq-min_seq+1`` approaches ++``MIN_NR_GENS``. The aging promotes hot pages to the youngest ++generation when it finds them accessed through page tables; the ++demotion of cold pages happens consequently when it increments ++``max_seq``. The aging uses page table walks and rmap walks to find ++young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` ++and calls ``walk_page_range()`` with each ``mm_struct`` on this list ++to scan PTEs, and after each iteration, it increments ``max_seq``. For ++the latter, when the eviction walks the rmap and finds a young PTE, ++the aging scans the adjacent PTEs. For both, on finding a young PTE, ++the aging clears the accessed bit and updates the gen counter of the ++page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. ++ ++Eviction ++-------- ++The eviction consumes old generations. Given an ``lruvec``, it ++increments ``min_seq`` when ``lrugen->lists[]`` indexed by ++``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to ++evict from, it first compares ``min_seq[]`` to select the older type. ++If both types are equally old, it selects the one whose first tier has ++a lower refault percentage. The first tier contains single-use ++unmapped clean pages, which are the best bet. The eviction sorts a ++page according to its gen counter if the aging has found this page ++accessed through page tables and updated its gen counter. It also ++moves a page to the next generation, i.e., ``min_seq+1``, if this page ++was accessed multiple times through file descriptors and the feedback ++loop has detected outlying refaults from the tier this page is in. To ++this end, the feedback loop uses the first tier as the baseline, for ++the reason stated earlier. ++ ++Summary ++------- ++The multi-gen LRU can be disassembled into the following parts: ++ ++* Generations ++* Rmap walks ++* Page table walks ++* Bloom filters ++* PID controller ++ ++The aging and the eviction form a producer-consumer model; ++specifically, the latter drives the former by the sliding window over ++generations. Within the aging, rmap walks drive page table walks by ++inserting hot densely populated page tables to the Bloom filters. ++Within the eviction, the PID controller uses refaults as the feedback ++to select types to evict and tiers to protect. +diff --git a/arch/Kconfig b/arch/Kconfig +index 8b311e400ec1..bf19a84fffa2 100644 +--- a/arch/Kconfig ++++ b/arch/Kconfig +@@ -1418,6 +1418,14 @@ config DYNAMIC_SIGFRAME + config HAVE_ARCH_NODE_DEV_GROUP + bool + ++config ARCH_HAS_NONLEAF_PMD_YOUNG ++ bool ++ help ++ Architectures that select this option are capable of setting the ++ accessed bit in non-leaf PMD entries when using them as part of linear ++ address translations. Page table walkers that clear the accessed bit ++ may use this capability to reduce their search space. ++ + source "kernel/gcov/Kconfig" + + source "scripts/gcc-plugins/Kconfig" +diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h +index b5df82aa99e6..71a1af42f0e8 100644 +--- a/arch/arm64/include/asm/pgtable.h ++++ b/arch/arm64/include/asm/pgtable.h +@@ -1082,24 +1082,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, + * page after fork() + CoW for pfn mappings. We don't always have a + * hardware-managed access flag on arm64. + */ +-static inline bool arch_faults_on_old_pte(void) +-{ +- /* The register read below requires a stable CPU to make any sense */ +- cant_migrate(); +- +- return !cpu_has_hw_af(); +-} +-#define arch_faults_on_old_pte arch_faults_on_old_pte ++#define arch_has_hw_pte_young cpu_has_hw_af + + /* + * Experimentally, it's cheap to set the access flag in hardware and we + * benefit from prefaulting mappings as 'old' to start with. + */ +-static inline bool arch_wants_old_prefaulted_pte(void) +-{ +- return !arch_faults_on_old_pte(); +-} +-#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte ++#define arch_wants_old_prefaulted_pte cpu_has_hw_af + + static inline bool pud_sect_supported(void) + { +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index f9920f1341c8..674d694a665e 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -85,6 +85,7 @@ config X86 + select ARCH_HAS_PMEM_API if X86_64 + select ARCH_HAS_PTE_DEVMAP if X86_64 + select ARCH_HAS_PTE_SPECIAL ++ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 + select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 + select ARCH_HAS_COPY_MC if X86_64 + select ARCH_HAS_SET_MEMORY +diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h +index 44e2d6f1dbaa..5059799bebe3 100644 +--- a/arch/x86/include/asm/pgtable.h ++++ b/arch/x86/include/asm/pgtable.h +@@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) + + static inline int pmd_bad(pmd_t pmd) + { +- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; ++ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != ++ (_KERNPG_TABLE & ~_PAGE_ACCESSED); + } + + static inline unsigned long pages_to_mb(unsigned long npg) +@@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) + return boot_cpu_has_bug(X86_BUG_L1TF); + } + +-#define arch_faults_on_old_pte arch_faults_on_old_pte +-static inline bool arch_faults_on_old_pte(void) ++#define arch_has_hw_pte_young arch_has_hw_pte_young ++static inline bool arch_has_hw_pte_young(void) + { +- return false; ++ return true; + } + + #ifdef CONFIG_PAGE_TABLE_CHECK +diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c +index a932d7712d85..8525f2876fb4 100644 +--- a/arch/x86/mm/pgtable.c ++++ b/arch/x86/mm/pgtable.c +@@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, + return ret; + } + +-#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) + int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) + { +@@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, + + return ret; + } ++#endif ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE + int pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pud_t *pudp) + { +diff --git a/fs/exec.c b/fs/exec.c +index d046dbb9cbd0..c67b12f0f577 100644 +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -1011,6 +1011,7 @@ static int exec_mmap(struct mm_struct *mm) + active_mm = tsk->active_mm; + tsk->active_mm = mm; + tsk->mm = mm; ++ lru_gen_add_mm(mm); + /* + * This prevents preemption while active_mm is being loaded and + * it and mm are being updated, which could cause problems for +@@ -1026,6 +1027,7 @@ static int exec_mmap(struct mm_struct *mm) + tsk->mm->vmacache_seqnum = 0; + vmacache_flush(tsk); + task_unlock(tsk); ++ lru_gen_use_mm(mm); + if (old_mm) { + mmap_read_unlock(old_mm); + BUG_ON(active_mm != old_mm); +diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c +index 51897427a534..b4a6e0a1b945 100644 +--- a/fs/fuse/dev.c ++++ b/fs/fuse/dev.c +@@ -776,7 +776,8 @@ static int fuse_check_page(struct page *page) + 1 << PG_active | + 1 << PG_workingset | + 1 << PG_reclaim | +- 1 << PG_waiters))) { ++ 1 << PG_waiters | ++ LRU_GEN_MASK | LRU_REFS_MASK))) { + dump_page(page, "fuse: trying to steal weird page"); + return 1; + } +diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h +index ac5d0515680e..9179463c3c9f 100644 +--- a/include/linux/cgroup.h ++++ b/include/linux/cgroup.h +@@ -432,6 +432,18 @@ static inline void cgroup_put(struct cgroup *cgrp) + css_put(&cgrp->self); + } + ++extern struct mutex cgroup_mutex; ++ ++static inline void cgroup_lock(void) ++{ ++ mutex_lock(&cgroup_mutex); ++} ++ ++static inline void cgroup_unlock(void) ++{ ++ mutex_unlock(&cgroup_mutex); ++} ++ + /** + * task_css_set_check - obtain a task's css_set with extra access conditions + * @task: the task to obtain css_set for +@@ -446,7 +458,6 @@ static inline void cgroup_put(struct cgroup *cgrp) + * as locks used during the cgroup_subsys::attach() methods. + */ + #ifdef CONFIG_PROVE_RCU +-extern struct mutex cgroup_mutex; + extern spinlock_t css_set_lock; + #define task_css_set_check(task, __c) \ + rcu_dereference_check((task)->cgroups, \ +@@ -708,6 +719,8 @@ struct cgroup; + static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } + static inline void css_get(struct cgroup_subsys_state *css) {} + static inline void css_put(struct cgroup_subsys_state *css) {} ++static inline void cgroup_lock(void) {} ++static inline void cgroup_unlock(void) {} + static inline int cgroup_attach_task_all(struct task_struct *from, + struct task_struct *t) { return 0; } + static inline int cgroupstats_build(struct cgroupstats *stats, +diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h +index 567f12323f55..877cbcbc6ed9 100644 +--- a/include/linux/memcontrol.h ++++ b/include/linux/memcontrol.h +@@ -350,6 +350,11 @@ struct mem_cgroup { + struct deferred_split deferred_split_queue; + #endif + ++#ifdef CONFIG_LRU_GEN ++ /* per-memcg mm_struct list */ ++ struct lru_gen_mm_list mm_list; ++#endif ++ + struct mem_cgroup_per_node *nodeinfo[]; + }; + +@@ -444,6 +449,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) + * - LRU isolation + * - lock_page_memcg() + * - exclusive reference ++ * - mem_cgroup_trylock_pages() + * + * For a kmem folio a caller should hold an rcu read lock to protect memcg + * associated with a kmem folio from being released. +@@ -505,6 +511,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) + * - LRU isolation + * - lock_page_memcg() + * - exclusive reference ++ * - mem_cgroup_trylock_pages() + * + * For a kmem page a caller should hold an rcu read lock to protect memcg + * associated with a kmem page from being released. +@@ -959,6 +966,23 @@ void unlock_page_memcg(struct page *page); + + void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); + ++/* try to stablize folio_memcg() for all the pages in a memcg */ ++static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) ++{ ++ rcu_read_lock(); ++ ++ if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) ++ return true; ++ ++ rcu_read_unlock(); ++ return false; ++} ++ ++static inline void mem_cgroup_unlock_pages(void) ++{ ++ rcu_read_unlock(); ++} ++ + /* idx can be of type enum memcg_stat_item or node_stat_item */ + static inline void mod_memcg_state(struct mem_cgroup *memcg, + int idx, int val) +@@ -1433,6 +1457,18 @@ static inline void folio_memcg_unlock(struct folio *folio) + { + } + ++static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) ++{ ++ /* to match folio_memcg_rcu() */ ++ rcu_read_lock(); ++ return true; ++} ++ ++static inline void mem_cgroup_unlock_pages(void) ++{ ++ rcu_read_unlock(); ++} ++ + static inline void mem_cgroup_handle_over_high(void) + { + } +diff --git a/include/linux/mm.h b/include/linux/mm.h +index 21f8b27bd9fd..88976a521ef5 100644 +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -1465,6 +1465,11 @@ static inline unsigned long folio_pfn(struct folio *folio) + return page_to_pfn(&folio->page); + } + ++static inline struct folio *pfn_folio(unsigned long pfn) ++{ ++ return page_folio(pfn_to_page(pfn)); ++} ++ + static inline atomic_t *folio_pincount_ptr(struct folio *folio) + { + return &folio_page(folio, 1)->compound_pincount; +diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h +index 7b25b53c474a..4949eda9a9a2 100644 +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) + return folio_is_file_lru(page_folio(page)); + } + +-static __always_inline void update_lru_size(struct lruvec *lruvec, ++static __always_inline void __update_lru_size(struct lruvec *lruvec, + enum lru_list lru, enum zone_type zid, + long nr_pages) + { + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + ++ lockdep_assert_held(&lruvec->lru_lock); ++ WARN_ON_ONCE(nr_pages != (int)nr_pages); ++ + __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); + __mod_zone_page_state(&pgdat->node_zones[zid], + NR_ZONE_LRU_BASE + lru, nr_pages); ++} ++ ++static __always_inline void update_lru_size(struct lruvec *lruvec, ++ enum lru_list lru, enum zone_type zid, ++ long nr_pages) ++{ ++ __update_lru_size(lruvec, lru, zid, nr_pages); + #ifdef CONFIG_MEMCG + mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); + #endif +@@ -94,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) + return lru; + } + ++#ifdef CONFIG_LRU_GEN ++ ++#ifdef CONFIG_LRU_GEN_ENABLED ++static inline bool lru_gen_enabled(void) ++{ ++ DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); ++ ++ return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); ++} ++#else ++static inline bool lru_gen_enabled(void) ++{ ++ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); ++ ++ return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); ++} ++#endif ++ ++static inline bool lru_gen_in_fault(void) ++{ ++ return current->in_lru_fault; ++} ++ ++static inline int lru_gen_from_seq(unsigned long seq) ++{ ++ return seq % MAX_NR_GENS; ++} ++ ++static inline int lru_hist_from_seq(unsigned long seq) ++{ ++ return seq % NR_HIST_GENS; ++} ++ ++static inline int lru_tier_from_refs(int refs) ++{ ++ VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); ++ ++ /* see the comment in folio_lru_refs() */ ++ return order_base_2(refs + 1); ++} ++ ++static inline int folio_lru_refs(struct folio *folio) ++{ ++ unsigned long flags = READ_ONCE(folio->flags); ++ bool workingset = flags & BIT(PG_workingset); ++ ++ /* ++ * Return the number of accesses beyond PG_referenced, i.e., N-1 if the ++ * total number of accesses is N>1, since N=0,1 both map to the first ++ * tier. lru_tier_from_refs() will account for this off-by-one. Also see ++ * the comment on MAX_NR_TIERS. ++ */ ++ return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; ++} ++ ++static inline int folio_lru_gen(struct folio *folio) ++{ ++ unsigned long flags = READ_ONCE(folio->flags); ++ ++ return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++} ++ ++static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) ++{ ++ unsigned long max_seq = lruvec->lrugen.max_seq; ++ ++ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); ++ ++ /* see the comment on MIN_NR_GENS */ ++ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); ++} ++ ++static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, ++ int old_gen, int new_gen) ++{ ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ int delta = folio_nr_pages(folio); ++ enum lru_list lru = type * LRU_INACTIVE_FILE; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); ++ ++ if (old_gen >= 0) ++ WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], ++ lrugen->nr_pages[old_gen][type][zone] - delta); ++ if (new_gen >= 0) ++ WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], ++ lrugen->nr_pages[new_gen][type][zone] + delta); ++ ++ /* addition */ ++ if (old_gen < 0) { ++ if (lru_gen_is_active(lruvec, new_gen)) ++ lru += LRU_ACTIVE; ++ __update_lru_size(lruvec, lru, zone, delta); ++ return; ++ } ++ ++ /* deletion */ ++ if (new_gen < 0) { ++ if (lru_gen_is_active(lruvec, old_gen)) ++ lru += LRU_ACTIVE; ++ __update_lru_size(lruvec, lru, zone, -delta); ++ return; ++ } ++ ++ /* promotion */ ++ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { ++ __update_lru_size(lruvec, lru, zone, -delta); ++ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); ++ } ++ ++ /* demotion requires isolation, e.g., lru_deactivate_fn() */ ++ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); ++} ++ ++static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ unsigned long seq; ++ unsigned long flags; ++ int gen = folio_lru_gen(folio); ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); ++ ++ if (folio_test_unevictable(folio) || !lrugen->enabled) ++ return false; ++ /* ++ * There are three common cases for this page: ++ * 1. If it's hot, e.g., freshly faulted in or previously hot and ++ * migrated, add it to the youngest generation. ++ * 2. If it's cold but can't be evicted immediately, i.e., an anon page ++ * not in swapcache or a dirty page pending writeback, add it to the ++ * second oldest generation. ++ * 3. Everything else (clean, cold) is added to the oldest generation. ++ */ ++ if (folio_test_active(folio)) ++ seq = lrugen->max_seq; ++ else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || ++ (folio_test_reclaim(folio) && ++ (folio_test_dirty(folio) || folio_test_writeback(folio)))) ++ seq = lrugen->min_seq[type] + 1; ++ else ++ seq = lrugen->min_seq[type]; ++ ++ gen = lru_gen_from_seq(seq); ++ flags = (gen + 1UL) << LRU_GEN_PGOFF; ++ /* see the comment on MIN_NR_GENS about PG_active */ ++ set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); ++ ++ lru_gen_update_size(lruvec, folio, -1, gen); ++ /* for folio_rotate_reclaimable() */ ++ if (reclaiming) ++ list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); ++ else ++ list_add(&folio->lru, &lrugen->lists[gen][type][zone]); ++ ++ return true; ++} ++ ++static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ unsigned long flags; ++ int gen = folio_lru_gen(folio); ++ ++ if (gen < 0) ++ return false; ++ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ ++ /* for folio_migrate_flags() */ ++ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; ++ flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); ++ gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ ++ lru_gen_update_size(lruvec, folio, gen, -1); ++ list_del(&folio->lru); ++ ++ return true; ++} ++ ++#else /* !CONFIG_LRU_GEN */ ++ ++static inline bool lru_gen_enabled(void) ++{ ++ return false; ++} ++ ++static inline bool lru_gen_in_fault(void) ++{ ++ return false; ++} ++ ++static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ return false; ++} ++ ++static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + static __always_inline + void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) + { + enum lru_list lru = folio_lru_list(folio); + ++ if (lru_gen_add_folio(lruvec, folio, false)) ++ return; ++ + update_lru_size(lruvec, lru, folio_zonenum(folio), + folio_nr_pages(folio)); + if (lru != LRU_UNEVICTABLE) +@@ -116,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) + { + enum lru_list lru = folio_lru_list(folio); + ++ if (lru_gen_add_folio(lruvec, folio, true)) ++ return; ++ + update_lru_size(lruvec, lru, folio_zonenum(folio), + folio_nr_pages(folio)); + /* This is not expected to be used on LRU_UNEVICTABLE */ +@@ -133,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) + { + enum lru_list lru = folio_lru_list(folio); + ++ if (lru_gen_del_folio(lruvec, folio, false)) ++ return; ++ + if (lru != LRU_UNEVICTABLE) + list_del(&folio->lru); + update_lru_size(lruvec, lru, folio_zonenum(folio), +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index cf97f3884fda..e1797813cc2c 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -672,6 +672,22 @@ struct mm_struct { + */ + unsigned long ksm_merging_pages; + #endif ++#ifdef CONFIG_LRU_GEN ++ struct { ++ /* this mm_struct is on lru_gen_mm_list */ ++ struct list_head list; ++ /* ++ * Set when switching to this mm_struct, as a hint of ++ * whether it has been used since the last time per-node ++ * page table walkers cleared the corresponding bits. ++ */ ++ unsigned long bitmap; ++#ifdef CONFIG_MEMCG ++ /* points to the memcg of "owner" above */ ++ struct mem_cgroup *memcg; ++#endif ++ } lru_gen; ++#endif /* CONFIG_LRU_GEN */ + } __randomize_layout; + + /* +@@ -698,6 +714,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) + return (struct cpumask *)&mm->cpu_bitmap; + } + ++#ifdef CONFIG_LRU_GEN ++ ++struct lru_gen_mm_list { ++ /* mm_struct list for page table walkers */ ++ struct list_head fifo; ++ /* protects the list above */ ++ spinlock_t lock; ++}; ++ ++void lru_gen_add_mm(struct mm_struct *mm); ++void lru_gen_del_mm(struct mm_struct *mm); ++#ifdef CONFIG_MEMCG ++void lru_gen_migrate_mm(struct mm_struct *mm); ++#endif ++ ++static inline void lru_gen_init_mm(struct mm_struct *mm) ++{ ++ INIT_LIST_HEAD(&mm->lru_gen.list); ++ mm->lru_gen.bitmap = 0; ++#ifdef CONFIG_MEMCG ++ mm->lru_gen.memcg = NULL; ++#endif ++} ++ ++static inline void lru_gen_use_mm(struct mm_struct *mm) ++{ ++ /* ++ * When the bitmap is set, page reclaim knows this mm_struct has been ++ * used since the last time it cleared the bitmap. So it might be worth ++ * walking the page tables of this mm_struct to clear the accessed bit. ++ */ ++ WRITE_ONCE(mm->lru_gen.bitmap, -1); ++} ++ ++#else /* !CONFIG_LRU_GEN */ ++ ++static inline void lru_gen_add_mm(struct mm_struct *mm) ++{ ++} ++ ++static inline void lru_gen_del_mm(struct mm_struct *mm) ++{ ++} ++ ++#ifdef CONFIG_MEMCG ++static inline void lru_gen_migrate_mm(struct mm_struct *mm) ++{ ++} ++#endif ++ ++static inline void lru_gen_init_mm(struct mm_struct *mm) ++{ ++} ++ ++static inline void lru_gen_use_mm(struct mm_struct *mm) ++{ ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index e24b40c52468..1543001feba9 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -306,6 +306,8 @@ static inline bool is_active_lru(enum lru_list lru) + return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); + } + ++#define WORKINGSET_ANON 0 ++#define WORKINGSET_FILE 1 + #define ANON_AND_FILE 2 + + enum lruvec_flags { +@@ -314,6 +316,207 @@ enum lruvec_flags { + */ + }; + ++#endif /* !__GENERATING_BOUNDS_H */ ++ ++/* ++ * Evictable pages are divided into multiple generations. The youngest and the ++ * oldest generation numbers, max_seq and min_seq, are monotonically increasing. ++ * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An ++ * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the ++ * corresponding generation. The gen counter in folio->flags stores gen+1 while ++ * a page is on one of lrugen->lists[]. Otherwise it stores 0. ++ * ++ * A page is added to the youngest generation on faulting. The aging needs to ++ * check the accessed bit at least twice before handing this page over to the ++ * eviction. The first check takes care of the accessed bit set on the initial ++ * fault; the second check makes sure this page hasn't been used since then. ++ * This process, AKA second chance, requires a minimum of two generations, ++ * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive ++ * LRU, e.g., /proc/vmstat, these two generations are considered active; the ++ * rest of generations, if they exist, are considered inactive. See ++ * lru_gen_is_active(). ++ * ++ * PG_active is always cleared while a page is on one of lrugen->lists[] so that ++ * the aging needs not to worry about it. And it's set again when a page ++ * considered active is isolated for non-reclaiming purposes, e.g., migration. ++ * See lru_gen_add_folio() and lru_gen_del_folio(). ++ * ++ * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the ++ * number of categories of the active/inactive LRU when keeping track of ++ * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits ++ * in folio->flags. ++ */ ++#define MIN_NR_GENS 2U ++#define MAX_NR_GENS 4U ++ ++/* ++ * Each generation is divided into multiple tiers. A page accessed N times ++ * through file descriptors is in tier order_base_2(N). A page in the first tier ++ * (N=0,1) is marked by PG_referenced unless it was faulted in through page ++ * tables or read ahead. A page in any other tier (N>1) is marked by ++ * PG_referenced and PG_workingset. This implies a minimum of two tiers is ++ * supported without using additional bits in folio->flags. ++ * ++ * In contrast to moving across generations which requires the LRU lock, moving ++ * across tiers only involves atomic operations on folio->flags and therefore ++ * has a negligible cost in the buffered access path. In the eviction path, ++ * comparisons of refaulted/(evicted+protected) from the first tier and the ++ * rest infer whether pages accessed multiple times through file descriptors ++ * are statistically hot and thus worth protecting. ++ * ++ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the ++ * number of categories of the active/inactive LRU when keeping track of ++ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in ++ * folio->flags. ++ */ ++#define MAX_NR_TIERS 4U ++ ++#ifndef __GENERATING_BOUNDS_H ++ ++struct lruvec; ++struct page_vma_mapped_walk; ++ ++#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) ++#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) ++ ++#ifdef CONFIG_LRU_GEN ++ ++enum { ++ LRU_GEN_ANON, ++ LRU_GEN_FILE, ++}; ++ ++enum { ++ LRU_GEN_CORE, ++ LRU_GEN_MM_WALK, ++ LRU_GEN_NONLEAF_YOUNG, ++ NR_LRU_GEN_CAPS ++}; ++ ++#define MIN_LRU_BATCH BITS_PER_LONG ++#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) ++ ++/* whether to keep historical stats from evicted generations */ ++#ifdef CONFIG_LRU_GEN_STATS ++#define NR_HIST_GENS MAX_NR_GENS ++#else ++#define NR_HIST_GENS 1U ++#endif ++ ++/* ++ * The youngest generation number is stored in max_seq for both anon and file ++ * types as they are aged on an equal footing. The oldest generation numbers are ++ * stored in min_seq[] separately for anon and file types as clean file pages ++ * can be evicted regardless of swap constraints. ++ * ++ * Normally anon and file min_seq are in sync. But if swapping is constrained, ++ * e.g., out of swap space, file min_seq is allowed to advance and leave anon ++ * min_seq behind. ++ * ++ * The number of pages in each generation is eventually consistent and therefore ++ * can be transiently negative when reset_batch_size() is pending. ++ */ ++struct lru_gen_struct { ++ /* the aging increments the youngest generation number */ ++ unsigned long max_seq; ++ /* the eviction increments the oldest generation numbers */ ++ unsigned long min_seq[ANON_AND_FILE]; ++ /* the birth time of each generation in jiffies */ ++ unsigned long timestamps[MAX_NR_GENS]; ++ /* the multi-gen LRU lists, lazily sorted on eviction */ ++ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* the multi-gen LRU sizes, eventually consistent */ ++ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* the exponential moving average of refaulted */ ++ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; ++ /* the exponential moving average of evicted+protected */ ++ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; ++ /* the first tier doesn't need protection, hence the minus one */ ++ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; ++ /* can be modified without holding the LRU lock */ ++ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; ++ atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; ++ /* whether the multi-gen LRU is enabled */ ++ bool enabled; ++}; ++ ++enum { ++ MM_LEAF_TOTAL, /* total leaf entries */ ++ MM_LEAF_OLD, /* old leaf entries */ ++ MM_LEAF_YOUNG, /* young leaf entries */ ++ MM_NONLEAF_TOTAL, /* total non-leaf entries */ ++ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ ++ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ ++ NR_MM_STATS ++}; ++ ++/* double-buffering Bloom filters */ ++#define NR_BLOOM_FILTERS 2 ++ ++struct lru_gen_mm_state { ++ /* set to max_seq after each iteration */ ++ unsigned long seq; ++ /* where the current iteration continues (inclusive) */ ++ struct list_head *head; ++ /* where the last iteration ended (exclusive) */ ++ struct list_head *tail; ++ /* to wait for the last page table walker to finish */ ++ struct wait_queue_head wait; ++ /* Bloom filters flip after each iteration */ ++ unsigned long *filters[NR_BLOOM_FILTERS]; ++ /* the mm stats for debugging */ ++ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; ++ /* the number of concurrent page table walkers */ ++ int nr_walkers; ++}; ++ ++struct lru_gen_mm_walk { ++ /* the lruvec under reclaim */ ++ struct lruvec *lruvec; ++ /* unstable max_seq from lru_gen_struct */ ++ unsigned long max_seq; ++ /* the next address within an mm to scan */ ++ unsigned long next_addr; ++ /* to batch promoted pages */ ++ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* to batch the mm stats */ ++ int mm_stats[NR_MM_STATS]; ++ /* total batched items */ ++ int batched; ++ bool can_swap; ++ bool force_scan; ++}; ++ ++void lru_gen_init_lruvec(struct lruvec *lruvec); ++void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); ++ ++#ifdef CONFIG_MEMCG ++void lru_gen_init_memcg(struct mem_cgroup *memcg); ++void lru_gen_exit_memcg(struct mem_cgroup *memcg); ++#endif ++ ++#else /* !CONFIG_LRU_GEN */ ++ ++static inline void lru_gen_init_lruvec(struct lruvec *lruvec) ++{ ++} ++ ++static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) ++{ ++} ++ ++#ifdef CONFIG_MEMCG ++static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) ++{ ++} ++ ++static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) ++{ ++} ++#endif ++ ++#endif /* CONFIG_LRU_GEN */ ++ + struct lruvec { + struct list_head lists[NR_LRU_LISTS]; + /* per lruvec lru_lock for memcg */ +@@ -331,6 +534,12 @@ struct lruvec { + unsigned long refaults[ANON_AND_FILE]; + /* Various lruvec state flags (enum lruvec_flags) */ + unsigned long flags; ++#ifdef CONFIG_LRU_GEN ++ /* evictable pages divided into generations */ ++ struct lru_gen_struct lrugen; ++ /* to concurrently iterate lru_gen_mm_list */ ++ struct lru_gen_mm_state mm_state; ++#endif + #ifdef CONFIG_MEMCG + struct pglist_data *pgdat; + #endif +@@ -746,6 +955,8 @@ static inline bool zone_is_empty(struct zone *zone) + #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) + #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) + #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) ++#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) ++#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) + + /* + * Define the bit shifts to access each section. For non-existent +@@ -1007,6 +1218,11 @@ typedef struct pglist_data { + + unsigned long flags; + ++#ifdef CONFIG_LRU_GEN ++ /* kswap mm walk data */ ++ struct lru_gen_mm_walk mm_walk; ++#endif ++ + ZONE_PADDING(_pad2_) + + /* Per-node vmstats */ +diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h +index 4b71a96190a8..3a0eec9f2faa 100644 +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) + #define first_online_node 0 + #define first_memory_node 0 + #define next_online_node(nid) (MAX_NUMNODES) ++#define next_memory_node(nid) (MAX_NUMNODES) + #define nr_node_ids 1U + #define nr_online_nodes 1U + +diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h +index ef1e3e736e14..7d79818dc065 100644 +--- a/include/linux/page-flags-layout.h ++++ b/include/linux/page-flags-layout.h +@@ -55,7 +55,8 @@ + #define SECTIONS_WIDTH 0 + #endif + +-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS ++#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ ++ <= BITS_PER_LONG - NR_PAGEFLAGS + #define NODES_WIDTH NODES_SHIFT + #elif defined(CONFIG_SPARSEMEM_VMEMMAP) + #error "Vmemmap: No space for nodes field in page flags" +@@ -89,8 +90,8 @@ + #define LAST_CPUPID_SHIFT 0 + #endif + +-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ +- <= BITS_PER_LONG - NR_PAGEFLAGS ++#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ ++ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS + #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT + #else + #define LAST_CPUPID_WIDTH 0 +@@ -100,10 +101,15 @@ + #define LAST_CPUPID_NOT_IN_PAGE_FLAGS + #endif + +-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ +- > BITS_PER_LONG - NR_PAGEFLAGS ++#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ ++ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS + #error "Not enough bits in page flags" + #endif + ++/* see the comment on MAX_NR_TIERS */ ++#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ ++ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ ++ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) ++ + #endif + #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ +diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h +index 465ff35a8c00..0b0ae5084e60 100644 +--- a/include/linux/page-flags.h ++++ b/include/linux/page-flags.h +@@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) + 1UL << PG_private | 1UL << PG_private_2 | \ + 1UL << PG_writeback | 1UL << PG_reserved | \ + 1UL << PG_slab | 1UL << PG_active | \ +- 1UL << PG_unevictable | __PG_MLOCKED) ++ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) + + /* + * Flags checked when a page is prepped for return by the page allocator. +@@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) + * alloc-free cycle to prevent from reusing the page. + */ + #define PAGE_FLAGS_CHECK_AT_PREP \ +- (PAGEFLAGS_MASK & ~__PG_HWPOISON) ++ ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) + + #define PAGE_FLAGS_PRIVATE \ + (1UL << PG_private | 1UL << PG_private_2) +diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h +index 014ee8f0fbaa..d9095251bffd 100644 +--- a/include/linux/pgtable.h ++++ b/include/linux/pgtable.h +@@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + #endif + + #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +-#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) + static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +@@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + BUILD_BUG(); + return 0; + } +-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ + #endif + + #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +@@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, + #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + #endif + ++#ifndef arch_has_hw_pte_young ++/* ++ * Return whether the accessed bit is supported on the local CPU. ++ * ++ * This stub assumes accessing through an old PTE triggers a page fault. ++ * Architectures that automatically set the access bit should overwrite it. ++ */ ++static inline bool arch_has_hw_pte_young(void) ++{ ++ return false; ++} ++#endif ++ + #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR + static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, +diff --git a/include/linux/sched.h b/include/linux/sched.h +index e7b2f8a5c711..8cc46a789193 100644 +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -914,6 +914,10 @@ struct task_struct { + #ifdef CONFIG_MEMCG + unsigned in_user_fault:1; + #endif ++#ifdef CONFIG_LRU_GEN ++ /* whether the LRU algorithm may apply to this access */ ++ unsigned in_lru_fault:1; ++#endif + #ifdef CONFIG_COMPAT_BRK + unsigned brk_randomized:1; + #endif +diff --git a/include/linux/swap.h b/include/linux/swap.h +index 43150b9bbc5c..6308150b234a 100644 +--- a/include/linux/swap.h ++++ b/include/linux/swap.h +@@ -162,6 +162,10 @@ union swap_header { + */ + struct reclaim_state { + unsigned long reclaimed_slab; ++#ifdef CONFIG_LRU_GEN ++ /* per-thread mm walk data */ ++ struct lru_gen_mm_walk *mm_walk; ++#endif + }; + + #ifdef __KERNEL__ +diff --git a/kernel/bounds.c b/kernel/bounds.c +index 9795d75b09b2..b529182e8b04 100644 +--- a/kernel/bounds.c ++++ b/kernel/bounds.c +@@ -22,6 +22,13 @@ int main(void) + DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); + #endif + DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); ++#ifdef CONFIG_LRU_GEN ++ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); ++ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); ++#else ++ DEFINE(LRU_GEN_WIDTH, 0); ++ DEFINE(__LRU_REFS_WIDTH, 0); ++#endif + /* End of constants */ + + return 0; +diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h +index 36b740cb3d59..63dc3e82be4f 100644 +--- a/kernel/cgroup/cgroup-internal.h ++++ b/kernel/cgroup/cgroup-internal.h +@@ -164,7 +164,6 @@ struct cgroup_mgctx { + #define DEFINE_CGROUP_MGCTX(name) \ + struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) + +-extern struct mutex cgroup_mutex; + extern spinlock_t css_set_lock; + extern struct cgroup_subsys *cgroup_subsys[]; + extern struct list_head cgroup_roots; +diff --git a/kernel/exit.c b/kernel/exit.c +index 84021b24f79e..98a33bd7c25c 100644 +--- a/kernel/exit.c ++++ b/kernel/exit.c +@@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm) + goto retry; + } + WRITE_ONCE(mm->owner, c); ++ lru_gen_migrate_mm(mm); + task_unlock(c); + put_task_struct(c); + } +diff --git a/kernel/fork.c b/kernel/fork.c +index 704fe6bc9cb4..b15aaae04403 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -1156,6 +1156,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, + goto fail_nocontext; + + mm->user_ns = get_user_ns(user_ns); ++ lru_gen_init_mm(mm); + return mm; + + fail_nocontext: +@@ -1198,6 +1199,7 @@ static inline void __mmput(struct mm_struct *mm) + } + if (mm->binfmt) + module_put(mm->binfmt->module); ++ lru_gen_del_mm(mm); + mmdrop(mm); + } + +@@ -2700,6 +2702,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) + get_task_struct(p); + } + ++ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { ++ /* lock the task to synchronize with memcg migration */ ++ task_lock(p); ++ lru_gen_add_mm(p->mm); ++ task_unlock(p); ++ } ++ + wake_up_new_task(p); + + /* forking complete and child started to run, tell ptracer */ +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index 29f1fff78059..dcdbb8f8dc52 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -5178,6 +5178,7 @@ context_switch(struct rq *rq, struct task_struct *prev, + * finish_task_switch()'s mmdrop(). + */ + switch_mm_irqs_off(prev->active_mm, next->mm, next); ++ lru_gen_use_mm(next->mm); + + if (!prev->mm) { // from kernel + /* will mmdrop() in finish_task_switch(). */ +diff --git a/mm/Kconfig b/mm/Kconfig +index 0331f1461f81..96cd3ae25c6f 100644 +--- a/mm/Kconfig ++++ b/mm/Kconfig +@@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP + purposes. It is required to enable userfaultfd write protection on + file-backed memory types like shmem and hugetlbfs. + ++# multi-gen LRU { ++config LRU_GEN ++ bool "Multi-Gen LRU" ++ depends on MMU ++ # make sure folio->flags has enough spare bits ++ depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP ++ help ++ A high performance LRU implementation to overcommit memory. See ++ Documentation/admin-guide/mm/multigen_lru.rst for details. ++ ++config LRU_GEN_ENABLED ++ bool "Enable by default" ++ depends on LRU_GEN ++ help ++ This option enables the multi-gen LRU by default. ++ ++config LRU_GEN_STATS ++ bool "Full stats for debugging" ++ depends on LRU_GEN ++ help ++ Do not enable this option unless you plan to look at historical stats ++ from evicted generations for debugging purpose. ++ ++ This option has a per-memcg and per-node memory overhead. ++# } ++ + source "mm/damon/Kconfig" + + endmenu +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index f42bb51e023a..79e0b08b4cf9 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2438,7 +2438,8 @@ static void __split_huge_page_tail(struct page *head, int tail, + #ifdef CONFIG_64BIT + (1L << PG_arch_2) | + #endif +- (1L << PG_dirty))); ++ (1L << PG_dirty) | ++ LRU_GEN_MASK | LRU_REFS_MASK)); + + /* ->mapping in first tail page is compound_mapcount */ + VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, +diff --git a/mm/internal.h b/mm/internal.h +index 785409805ed7..a1fddea6b34f 100644 +--- a/mm/internal.h ++++ b/mm/internal.h +@@ -83,6 +83,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); + void folio_rotate_reclaimable(struct folio *folio); + bool __folio_end_writeback(struct folio *folio); + void deactivate_file_folio(struct folio *folio); ++void folio_activate(struct folio *folio); + + void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, + unsigned long floor, unsigned long ceiling); +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index b69979c9ced5..1c18d7c1ce71 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -2789,6 +2789,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) + * - LRU isolation + * - lock_page_memcg() + * - exclusive reference ++ * - mem_cgroup_trylock_pages() + */ + folio->memcg_data = (unsigned long)memcg; + } +@@ -5170,6 +5171,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) + + static void mem_cgroup_free(struct mem_cgroup *memcg) + { ++ lru_gen_exit_memcg(memcg); + memcg_wb_domain_exit(memcg); + __mem_cgroup_free(memcg); + } +@@ -5228,6 +5230,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) + memcg->deferred_split_queue.split_queue_len = 0; + #endif + idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); ++ lru_gen_init_memcg(memcg); + return memcg; + fail: + mem_cgroup_id_remove(memcg); +@@ -6196,6 +6199,30 @@ static void mem_cgroup_move_task(void) + } + #endif + ++#ifdef CONFIG_LRU_GEN ++static void mem_cgroup_attach(struct cgroup_taskset *tset) ++{ ++ struct task_struct *task; ++ struct cgroup_subsys_state *css; ++ ++ /* find the first leader if there is any */ ++ cgroup_taskset_for_each_leader(task, css, tset) ++ break; ++ ++ if (!task) ++ return; ++ ++ task_lock(task); ++ if (task->mm && READ_ONCE(task->mm->owner) == task) ++ lru_gen_migrate_mm(task->mm); ++ task_unlock(task); ++} ++#else ++static void mem_cgroup_attach(struct cgroup_taskset *tset) ++{ ++} ++#endif /* CONFIG_LRU_GEN */ ++ + static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) + { + if (value == PAGE_COUNTER_MAX) +@@ -6601,6 +6628,7 @@ struct cgroup_subsys memory_cgrp_subsys = { + .css_reset = mem_cgroup_css_reset, + .css_rstat_flush = mem_cgroup_css_rstat_flush, + .can_attach = mem_cgroup_can_attach, ++ .attach = mem_cgroup_attach, + .cancel_attach = mem_cgroup_cancel_attach, + .post_attach = mem_cgroup_move_task, + .dfl_cftypes = memory_files, +diff --git a/mm/memory.c b/mm/memory.c +index a78814413ac0..cd1b5bfd9f3e 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -125,18 +125,6 @@ int randomize_va_space __read_mostly = + 2; + #endif + +-#ifndef arch_faults_on_old_pte +-static inline bool arch_faults_on_old_pte(void) +-{ +- /* +- * Those arches which don't have hw access flag feature need to +- * implement their own helper. By default, "true" means pagefault +- * will be hit on old pte. +- */ +- return true; +-} +-#endif +- + #ifndef arch_wants_old_prefaulted_pte + static inline bool arch_wants_old_prefaulted_pte(void) + { +@@ -2870,7 +2858,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, + * On architectures with software "accessed" bits, we would + * take a double page fault, so mark it accessed here. + */ +- if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { ++ if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { + pte_t entry; + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); +@@ -5120,6 +5108,27 @@ static inline void mm_account_fault(struct pt_regs *regs, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); + } + ++#ifdef CONFIG_LRU_GEN ++static void lru_gen_enter_fault(struct vm_area_struct *vma) ++{ ++ /* the LRU algorithm doesn't apply to sequential or random reads */ ++ current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); ++} ++ ++static void lru_gen_exit_fault(void) ++{ ++ current->in_lru_fault = false; ++} ++#else ++static void lru_gen_enter_fault(struct vm_area_struct *vma) ++{ ++} ++ ++static void lru_gen_exit_fault(void) ++{ ++} ++#endif /* CONFIG_LRU_GEN */ ++ + /* + * By the time we get here, we already hold the mm semaphore + * +@@ -5151,11 +5160,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + if (flags & FAULT_FLAG_USER) + mem_cgroup_enter_user_fault(); + ++ lru_gen_enter_fault(vma); ++ + if (unlikely(is_vm_hugetlb_page(vma))) + ret = hugetlb_fault(vma->vm_mm, vma, address, flags); + else + ret = __handle_mm_fault(vma, address, flags); + ++ lru_gen_exit_fault(); ++ + if (flags & FAULT_FLAG_USER) { + mem_cgroup_exit_user_fault(); + /* +diff --git a/mm/mm_init.c b/mm/mm_init.c +index 9ddaf0e1b0ab..0d7b2bd2454a 100644 +--- a/mm/mm_init.c ++++ b/mm/mm_init.c +@@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) + + shift = 8 * sizeof(unsigned long); + width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH +- - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; ++ - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; + mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", +- "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", ++ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", + SECTIONS_WIDTH, + NODES_WIDTH, + ZONES_WIDTH, + LAST_CPUPID_WIDTH, + KASAN_TAG_WIDTH, ++ LRU_GEN_WIDTH, ++ LRU_REFS_WIDTH, + NR_PAGEFLAGS); + mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", +diff --git a/mm/mmzone.c b/mm/mmzone.c +index 0ae7571e35ab..68e1511be12d 100644 +--- a/mm/mmzone.c ++++ b/mm/mmzone.c +@@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) + * Poison its list head, so that any operations on it would crash. + */ + list_del(&lruvec->lists[LRU_UNEVICTABLE]); ++ ++ lru_gen_init_lruvec(lruvec); + } + + #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) +diff --git a/mm/rmap.c b/mm/rmap.c +index 93d5a6f793d2..9e0ce48bca08 100644 +--- a/mm/rmap.c ++++ b/mm/rmap.c +@@ -833,6 +833,12 @@ static bool folio_referenced_one(struct folio *folio, + } + + if (pvmw.pte) { ++ if (lru_gen_enabled() && pte_young(*pvmw.pte) && ++ !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { ++ lru_gen_look_around(&pvmw); ++ referenced++; ++ } ++ + if (ptep_clear_flush_young_notify(vma, address, + pvmw.pte)) { + /* +diff --git a/mm/swap.c b/mm/swap.c +index 186b4e5dcecf..027943b341a0 100644 +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu) + folio_batch_move_lru(fbatch, folio_activate_fn); + } + +-static void folio_activate(struct folio *folio) ++void folio_activate(struct folio *folio) + { + if (folio_test_lru(folio) && !folio_test_active(folio) && + !folio_test_unevictable(folio)) { +@@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu) + { + } + +-static void folio_activate(struct folio *folio) ++void folio_activate(struct folio *folio) + { + struct lruvec *lruvec; + +@@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio) + local_unlock(&cpu_fbatches.lock); + } + ++#ifdef CONFIG_LRU_GEN ++static void folio_inc_refs(struct folio *folio) ++{ ++ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); ++ ++ if (folio_test_unevictable(folio)) ++ return; ++ ++ if (!folio_test_referenced(folio)) { ++ folio_set_referenced(folio); ++ return; ++ } ++ ++ if (!folio_test_workingset(folio)) { ++ folio_set_workingset(folio); ++ return; ++ } ++ ++ /* see the comment on MAX_NR_TIERS */ ++ do { ++ new_flags = old_flags & LRU_REFS_MASK; ++ if (new_flags == LRU_REFS_MASK) ++ break; ++ ++ new_flags += BIT(LRU_REFS_PGOFF); ++ new_flags |= old_flags & ~LRU_REFS_MASK; ++ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); ++} ++#else ++static void folio_inc_refs(struct folio *folio) ++{ ++} ++#endif /* CONFIG_LRU_GEN */ ++ + /* + * Mark a page as having seen activity. + * +@@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio) + */ + void folio_mark_accessed(struct folio *folio) + { ++ if (lru_gen_enabled()) { ++ folio_inc_refs(folio); ++ return; ++ } ++ + if (!folio_test_referenced(folio)) { + folio_set_referenced(folio); + } else if (folio_test_unevictable(folio)) { +@@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio) + folio_test_unevictable(folio), folio); + VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + ++ /* see the comment in lru_gen_add_folio() */ ++ if (lru_gen_enabled() && !folio_test_unevictable(folio) && ++ lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) ++ folio_set_active(folio); ++ + folio_get(folio); + local_lock(&cpu_fbatches.lock); + fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); +@@ -575,7 +619,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) + + static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) + { +- if (folio_test_active(folio) && !folio_test_unevictable(folio)) { ++ if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { + long nr_pages = folio_nr_pages(folio); + + lruvec_del_folio(lruvec, folio); +@@ -688,8 +732,8 @@ void deactivate_page(struct page *page) + { + struct folio *folio = page_folio(page); + +- if (folio_test_lru(folio) && folio_test_active(folio) && +- !folio_test_unevictable(folio)) { ++ if (folio_test_lru(folio) && !folio_test_unevictable(folio) && ++ (folio_test_active(folio) || lru_gen_enabled())) { + struct folio_batch *fbatch; + + folio_get(folio); +diff --git a/mm/vmscan.c b/mm/vmscan.c +index fbc8c8f4fe60..bedf4ae55c31 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -49,6 +49,10 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + + #include + #include +@@ -129,6 +133,12 @@ struct scan_control { + /* Always discard instead of demoting to lower tier memory */ + unsigned int no_demotion:1; + ++#ifdef CONFIG_LRU_GEN ++ /* help kswapd make better choices among multiple memcgs */ ++ unsigned int memcgs_need_aging:1; ++ unsigned long last_reclaimed; ++#endif ++ + /* Allocation order */ + s8 order; + +@@ -1338,9 +1348,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, + + if (folio_test_swapcache(folio)) { + swp_entry_t swap = folio_swap_entry(folio); +- mem_cgroup_swapout(folio, swap); ++ ++ /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ + if (reclaimed && !mapping_exiting(mapping)) + shadow = workingset_eviction(folio, target_memcg); ++ mem_cgroup_swapout(folio, swap); + __delete_from_swap_cache(folio, swap, shadow); + xa_unlock_irq(&mapping->i_pages); + put_swap_page(&folio->page, swap); +@@ -1637,6 +1649,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, + if (!sc->may_unmap && folio_mapped(folio)) + goto keep_locked; + ++ /* folio_update_gen() tried to promote this page? */ ++ if (lru_gen_enabled() && !ignore_references && ++ folio_mapped(folio) && folio_test_referenced(folio)) ++ goto keep_locked; ++ + /* + * The number of dirty pages determines if a node is marked + * reclaim_congested. kswapd will stall and start writing +@@ -2732,6 +2749,112 @@ enum scan_balance { + SCAN_FILE, + }; + ++static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) ++{ ++ unsigned long file; ++ struct lruvec *target_lruvec; ++ ++ if (lru_gen_enabled()) ++ return; ++ ++ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); ++ ++ /* ++ * Flush the memory cgroup stats, so that we read accurate per-memcg ++ * lruvec stats for heuristics. ++ */ ++ mem_cgroup_flush_stats(); ++ ++ /* ++ * Determine the scan balance between anon and file LRUs. ++ */ ++ spin_lock_irq(&target_lruvec->lru_lock); ++ sc->anon_cost = target_lruvec->anon_cost; ++ sc->file_cost = target_lruvec->file_cost; ++ spin_unlock_irq(&target_lruvec->lru_lock); ++ ++ /* ++ * Target desirable inactive:active list ratios for the anon ++ * and file LRU lists. ++ */ ++ if (!sc->force_deactivate) { ++ unsigned long refaults; ++ ++ /* ++ * When refaults are being observed, it means a new ++ * workingset is being established. Deactivate to get ++ * rid of any stale active pages quickly. ++ */ ++ refaults = lruvec_page_state(target_lruvec, ++ WORKINGSET_ACTIVATE_ANON); ++ if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || ++ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) ++ sc->may_deactivate |= DEACTIVATE_ANON; ++ else ++ sc->may_deactivate &= ~DEACTIVATE_ANON; ++ ++ refaults = lruvec_page_state(target_lruvec, ++ WORKINGSET_ACTIVATE_FILE); ++ if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || ++ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) ++ sc->may_deactivate |= DEACTIVATE_FILE; ++ else ++ sc->may_deactivate &= ~DEACTIVATE_FILE; ++ } else ++ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; ++ ++ /* ++ * If we have plenty of inactive file pages that aren't ++ * thrashing, try to reclaim those first before touching ++ * anonymous pages. ++ */ ++ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); ++ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) ++ sc->cache_trim_mode = 1; ++ else ++ sc->cache_trim_mode = 0; ++ ++ /* ++ * Prevent the reclaimer from falling into the cache trap: as ++ * cache pages start out inactive, every cache fault will tip ++ * the scan balance towards the file LRU. And as the file LRU ++ * shrinks, so does the window for rotation from references. ++ * This means we have a runaway feedback loop where a tiny ++ * thrashing file LRU becomes infinitely more attractive than ++ * anon pages. Try to detect this based on file LRU size. ++ */ ++ if (!cgroup_reclaim(sc)) { ++ unsigned long total_high_wmark = 0; ++ unsigned long free, anon; ++ int z; ++ ++ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); ++ file = node_page_state(pgdat, NR_ACTIVE_FILE) + ++ node_page_state(pgdat, NR_INACTIVE_FILE); ++ ++ for (z = 0; z < MAX_NR_ZONES; z++) { ++ struct zone *zone = &pgdat->node_zones[z]; ++ ++ if (!managed_zone(zone)) ++ continue; ++ ++ total_high_wmark += high_wmark_pages(zone); ++ } ++ ++ /* ++ * Consider anon: if that's low too, this isn't a ++ * runaway file reclaim problem, but rather just ++ * extreme pressure. Reclaim as per usual then. ++ */ ++ anon = node_page_state(pgdat, NR_INACTIVE_ANON); ++ ++ sc->file_is_tiny = ++ file + free <= total_high_wmark && ++ !(sc->may_deactivate & DEACTIVATE_ANON) && ++ anon >> sc->priority; ++ } ++} ++ + /* + * Determine how aggressively the anon and file LRU lists should be + * scanned. +@@ -2951,159 +3074,2909 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, + return can_demote(pgdat->node_id, sc); + } + +-static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +-{ +- unsigned long nr[NR_LRU_LISTS]; +- unsigned long targets[NR_LRU_LISTS]; +- unsigned long nr_to_scan; +- enum lru_list lru; +- unsigned long nr_reclaimed = 0; +- unsigned long nr_to_reclaim = sc->nr_to_reclaim; +- struct blk_plug plug; +- bool scan_adjusted; ++#ifdef CONFIG_LRU_GEN + +- get_scan_count(lruvec, sc, nr); ++#ifdef CONFIG_LRU_GEN_ENABLED ++DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); ++#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) ++#else ++DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); ++#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) ++#endif + +- /* Record the original scan target for proportional adjustments later */ +- memcpy(targets, nr, sizeof(nr)); ++/****************************************************************************** ++ * shorthand helpers ++ ******************************************************************************/ + +- /* +- * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal +- * event that can occur when there is little memory pressure e.g. +- * multiple streaming readers/writers. Hence, we do not abort scanning +- * when the requested number of pages are reclaimed when scanning at +- * DEF_PRIORITY on the assumption that the fact we are direct +- * reclaiming implies that kswapd is not keeping up and it is best to +- * do a batch of work at once. For memcg reclaim one check is made to +- * abort proportional reclaim if either the file or anon lru has already +- * dropped to zero at the first pass. +- */ +- scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && +- sc->priority == DEF_PRIORITY); ++#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) + +- blk_start_plug(&plug); +- while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || +- nr[LRU_INACTIVE_FILE]) { +- unsigned long nr_anon, nr_file, percentage; +- unsigned long nr_scanned; ++#define DEFINE_MAX_SEQ(lruvec) \ ++ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) + +- for_each_evictable_lru(lru) { +- if (nr[lru]) { +- nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); +- nr[lru] -= nr_to_scan; ++#define DEFINE_MIN_SEQ(lruvec) \ ++ unsigned long min_seq[ANON_AND_FILE] = { \ ++ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ ++ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ ++ } + +- nr_reclaimed += shrink_list(lru, nr_to_scan, +- lruvec, sc); +- } +- } ++#define for_each_gen_type_zone(gen, type, zone) \ ++ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ ++ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ ++ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) + +- cond_resched(); ++static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) ++{ ++ struct pglist_data *pgdat = NODE_DATA(nid); + +- if (nr_reclaimed < nr_to_reclaim || scan_adjusted) +- continue; ++#ifdef CONFIG_MEMCG ++ if (memcg) { ++ struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; + +- /* +- * For kswapd and memcg, reclaim at least the number of pages +- * requested. Ensure that the anon and file LRUs are scanned +- * proportionally what was requested by get_scan_count(). We +- * stop reclaiming one LRU and reduce the amount scanning +- * proportional to the original scan target. +- */ +- nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; +- nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; ++ /* for hotadd_new_pgdat() */ ++ if (!lruvec->pgdat) ++ lruvec->pgdat = pgdat; + +- /* +- * It's just vindictive to attack the larger once the smaller +- * has gone to zero. And given the way we stop scanning the +- * smaller below, this makes sure that we only make one nudge +- * towards proportionality once we've got nr_to_reclaim. +- */ +- if (!nr_file || !nr_anon) +- break; ++ return lruvec; ++ } ++#endif ++ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); + +- if (nr_file > nr_anon) { +- unsigned long scan_target = targets[LRU_INACTIVE_ANON] + +- targets[LRU_ACTIVE_ANON] + 1; +- lru = LRU_BASE; +- percentage = nr_anon * 100 / scan_target; +- } else { +- unsigned long scan_target = targets[LRU_INACTIVE_FILE] + +- targets[LRU_ACTIVE_FILE] + 1; +- lru = LRU_FILE; +- percentage = nr_file * 100 / scan_target; +- } ++ return pgdat ? &pgdat->__lruvec : NULL; ++} + +- /* Stop scanning the smaller of the LRU */ +- nr[lru] = 0; +- nr[lru + LRU_ACTIVE] = 0; ++static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) ++{ ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); + +- /* +- * Recalculate the other LRU scan count based on its original +- * scan target and the percentage scanning already complete +- */ +- lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; +- nr_scanned = targets[lru] - nr[lru]; +- nr[lru] = targets[lru] * (100 - percentage) / 100; +- nr[lru] -= min(nr[lru], nr_scanned); ++ if (!can_demote(pgdat->node_id, sc) && ++ mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) ++ return 0; + +- lru += LRU_ACTIVE; +- nr_scanned = targets[lru] - nr[lru]; +- nr[lru] = targets[lru] * (100 - percentage) / 100; +- nr[lru] -= min(nr[lru], nr_scanned); ++ return mem_cgroup_swappiness(memcg); ++} + +- scan_adjusted = true; +- } +- blk_finish_plug(&plug); +- sc->nr_reclaimed += nr_reclaimed; ++static int get_nr_gens(struct lruvec *lruvec, int type) ++{ ++ return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; ++} + +- /* +- * Even if we did not try to evict anon pages at all, we want to +- * rebalance the anon lru active/inactive ratio. +- */ +- if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && +- inactive_is_low(lruvec, LRU_INACTIVE_ANON)) +- shrink_active_list(SWAP_CLUSTER_MAX, lruvec, +- sc, LRU_ACTIVE_ANON); ++static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) ++{ ++ /* see the comment on lru_gen_struct */ ++ return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && ++ get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && ++ get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; + } + +-/* Use reclaim/compaction for costly allocs or under memory pressure */ +-static bool in_reclaim_compaction(struct scan_control *sc) ++/****************************************************************************** ++ * mm_struct list ++ ******************************************************************************/ ++ ++static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) + { +- if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && +- (sc->order > PAGE_ALLOC_COSTLY_ORDER || +- sc->priority < DEF_PRIORITY - 2)) +- return true; ++ static struct lru_gen_mm_list mm_list = { ++ .fifo = LIST_HEAD_INIT(mm_list.fifo), ++ .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), ++ }; + +- return false; ++#ifdef CONFIG_MEMCG ++ if (memcg) ++ return &memcg->mm_list; ++#endif ++ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); ++ ++ return &mm_list; + } + +-/* +- * Reclaim/compaction is used for high-order allocation requests. It reclaims +- * order-0 pages before compacting the zone. should_continue_reclaim() returns +- * true if more pages should be reclaimed such that when the page allocator +- * calls try_to_compact_pages() that it will have enough free pages to succeed. +- * It will give up earlier than that if there is difficulty reclaiming pages. +- */ +-static inline bool should_continue_reclaim(struct pglist_data *pgdat, +- unsigned long nr_reclaimed, +- struct scan_control *sc) ++void lru_gen_add_mm(struct mm_struct *mm) + { +- unsigned long pages_for_compaction; +- unsigned long inactive_lru_pages; +- int z; ++ int nid; ++ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + +- /* If not in reclaim/compaction mode, stop */ +- if (!in_reclaim_compaction(sc)) +- return false; ++ VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); ++#ifdef CONFIG_MEMCG ++ VM_WARN_ON_ONCE(mm->lru_gen.memcg); ++ mm->lru_gen.memcg = memcg; ++#endif ++ spin_lock(&mm_list->lock); + +- /* +- * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX +- * number of pages that were scanned. This will return to the caller +- * with the risk reclaim/compaction and the resulting allocation attempt +- * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL +- * allocations through requiring that the full LRU list has been scanned +- * first, by assuming that zero delta of sc->nr_scanned means full LRU +- * scan, but that approximation was wrong, and there were corner cases ++ for_each_node_state(nid, N_MEMORY) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ ++ if (!lruvec) ++ continue; ++ ++ /* the first addition since the last iteration */ ++ if (lruvec->mm_state.tail == &mm_list->fifo) ++ lruvec->mm_state.tail = &mm->lru_gen.list; ++ } ++ ++ list_add_tail(&mm->lru_gen.list, &mm_list->fifo); ++ ++ spin_unlock(&mm_list->lock); ++} ++ ++void lru_gen_del_mm(struct mm_struct *mm) ++{ ++ int nid; ++ struct lru_gen_mm_list *mm_list; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (list_empty(&mm->lru_gen.list)) ++ return; ++ ++#ifdef CONFIG_MEMCG ++ memcg = mm->lru_gen.memcg; ++#endif ++ mm_list = get_mm_list(memcg); ++ ++ spin_lock(&mm_list->lock); ++ ++ for_each_node(nid) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ ++ if (!lruvec) ++ continue; ++ ++ /* where the last iteration ended (exclusive) */ ++ if (lruvec->mm_state.tail == &mm->lru_gen.list) ++ lruvec->mm_state.tail = lruvec->mm_state.tail->next; ++ ++ /* where the current iteration continues (inclusive) */ ++ if (lruvec->mm_state.head != &mm->lru_gen.list) ++ continue; ++ ++ lruvec->mm_state.head = lruvec->mm_state.head->next; ++ /* the deletion ends the current iteration */ ++ if (lruvec->mm_state.head == &mm_list->fifo) ++ WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); ++ } ++ ++ list_del_init(&mm->lru_gen.list); ++ ++ spin_unlock(&mm_list->lock); ++ ++#ifdef CONFIG_MEMCG ++ mem_cgroup_put(mm->lru_gen.memcg); ++ mm->lru_gen.memcg = NULL; ++#endif ++} ++ ++#ifdef CONFIG_MEMCG ++void lru_gen_migrate_mm(struct mm_struct *mm) ++{ ++ struct mem_cgroup *memcg; ++ struct task_struct *task = rcu_dereference_protected(mm->owner, true); ++ ++ VM_WARN_ON_ONCE(task->mm != mm); ++ lockdep_assert_held(&task->alloc_lock); ++ ++ /* for mm_update_next_owner() */ ++ if (mem_cgroup_disabled()) ++ return; ++ ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_task(task); ++ rcu_read_unlock(); ++ if (memcg == mm->lru_gen.memcg) ++ return; ++ ++ VM_WARN_ON_ONCE(!mm->lru_gen.memcg); ++ VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); ++ ++ lru_gen_del_mm(mm); ++ lru_gen_add_mm(mm); ++} ++#endif ++ ++/* ++ * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when ++ * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of ++ * bits in a bitmap, k is the number of hash functions and n is the number of ++ * inserted items. ++ * ++ * Page table walkers use one of the two filters to reduce their search space. ++ * To get rid of non-leaf entries that no longer have enough leaf entries, the ++ * aging uses the double-buffering technique to flip to the other filter each ++ * time it produces a new generation. For non-leaf entries that have enough ++ * leaf entries, the aging carries them over to the next generation in ++ * walk_pmd_range(); the eviction also report them when walking the rmap ++ * in lru_gen_look_around(). ++ * ++ * For future optimizations: ++ * 1. It's not necessary to keep both filters all the time. The spare one can be ++ * freed after the RCU grace period and reallocated if needed again. ++ * 2. And when reallocating, it's worth scaling its size according to the number ++ * of inserted entries in the other filter, to reduce the memory overhead on ++ * small systems and false positives on large systems. ++ * 3. Jenkins' hash function is an alternative to Knuth's. ++ */ ++#define BLOOM_FILTER_SHIFT 15 ++ ++static inline int filter_gen_from_seq(unsigned long seq) ++{ ++ return seq % NR_BLOOM_FILTERS; ++} ++ ++static void get_item_key(void *item, int *key) ++{ ++ u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); ++ ++ BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); ++ ++ key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); ++ key[1] = hash >> BLOOM_FILTER_SHIFT; ++} ++ ++static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) ++{ ++ unsigned long *filter; ++ int gen = filter_gen_from_seq(seq); ++ ++ filter = lruvec->mm_state.filters[gen]; ++ if (filter) { ++ bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); ++ return; ++ } ++ ++ filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), ++ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); ++ WRITE_ONCE(lruvec->mm_state.filters[gen], filter); ++} ++ ++static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) ++{ ++ int key[2]; ++ unsigned long *filter; ++ int gen = filter_gen_from_seq(seq); ++ ++ filter = READ_ONCE(lruvec->mm_state.filters[gen]); ++ if (!filter) ++ return; ++ ++ get_item_key(item, key); ++ ++ if (!test_bit(key[0], filter)) ++ set_bit(key[0], filter); ++ if (!test_bit(key[1], filter)) ++ set_bit(key[1], filter); ++} ++ ++static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) ++{ ++ int key[2]; ++ unsigned long *filter; ++ int gen = filter_gen_from_seq(seq); ++ ++ filter = READ_ONCE(lruvec->mm_state.filters[gen]); ++ if (!filter) ++ return true; ++ ++ get_item_key(item, key); ++ ++ return test_bit(key[0], filter) && test_bit(key[1], filter); ++} ++ ++static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) ++{ ++ int i; ++ int hist; ++ ++ lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); ++ ++ if (walk) { ++ hist = lru_hist_from_seq(walk->max_seq); ++ ++ for (i = 0; i < NR_MM_STATS; i++) { ++ WRITE_ONCE(lruvec->mm_state.stats[hist][i], ++ lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); ++ walk->mm_stats[i] = 0; ++ } ++ } ++ ++ if (NR_HIST_GENS > 1 && last) { ++ hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); ++ ++ for (i = 0; i < NR_MM_STATS; i++) ++ WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); ++ } ++} ++ ++static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) ++{ ++ int type; ++ unsigned long size = 0; ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); ++ ++ if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) ++ return true; ++ ++ clear_bit(key, &mm->lru_gen.bitmap); ++ ++ for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { ++ size += type ? get_mm_counter(mm, MM_FILEPAGES) : ++ get_mm_counter(mm, MM_ANONPAGES) + ++ get_mm_counter(mm, MM_SHMEMPAGES); ++ } ++ ++ if (size < MIN_LRU_BATCH) ++ return true; ++ ++ return !mmget_not_zero(mm); ++} ++ ++static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, ++ struct mm_struct **iter) ++{ ++ bool first = false; ++ bool last = true; ++ struct mm_struct *mm = NULL; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; ++ ++ /* ++ * There are four interesting cases for this page table walker: ++ * 1. It tries to start a new iteration of mm_list with a stale max_seq; ++ * there is nothing left to do. ++ * 2. It's the first of the current generation, and it needs to reset ++ * the Bloom filter for the next generation. ++ * 3. It reaches the end of mm_list, and it needs to increment ++ * mm_state->seq; the iteration is done. ++ * 4. It's the last of the current generation, and it needs to reset the ++ * mm stats counters for the next generation. ++ */ ++ spin_lock(&mm_list->lock); ++ ++ VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); ++ VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); ++ VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); ++ ++ if (walk->max_seq <= mm_state->seq) { ++ if (!*iter) ++ last = false; ++ goto done; ++ } ++ ++ if (!mm_state->nr_walkers) { ++ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); ++ ++ mm_state->head = mm_list->fifo.next; ++ first = true; ++ } ++ ++ while (!mm && mm_state->head != &mm_list->fifo) { ++ mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); ++ ++ mm_state->head = mm_state->head->next; ++ ++ /* force scan for those added after the last iteration */ ++ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { ++ mm_state->tail = mm_state->head; ++ walk->force_scan = true; ++ } ++ ++ if (should_skip_mm(mm, walk)) ++ mm = NULL; ++ } ++ ++ if (mm_state->head == &mm_list->fifo) ++ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); ++done: ++ if (*iter && !mm) ++ mm_state->nr_walkers--; ++ if (!*iter && mm) ++ mm_state->nr_walkers++; ++ ++ if (mm_state->nr_walkers) ++ last = false; ++ ++ if (*iter || last) ++ reset_mm_stats(lruvec, walk, last); ++ ++ spin_unlock(&mm_list->lock); ++ ++ if (mm && first) ++ reset_bloom_filter(lruvec, walk->max_seq + 1); ++ ++ if (*iter) ++ mmput_async(*iter); ++ ++ *iter = mm; ++ ++ return last; ++} ++ ++static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) ++{ ++ bool success = false; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; ++ ++ spin_lock(&mm_list->lock); ++ ++ VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); ++ ++ if (max_seq > mm_state->seq && !mm_state->nr_walkers) { ++ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); ++ ++ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); ++ reset_mm_stats(lruvec, NULL, true); ++ success = true; ++ } ++ ++ spin_unlock(&mm_list->lock); ++ ++ return success; ++} ++ ++/****************************************************************************** ++ * refault feedback loop ++ ******************************************************************************/ ++ ++/* ++ * A feedback loop based on Proportional-Integral-Derivative (PID) controller. ++ * ++ * The P term is refaulted/(evicted+protected) from a tier in the generation ++ * currently being evicted; the I term is the exponential moving average of the ++ * P term over the generations previously evicted, using the smoothing factor ++ * 1/2; the D term isn't supported. ++ * ++ * The setpoint (SP) is always the first tier of one type; the process variable ++ * (PV) is either any tier of the other type or any other tier of the same ++ * type. ++ * ++ * The error is the difference between the SP and the PV; the correction is to ++ * turn off protection when SP>PV or turn on protection when SPlrugen; ++ int hist = lru_hist_from_seq(lrugen->min_seq[type]); ++ ++ pos->refaulted = lrugen->avg_refaulted[type][tier] + ++ atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ pos->total = lrugen->avg_total[type][tier] + ++ atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ pos->total += lrugen->protected[hist][type][tier - 1]; ++ pos->gain = gain; ++} ++ ++static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) ++{ ++ int hist, tier; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; ++ unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; ++ ++ lockdep_assert_held(&lruvec->lru_lock); ++ ++ if (!carryover && !clear) ++ return; ++ ++ hist = lru_hist_from_seq(seq); ++ ++ for (tier = 0; tier < MAX_NR_TIERS; tier++) { ++ if (carryover) { ++ unsigned long sum; ++ ++ sum = lrugen->avg_refaulted[type][tier] + ++ atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); ++ ++ sum = lrugen->avg_total[type][tier] + ++ atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ sum += lrugen->protected[hist][type][tier - 1]; ++ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); ++ } ++ ++ if (clear) { ++ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); ++ atomic_long_set(&lrugen->evicted[hist][type][tier], 0); ++ if (tier) ++ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); ++ } ++ } ++} ++ ++static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ++{ ++ /* ++ * Return true if the PV has a limited number of refaults or a lower ++ * refaulted/total than the SP. ++ */ ++ return pv->refaulted < MIN_LRU_BATCH || ++ pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= ++ (sp->refaulted + 1) * pv->total * pv->gain; ++} ++ ++/****************************************************************************** ++ * the aging ++ ******************************************************************************/ ++ ++/* promote pages accessed through page tables */ ++static int folio_update_gen(struct folio *folio, int gen) ++{ ++ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); ++ ++ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ do { ++ /* lru_gen_del_folio() has isolated this page? */ ++ if (!(old_flags & LRU_GEN_MASK)) { ++ /* for shrink_page_list() */ ++ new_flags = old_flags | BIT(PG_referenced); ++ continue; ++ } ++ ++ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); ++ new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; ++ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); ++ ++ return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++} ++ ++/* protect pages accessed multiple times through file descriptors */ ++static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ int type = folio_is_file_lru(folio); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); ++ ++ VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); ++ ++ do { ++ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ /* folio_update_gen() has promoted this page? */ ++ if (new_gen >= 0 && new_gen != old_gen) ++ return new_gen; ++ ++ new_gen = (old_gen + 1) % MAX_NR_GENS; ++ ++ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); ++ new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; ++ /* for folio_end_writeback() */ ++ if (reclaiming) ++ new_flags |= BIT(PG_reclaim); ++ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); ++ ++ lru_gen_update_size(lruvec, folio, old_gen, new_gen); ++ ++ return new_gen; ++} ++ ++static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, ++ int old_gen, int new_gen) ++{ ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ int delta = folio_nr_pages(folio); ++ ++ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); ++ ++ walk->batched++; ++ ++ walk->nr_pages[old_gen][type][zone] -= delta; ++ walk->nr_pages[new_gen][type][zone] += delta; ++} ++ ++static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) ++{ ++ int gen, type, zone; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ walk->batched = 0; ++ ++ for_each_gen_type_zone(gen, type, zone) { ++ enum lru_list lru = type * LRU_INACTIVE_FILE; ++ int delta = walk->nr_pages[gen][type][zone]; ++ ++ if (!delta) ++ continue; ++ ++ walk->nr_pages[gen][type][zone] = 0; ++ WRITE_ONCE(lrugen->nr_pages[gen][type][zone], ++ lrugen->nr_pages[gen][type][zone] + delta); ++ ++ if (lru_gen_is_active(lruvec, gen)) ++ lru += LRU_ACTIVE; ++ __update_lru_size(lruvec, lru, zone, delta); ++ } ++} ++ ++static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) ++{ ++ struct address_space *mapping; ++ struct vm_area_struct *vma = args->vma; ++ struct lru_gen_mm_walk *walk = args->private; ++ ++ if (!vma_is_accessible(vma)) ++ return true; ++ ++ if (is_vm_hugetlb_page(vma)) ++ return true; ++ ++ if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) ++ return true; ++ ++ if (vma == get_gate_vma(vma->vm_mm)) ++ return true; ++ ++ if (vma_is_anonymous(vma)) ++ return !walk->can_swap; ++ ++ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) ++ return true; ++ ++ mapping = vma->vm_file->f_mapping; ++ if (mapping_unevictable(mapping)) ++ return true; ++ ++ if (shmem_mapping(mapping)) ++ return !walk->can_swap; ++ ++ /* to exclude special mappings like dax, etc. */ ++ return !mapping->a_ops->read_folio; ++} ++ ++/* ++ * Some userspace memory allocators map many single-page VMAs. Instead of ++ * returning back to the PGD table for each of such VMAs, finish an entire PMD ++ * table to reduce zigzags and improve cache performance. ++ */ ++static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, ++ unsigned long *vm_start, unsigned long *vm_end) ++{ ++ unsigned long start = round_up(*vm_end, size); ++ unsigned long end = (start | ~mask) + 1; ++ ++ VM_WARN_ON_ONCE(mask & size); ++ VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); ++ ++ while (args->vma) { ++ if (start >= args->vma->vm_end) { ++ args->vma = args->vma->vm_next; ++ continue; ++ } ++ ++ if (end && end <= args->vma->vm_start) ++ return false; ++ ++ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { ++ args->vma = args->vma->vm_next; ++ continue; ++ } ++ ++ *vm_start = max(start, args->vma->vm_start); ++ *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) ++{ ++ unsigned long pfn = pte_pfn(pte); ++ ++ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); ++ ++ if (!pte_present(pte) || is_zero_pfn(pfn)) ++ return -1; ++ ++ if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) ++ return -1; ++ ++ if (WARN_ON_ONCE(!pfn_valid(pfn))) ++ return -1; ++ ++ return pfn; ++} ++ ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) ++static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) ++{ ++ unsigned long pfn = pmd_pfn(pmd); ++ ++ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); ++ ++ if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) ++ return -1; ++ ++ if (WARN_ON_ONCE(pmd_devmap(pmd))) ++ return -1; ++ ++ if (WARN_ON_ONCE(!pfn_valid(pfn))) ++ return -1; ++ ++ return pfn; ++} ++#endif ++ ++static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, ++ struct pglist_data *pgdat, bool can_swap) ++{ ++ struct folio *folio; ++ ++ /* try to avoid unnecessary memory loads */ ++ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) ++ return NULL; ++ ++ folio = pfn_folio(pfn); ++ if (folio_nid(folio) != pgdat->node_id) ++ return NULL; ++ ++ if (folio_memcg_rcu(folio) != memcg) ++ return NULL; ++ ++ /* file VMAs can contain anon pages from COW */ ++ if (!folio_is_file_lru(folio) && !can_swap) ++ return NULL; ++ ++ return folio; ++} ++ ++static bool suitable_to_scan(int total, int young) ++{ ++ int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); ++ ++ /* suitable if the average number of young PTEs per cacheline is >=1 */ ++ return young * n >= total; ++} ++ ++static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, ++ struct mm_walk *args) ++{ ++ int i; ++ pte_t *pte; ++ spinlock_t *ptl; ++ unsigned long addr; ++ int total = 0; ++ int young = 0; ++ struct lru_gen_mm_walk *walk = args->private; ++ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); ++ ++ VM_WARN_ON_ONCE(pmd_leaf(*pmd)); ++ ++ ptl = pte_lockptr(args->mm, pmd); ++ if (!spin_trylock(ptl)) ++ return false; ++ ++ arch_enter_lazy_mmu_mode(); ++ ++ pte = pte_offset_map(pmd, start & PMD_MASK); ++restart: ++ for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { ++ unsigned long pfn; ++ struct folio *folio; ++ ++ total++; ++ walk->mm_stats[MM_LEAF_TOTAL]++; ++ ++ pfn = get_pte_pfn(pte[i], args->vma, addr); ++ if (pfn == -1) ++ continue; ++ ++ if (!pte_young(pte[i])) { ++ walk->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } ++ ++ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); ++ if (!folio) ++ continue; ++ ++ if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) ++ VM_WARN_ON_ONCE(true); ++ ++ young++; ++ walk->mm_stats[MM_LEAF_YOUNG]++; ++ ++ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && ++ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && ++ !folio_test_swapcache(folio))) ++ folio_mark_dirty(folio); ++ ++ old_gen = folio_update_gen(folio, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ update_batch_size(walk, folio, old_gen, new_gen); ++ } ++ ++ if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) ++ goto restart; ++ ++ pte_unmap(pte); ++ ++ arch_leave_lazy_mmu_mode(); ++ spin_unlock(ptl); ++ ++ return suitable_to_scan(total, young); ++} ++ ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) ++static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, ++ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) ++{ ++ int i; ++ pmd_t *pmd; ++ spinlock_t *ptl; ++ struct lru_gen_mm_walk *walk = args->private; ++ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); ++ ++ VM_WARN_ON_ONCE(pud_leaf(*pud)); ++ ++ /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ ++ if (*start == -1) { ++ *start = next; ++ return; ++ } ++ ++ i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); ++ if (i && i <= MIN_LRU_BATCH) { ++ __set_bit(i - 1, bitmap); ++ return; ++ } ++ ++ pmd = pmd_offset(pud, *start); ++ ++ ptl = pmd_lockptr(args->mm, pmd); ++ if (!spin_trylock(ptl)) ++ goto done; ++ ++ arch_enter_lazy_mmu_mode(); ++ ++ do { ++ unsigned long pfn; ++ struct folio *folio; ++ unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; ++ ++ pfn = get_pmd_pfn(pmd[i], vma, addr); ++ if (pfn == -1) ++ goto next; ++ ++ if (!pmd_trans_huge(pmd[i])) { ++ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && ++ get_cap(LRU_GEN_NONLEAF_YOUNG)) ++ pmdp_test_and_clear_young(vma, addr, pmd + i); ++ goto next; ++ } ++ ++ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); ++ if (!folio) ++ goto next; ++ ++ if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) ++ goto next; ++ ++ walk->mm_stats[MM_LEAF_YOUNG]++; ++ ++ if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && ++ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && ++ !folio_test_swapcache(folio))) ++ folio_mark_dirty(folio); ++ ++ old_gen = folio_update_gen(folio, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ update_batch_size(walk, folio, old_gen, new_gen); ++next: ++ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; ++ } while (i <= MIN_LRU_BATCH); ++ ++ arch_leave_lazy_mmu_mode(); ++ spin_unlock(ptl); ++done: ++ *start = -1; ++ bitmap_zero(bitmap, MIN_LRU_BATCH); ++} ++#else ++static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, ++ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) ++{ ++} ++#endif ++ ++static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, ++ struct mm_walk *args) ++{ ++ int i; ++ pmd_t *pmd; ++ unsigned long next; ++ unsigned long addr; ++ struct vm_area_struct *vma; ++ unsigned long pos = -1; ++ struct lru_gen_mm_walk *walk = args->private; ++ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; ++ ++ VM_WARN_ON_ONCE(pud_leaf(*pud)); ++ ++ /* ++ * Finish an entire PMD in two passes: the first only reaches to PTE ++ * tables to avoid taking the PMD lock; the second, if necessary, takes ++ * the PMD lock to clear the accessed bit in PMD entries. ++ */ ++ pmd = pmd_offset(pud, start & PUD_MASK); ++restart: ++ /* walk_pte_range() may call get_next_vma() */ ++ vma = args->vma; ++ for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { ++ pmd_t val = pmd_read_atomic(pmd + i); ++ ++ /* for pmd_read_atomic() */ ++ barrier(); ++ ++ next = pmd_addr_end(addr, end); ++ ++ if (!pmd_present(val) || is_huge_zero_pmd(val)) { ++ walk->mm_stats[MM_LEAF_TOTAL]++; ++ continue; ++ } ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (pmd_trans_huge(val)) { ++ unsigned long pfn = pmd_pfn(val); ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ ++ walk->mm_stats[MM_LEAF_TOTAL]++; ++ ++ if (!pmd_young(val)) { ++ walk->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } ++ ++ /* try to avoid unnecessary memory loads */ ++ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) ++ continue; ++ ++ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); ++ continue; ++ } ++#endif ++ walk->mm_stats[MM_NONLEAF_TOTAL]++; ++ ++#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG ++ if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { ++ if (!pmd_young(val)) ++ continue; ++ ++ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); ++ } ++#endif ++ if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) ++ continue; ++ ++ walk->mm_stats[MM_NONLEAF_FOUND]++; ++ ++ if (!walk_pte_range(&val, addr, next, args)) ++ continue; ++ ++ walk->mm_stats[MM_NONLEAF_ADDED]++; ++ ++ /* carry over to the next generation */ ++ update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); ++ } ++ ++ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); ++ ++ if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) ++ goto restart; ++} ++ ++static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, ++ struct mm_walk *args) ++{ ++ int i; ++ pud_t *pud; ++ unsigned long addr; ++ unsigned long next; ++ struct lru_gen_mm_walk *walk = args->private; ++ ++ VM_WARN_ON_ONCE(p4d_leaf(*p4d)); ++ ++ pud = pud_offset(p4d, start & P4D_MASK); ++restart: ++ for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { ++ pud_t val = READ_ONCE(pud[i]); ++ ++ next = pud_addr_end(addr, end); ++ ++ if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) ++ continue; ++ ++ walk_pmd_range(&val, addr, next, args); ++ ++ /* a racy check to curtail the waiting time */ ++ if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) ++ return 1; ++ ++ if (need_resched() || walk->batched >= MAX_LRU_BATCH) { ++ end = (addr | ~PUD_MASK) + 1; ++ goto done; ++ } ++ } ++ ++ if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) ++ goto restart; ++ ++ end = round_up(end, P4D_SIZE); ++done: ++ if (!end || !args->vma) ++ return 1; ++ ++ walk->next_addr = max(end, args->vma->vm_start); ++ ++ return -EAGAIN; ++} ++ ++static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) ++{ ++ static const struct mm_walk_ops mm_walk_ops = { ++ .test_walk = should_skip_vma, ++ .p4d_entry = walk_pud_range, ++ }; ++ ++ int err; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ ++ walk->next_addr = FIRST_USER_ADDRESS; ++ ++ do { ++ err = -EBUSY; ++ ++ /* folio_update_gen() requires stable folio_memcg() */ ++ if (!mem_cgroup_trylock_pages(memcg)) ++ break; ++ ++ /* the caller might be holding the lock for write */ ++ if (mmap_read_trylock(mm)) { ++ err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); ++ ++ mmap_read_unlock(mm); ++ } ++ ++ mem_cgroup_unlock_pages(); ++ ++ if (walk->batched) { ++ spin_lock_irq(&lruvec->lru_lock); ++ reset_batch_size(lruvec, walk); ++ spin_unlock_irq(&lruvec->lru_lock); ++ } ++ ++ cond_resched(); ++ } while (err == -EAGAIN); ++} ++ ++static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) ++{ ++ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; ++ ++ if (pgdat && current_is_kswapd()) { ++ VM_WARN_ON_ONCE(walk); ++ ++ walk = &pgdat->mm_walk; ++ } else if (!pgdat && !walk) { ++ VM_WARN_ON_ONCE(current_is_kswapd()); ++ ++ walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); ++ } ++ ++ current->reclaim_state->mm_walk = walk; ++ ++ return walk; ++} ++ ++static void clear_mm_walk(void) ++{ ++ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; ++ ++ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); ++ VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); ++ ++ current->reclaim_state->mm_walk = NULL; ++ ++ if (!current_is_kswapd()) ++ kfree(walk); ++} ++ ++static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) ++{ ++ int zone; ++ int remaining = MAX_LRU_BATCH; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ if (type == LRU_GEN_ANON && !can_swap) ++ goto done; ++ ++ /* prevent cold/hot inversion if force_scan is true */ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ struct list_head *head = &lrugen->lists[old_gen][type][zone]; ++ ++ while (!list_empty(head)) { ++ struct folio *folio = lru_to_folio(head); ++ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); ++ ++ new_gen = folio_inc_gen(lruvec, folio, false); ++ list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); ++ ++ if (!--remaining) ++ return false; ++ } ++ } ++done: ++ reset_ctrl_pos(lruvec, type, true); ++ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); ++ ++ return true; ++} ++ ++static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) ++{ ++ int gen, type, zone; ++ bool success = false; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ DEFINE_MIN_SEQ(lruvec); ++ ++ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); ++ ++ /* find the oldest populated generation */ ++ for (type = !can_swap; type < ANON_AND_FILE; type++) { ++ while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { ++ gen = lru_gen_from_seq(min_seq[type]); ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ if (!list_empty(&lrugen->lists[gen][type][zone])) ++ goto next; ++ } ++ ++ min_seq[type]++; ++ } ++next: ++ ; ++ } ++ ++ /* see the comment on lru_gen_struct */ ++ if (can_swap) { ++ min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); ++ min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); ++ } ++ ++ for (type = !can_swap; type < ANON_AND_FILE; type++) { ++ if (min_seq[type] == lrugen->min_seq[type]) ++ continue; ++ ++ reset_ctrl_pos(lruvec, type, true); ++ WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); ++ success = true; ++ } ++ ++ return success; ++} ++ ++static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) ++{ ++ int prev, next; ++ int type, zone; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ spin_lock_irq(&lruvec->lru_lock); ++ ++ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); ++ ++ for (type = ANON_AND_FILE - 1; type >= 0; type--) { ++ if (get_nr_gens(lruvec, type) != MAX_NR_GENS) ++ continue; ++ ++ VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); ++ ++ while (!inc_min_seq(lruvec, type, can_swap)) { ++ spin_unlock_irq(&lruvec->lru_lock); ++ cond_resched(); ++ spin_lock_irq(&lruvec->lru_lock); ++ } ++ } ++ ++ /* ++ * Update the active/inactive LRU sizes for compatibility. Both sides of ++ * the current max_seq need to be covered, since max_seq+1 can overlap ++ * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do ++ * overlap, cold/hot inversion happens. ++ */ ++ prev = lru_gen_from_seq(lrugen->max_seq - 1); ++ next = lru_gen_from_seq(lrugen->max_seq + 1); ++ ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ enum lru_list lru = type * LRU_INACTIVE_FILE; ++ long delta = lrugen->nr_pages[prev][type][zone] - ++ lrugen->nr_pages[next][type][zone]; ++ ++ if (!delta) ++ continue; ++ ++ __update_lru_size(lruvec, lru, zone, delta); ++ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); ++ } ++ } ++ ++ for (type = 0; type < ANON_AND_FILE; type++) ++ reset_ctrl_pos(lruvec, type, false); ++ ++ WRITE_ONCE(lrugen->timestamps[next], jiffies); ++ /* make sure preceding modifications appear */ ++ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); ++ ++ spin_unlock_irq(&lruvec->lru_lock); ++} ++ ++static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, ++ struct scan_control *sc, bool can_swap, bool force_scan) ++{ ++ bool success; ++ struct lru_gen_mm_walk *walk; ++ struct mm_struct *mm = NULL; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); ++ ++ /* see the comment in iterate_mm_list() */ ++ if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { ++ success = false; ++ goto done; ++ } ++ ++ /* ++ * If the hardware doesn't automatically set the accessed bit, fallback ++ * to lru_gen_look_around(), which only clears the accessed bit in a ++ * handful of PTEs. Spreading the work out over a period of time usually ++ * is less efficient, but it avoids bursty page faults. ++ */ ++ if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { ++ success = iterate_mm_list_nowalk(lruvec, max_seq); ++ goto done; ++ } ++ ++ walk = set_mm_walk(NULL); ++ if (!walk) { ++ success = iterate_mm_list_nowalk(lruvec, max_seq); ++ goto done; ++ } ++ ++ walk->lruvec = lruvec; ++ walk->max_seq = max_seq; ++ walk->can_swap = can_swap; ++ walk->force_scan = force_scan; ++ ++ do { ++ success = iterate_mm_list(lruvec, walk, &mm); ++ if (mm) ++ walk_mm(lruvec, mm, walk); ++ ++ cond_resched(); ++ } while (mm); ++done: ++ if (!success) { ++ if (sc->priority <= DEF_PRIORITY - 2) ++ wait_event_killable(lruvec->mm_state.wait, ++ max_seq < READ_ONCE(lrugen->max_seq)); ++ ++ return max_seq < READ_ONCE(lrugen->max_seq); ++ } ++ ++ VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); ++ ++ inc_max_seq(lruvec, can_swap, force_scan); ++ /* either this sees any waiters or they will see updated max_seq */ ++ if (wq_has_sleeper(&lruvec->mm_state.wait)) ++ wake_up_all(&lruvec->mm_state.wait); ++ ++ return true; ++} ++ ++static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, ++ struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) ++{ ++ int gen, type, zone; ++ unsigned long old = 0; ++ unsigned long young = 0; ++ unsigned long total = 0; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ ++ for (type = !can_swap; type < ANON_AND_FILE; type++) { ++ unsigned long seq; ++ ++ for (seq = min_seq[type]; seq <= max_seq; seq++) { ++ unsigned long size = 0; ++ ++ gen = lru_gen_from_seq(seq); ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) ++ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); ++ ++ total += size; ++ if (seq == max_seq) ++ young += size; ++ else if (seq + MIN_NR_GENS == max_seq) ++ old += size; ++ } ++ } ++ ++ /* try to scrape all its memory if this memcg was deleted */ ++ *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; ++ ++ /* ++ * The aging tries to be lazy to reduce the overhead, while the eviction ++ * stalls when the number of generations reaches MIN_NR_GENS. Hence, the ++ * ideal number of generations is MIN_NR_GENS+1. ++ */ ++ if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) ++ return true; ++ if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) ++ return false; ++ ++ /* ++ * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) ++ * of the total number of pages for each generation. A reasonable range ++ * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The ++ * aging cares about the upper bound of hot pages, while the eviction ++ * cares about the lower bound of cold pages. ++ */ ++ if (young * MIN_NR_GENS > total) ++ return true; ++ if (old * (MIN_NR_GENS + 2) < total) ++ return true; ++ ++ return false; ++} ++ ++static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) ++{ ++ bool need_aging; ++ unsigned long nr_to_scan; ++ int swappiness = get_swappiness(lruvec, sc); ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); ++ ++ VM_WARN_ON_ONCE(sc->memcg_low_reclaim); ++ ++ mem_cgroup_calculate_protection(NULL, memcg); ++ ++ if (mem_cgroup_below_min(memcg)) ++ return false; ++ ++ need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); ++ ++ if (min_ttl) { ++ int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); ++ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); ++ ++ if (time_is_after_jiffies(birth + min_ttl)) ++ return false; ++ ++ /* the size is likely too small to be helpful */ ++ if (!nr_to_scan && sc->priority != DEF_PRIORITY) ++ return false; ++ } ++ ++ if (need_aging) ++ try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); ++ ++ return true; ++} ++ ++/* to protect the working set of the last N jiffies */ ++static unsigned long lru_gen_min_ttl __read_mostly; ++ ++static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) ++{ ++ struct mem_cgroup *memcg; ++ bool success = false; ++ unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); ++ ++ VM_WARN_ON_ONCE(!current_is_kswapd()); ++ ++ sc->last_reclaimed = sc->nr_reclaimed; ++ ++ /* ++ * To reduce the chance of going into the aging path, which can be ++ * costly, optimistically skip it if the flag below was cleared in the ++ * eviction path. This improves the overall performance when multiple ++ * memcgs are available. ++ */ ++ if (!sc->memcgs_need_aging) { ++ sc->memcgs_need_aging = true; ++ return; ++ } ++ ++ set_mm_walk(pgdat); ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ ++ if (age_lruvec(lruvec, sc, min_ttl)) ++ success = true; ++ ++ cond_resched(); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++ ++ clear_mm_walk(); ++ ++ /* check the order to exclude compaction-induced reclaim */ ++ if (success || !min_ttl || sc->order) ++ return; ++ ++ /* ++ * The main goal is to OOM kill if every generation from all memcgs is ++ * younger than min_ttl. However, another possibility is all memcgs are ++ * either below min or empty. ++ */ ++ if (mutex_trylock(&oom_lock)) { ++ struct oom_control oc = { ++ .gfp_mask = sc->gfp_mask, ++ }; ++ ++ out_of_memory(&oc); ++ ++ mutex_unlock(&oom_lock); ++ } ++} ++ ++/* ++ * This function exploits spatial locality when shrink_page_list() walks the ++ * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If ++ * the scan was done cacheline efficiently, it adds the PMD entry pointing to ++ * the PTE table to the Bloom filter. This forms a feedback loop between the ++ * eviction and the aging. ++ */ ++void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) ++{ ++ int i; ++ pte_t *pte; ++ unsigned long start; ++ unsigned long end; ++ unsigned long addr; ++ struct lru_gen_mm_walk *walk; ++ int young = 0; ++ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; ++ struct folio *folio = pfn_folio(pvmw->pfn); ++ struct mem_cgroup *memcg = folio_memcg(folio); ++ struct pglist_data *pgdat = folio_pgdat(folio); ++ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ DEFINE_MAX_SEQ(lruvec); ++ int old_gen, new_gen = lru_gen_from_seq(max_seq); ++ ++ lockdep_assert_held(pvmw->ptl); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); ++ ++ if (spin_is_contended(pvmw->ptl)) ++ return; ++ ++ /* avoid taking the LRU lock under the PTL when possible */ ++ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; ++ ++ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); ++ end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; ++ ++ if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { ++ if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) ++ end = start + MIN_LRU_BATCH * PAGE_SIZE; ++ else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) ++ start = end - MIN_LRU_BATCH * PAGE_SIZE; ++ else { ++ start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; ++ end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; ++ } ++ } ++ ++ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; ++ ++ rcu_read_lock(); ++ arch_enter_lazy_mmu_mode(); ++ ++ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { ++ unsigned long pfn; ++ ++ pfn = get_pte_pfn(pte[i], pvmw->vma, addr); ++ if (pfn == -1) ++ continue; ++ ++ if (!pte_young(pte[i])) ++ continue; ++ ++ folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); ++ if (!folio) ++ continue; ++ ++ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) ++ VM_WARN_ON_ONCE(true); ++ ++ young++; ++ ++ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && ++ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && ++ !folio_test_swapcache(folio))) ++ folio_mark_dirty(folio); ++ ++ old_gen = folio_lru_gen(folio); ++ if (old_gen < 0) ++ folio_set_referenced(folio); ++ else if (old_gen != new_gen) ++ __set_bit(i, bitmap); ++ } ++ ++ arch_leave_lazy_mmu_mode(); ++ rcu_read_unlock(); ++ ++ /* feedback from rmap walkers to page table walkers */ ++ if (suitable_to_scan(i, young)) ++ update_bloom_filter(lruvec, max_seq, pvmw->pmd); ++ ++ if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { ++ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { ++ folio = pfn_folio(pte_pfn(pte[i])); ++ folio_activate(folio); ++ } ++ return; ++ } ++ ++ /* folio_update_gen() requires stable folio_memcg() */ ++ if (!mem_cgroup_trylock_pages(memcg)) ++ return; ++ ++ if (!walk) { ++ spin_lock_irq(&lruvec->lru_lock); ++ new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); ++ } ++ ++ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { ++ folio = pfn_folio(pte_pfn(pte[i])); ++ if (folio_memcg_rcu(folio) != memcg) ++ continue; ++ ++ old_gen = folio_update_gen(folio, new_gen); ++ if (old_gen < 0 || old_gen == new_gen) ++ continue; ++ ++ if (walk) ++ update_batch_size(walk, folio, old_gen, new_gen); ++ else ++ lru_gen_update_size(lruvec, folio, old_gen, new_gen); ++ } ++ ++ if (!walk) ++ spin_unlock_irq(&lruvec->lru_lock); ++ ++ mem_cgroup_unlock_pages(); ++} ++ ++/****************************************************************************** ++ * the eviction ++ ******************************************************************************/ ++ ++static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) ++{ ++ bool success; ++ int gen = folio_lru_gen(folio); ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ int delta = folio_nr_pages(folio); ++ int refs = folio_lru_refs(folio); ++ int tier = lru_tier_from_refs(refs); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); ++ ++ /* unevictable */ ++ if (!folio_evictable(folio)) { ++ success = lru_gen_del_folio(lruvec, folio, true); ++ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ folio_set_unevictable(folio); ++ lruvec_add_folio(lruvec, folio); ++ __count_vm_events(UNEVICTABLE_PGCULLED, delta); ++ return true; ++ } ++ ++ /* dirty lazyfree */ ++ if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { ++ success = lru_gen_del_folio(lruvec, folio, true); ++ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ folio_set_swapbacked(folio); ++ lruvec_add_folio_tail(lruvec, folio); ++ return true; ++ } ++ ++ /* promoted */ ++ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { ++ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); ++ return true; ++ } ++ ++ /* protected */ ++ if (tier > tier_idx) { ++ int hist = lru_hist_from_seq(lrugen->min_seq[type]); ++ ++ gen = folio_inc_gen(lruvec, folio, false); ++ list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); ++ ++ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], ++ lrugen->protected[hist][type][tier - 1] + delta); ++ __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); ++ return true; ++ } ++ ++ /* waiting for writeback */ ++ if (folio_test_locked(folio) || folio_test_writeback(folio) || ++ (type == LRU_GEN_FILE && folio_test_dirty(folio))) { ++ gen = folio_inc_gen(lruvec, folio, true); ++ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); ++ return true; ++ } ++ ++ return false; ++} ++ ++static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) ++{ ++ bool success; ++ ++ /* unmapping inhibited */ ++ if (!sc->may_unmap && folio_mapped(folio)) ++ return false; ++ ++ /* swapping inhibited */ ++ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && ++ (folio_test_dirty(folio) || ++ (folio_test_anon(folio) && !folio_test_swapcache(folio)))) ++ return false; ++ ++ /* raced with release_pages() */ ++ if (!folio_try_get(folio)) ++ return false; ++ ++ /* raced with another isolation */ ++ if (!folio_test_clear_lru(folio)) { ++ folio_put(folio); ++ return false; ++ } ++ ++ /* see the comment on MAX_NR_TIERS */ ++ if (!folio_test_referenced(folio)) ++ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); ++ ++ /* for shrink_page_list() */ ++ folio_clear_reclaim(folio); ++ folio_clear_referenced(folio); ++ ++ success = lru_gen_del_folio(lruvec, folio, true); ++ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ ++ return true; ++} ++ ++static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, ++ int type, int tier, struct list_head *list) ++{ ++ int gen, zone; ++ enum vm_event_item item; ++ int sorted = 0; ++ int scanned = 0; ++ int isolated = 0; ++ int remaining = MAX_LRU_BATCH; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ ++ VM_WARN_ON_ONCE(!list_empty(list)); ++ ++ if (get_nr_gens(lruvec, type) == MIN_NR_GENS) ++ return 0; ++ ++ gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ ++ for (zone = sc->reclaim_idx; zone >= 0; zone--) { ++ LIST_HEAD(moved); ++ int skipped = 0; ++ struct list_head *head = &lrugen->lists[gen][type][zone]; ++ ++ while (!list_empty(head)) { ++ struct folio *folio = lru_to_folio(head); ++ int delta = folio_nr_pages(folio); ++ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); ++ ++ scanned += delta; ++ ++ if (sort_folio(lruvec, folio, tier)) ++ sorted += delta; ++ else if (isolate_folio(lruvec, folio, sc)) { ++ list_add(&folio->lru, list); ++ isolated += delta; ++ } else { ++ list_move(&folio->lru, &moved); ++ skipped += delta; ++ } ++ ++ if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) ++ break; ++ } ++ ++ if (skipped) { ++ list_splice(&moved, head); ++ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); ++ } ++ ++ if (!remaining || isolated >= MIN_LRU_BATCH) ++ break; ++ } ++ ++ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; ++ if (!cgroup_reclaim(sc)) { ++ __count_vm_events(item, isolated); ++ __count_vm_events(PGREFILL, sorted); ++ } ++ __count_memcg_events(memcg, item, isolated); ++ __count_memcg_events(memcg, PGREFILL, sorted); ++ __count_vm_events(PGSCAN_ANON + type, isolated); ++ ++ /* ++ * There might not be eligible pages due to reclaim_idx, may_unmap and ++ * may_writepage. Check the remaining to prevent livelock if it's not ++ * making progress. ++ */ ++ return isolated || !remaining ? scanned : 0; ++} ++ ++static int get_tier_idx(struct lruvec *lruvec, int type) ++{ ++ int tier; ++ struct ctrl_pos sp, pv; ++ ++ /* ++ * To leave a margin for fluctuations, use a larger gain factor (1:2). ++ * This value is chosen because any other tier would have at least twice ++ * as many refaults as the first tier. ++ */ ++ read_ctrl_pos(lruvec, type, 0, 1, &sp); ++ for (tier = 1; tier < MAX_NR_TIERS; tier++) { ++ read_ctrl_pos(lruvec, type, tier, 2, &pv); ++ if (!positive_ctrl_err(&sp, &pv)) ++ break; ++ } ++ ++ return tier - 1; ++} ++ ++static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) ++{ ++ int type, tier; ++ struct ctrl_pos sp, pv; ++ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; ++ ++ /* ++ * Compare the first tier of anon with that of file to determine which ++ * type to scan. Also need to compare other tiers of the selected type ++ * with the first tier of the other type to determine the last tier (of ++ * the selected type) to evict. ++ */ ++ read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); ++ read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); ++ type = positive_ctrl_err(&sp, &pv); ++ ++ read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); ++ for (tier = 1; tier < MAX_NR_TIERS; tier++) { ++ read_ctrl_pos(lruvec, type, tier, gain[type], &pv); ++ if (!positive_ctrl_err(&sp, &pv)) ++ break; ++ } ++ ++ *tier_idx = tier - 1; ++ ++ return type; ++} ++ ++static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, ++ int *type_scanned, struct list_head *list) ++{ ++ int i; ++ int type; ++ int scanned; ++ int tier = -1; ++ DEFINE_MIN_SEQ(lruvec); ++ ++ /* ++ * Try to make the obvious choice first. When anon and file are both ++ * available from the same generation, interpret swappiness 1 as file ++ * first and 200 as anon first. ++ */ ++ if (!swappiness) ++ type = LRU_GEN_FILE; ++ else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) ++ type = LRU_GEN_ANON; ++ else if (swappiness == 1) ++ type = LRU_GEN_FILE; ++ else if (swappiness == 200) ++ type = LRU_GEN_ANON; ++ else ++ type = get_type_to_scan(lruvec, swappiness, &tier); ++ ++ for (i = !swappiness; i < ANON_AND_FILE; i++) { ++ if (tier < 0) ++ tier = get_tier_idx(lruvec, type); ++ ++ scanned = scan_folios(lruvec, sc, type, tier, list); ++ if (scanned) ++ break; ++ ++ type = !type; ++ tier = -1; ++ } ++ ++ *type_scanned = type; ++ ++ return scanned; ++} ++ ++static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, ++ bool *need_swapping) ++{ ++ int type; ++ int scanned; ++ int reclaimed; ++ LIST_HEAD(list); ++ struct folio *folio; ++ enum vm_event_item item; ++ struct reclaim_stat stat; ++ struct lru_gen_mm_walk *walk; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ ++ spin_lock_irq(&lruvec->lru_lock); ++ ++ scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); ++ ++ scanned += try_to_inc_min_seq(lruvec, swappiness); ++ ++ if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) ++ scanned = 0; ++ ++ spin_unlock_irq(&lruvec->lru_lock); ++ ++ if (list_empty(&list)) ++ return scanned; ++ ++ reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); ++ ++ list_for_each_entry(folio, &list, lru) { ++ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ ++ if (folio_test_workingset(folio)) ++ folio_set_referenced(folio); ++ ++ /* don't add rejected pages to the oldest generation */ ++ if (folio_test_reclaim(folio) && ++ (folio_test_dirty(folio) || folio_test_writeback(folio))) ++ folio_clear_active(folio); ++ else ++ folio_set_active(folio); ++ } ++ ++ spin_lock_irq(&lruvec->lru_lock); ++ ++ move_pages_to_lru(lruvec, &list); ++ ++ walk = current->reclaim_state->mm_walk; ++ if (walk && walk->batched) ++ reset_batch_size(lruvec, walk); ++ ++ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; ++ if (!cgroup_reclaim(sc)) ++ __count_vm_events(item, reclaimed); ++ __count_memcg_events(memcg, item, reclaimed); ++ __count_vm_events(PGSTEAL_ANON + type, reclaimed); ++ ++ spin_unlock_irq(&lruvec->lru_lock); ++ ++ mem_cgroup_uncharge_list(&list); ++ free_unref_page_list(&list); ++ ++ sc->nr_reclaimed += reclaimed; ++ ++ if (need_swapping && type == LRU_GEN_ANON) ++ *need_swapping = true; ++ ++ return scanned; ++} ++ ++/* ++ * For future optimizations: ++ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg ++ * reclaim. ++ */ ++static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, ++ bool can_swap, bool *need_aging) ++{ ++ unsigned long nr_to_scan; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); ++ ++ if (mem_cgroup_below_min(memcg) || ++ (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) ++ return 0; ++ ++ *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); ++ if (!*need_aging) ++ return nr_to_scan; ++ ++ /* skip the aging path at the default priority */ ++ if (sc->priority == DEF_PRIORITY) ++ goto done; ++ ++ /* leave the work to lru_gen_age_node() */ ++ if (current_is_kswapd()) ++ return 0; ++ ++ if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) ++ return nr_to_scan; ++done: ++ return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; ++} ++ ++static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, ++ struct scan_control *sc, bool need_swapping) ++{ ++ int i; ++ DEFINE_MAX_SEQ(lruvec); ++ ++ if (!current_is_kswapd()) { ++ /* age each memcg at most once to ensure fairness */ ++ if (max_seq - seq > 1) ++ return true; ++ ++ /* over-swapping can increase allocation latency */ ++ if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) ++ return true; ++ ++ /* give this thread a chance to exit and free its memory */ ++ if (fatal_signal_pending(current)) { ++ sc->nr_reclaimed += MIN_LRU_BATCH; ++ return true; ++ } ++ ++ if (cgroup_reclaim(sc)) ++ return false; ++ } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) ++ return false; ++ ++ /* keep scanning at low priorities to ensure fairness */ ++ if (sc->priority > DEF_PRIORITY - 2) ++ return false; ++ ++ /* ++ * A minimum amount of work was done under global memory pressure. For ++ * kswapd, it may be overshooting. For direct reclaim, the allocation ++ * may succeed if all suitable zones are somewhat safe. In either case, ++ * it's better to stop now, and restart later if necessary. ++ */ ++ for (i = 0; i <= sc->reclaim_idx; i++) { ++ unsigned long wmark; ++ struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; ++ ++ if (!managed_zone(zone)) ++ continue; ++ ++ wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); ++ if (wmark > zone_page_state(zone, NR_FREE_PAGES)) ++ return false; ++ } ++ ++ sc->nr_reclaimed += MIN_LRU_BATCH; ++ ++ return true; ++} ++ ++static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++{ ++ struct blk_plug plug; ++ bool need_aging = false; ++ bool need_swapping = false; ++ unsigned long scanned = 0; ++ unsigned long reclaimed = sc->nr_reclaimed; ++ DEFINE_MAX_SEQ(lruvec); ++ ++ lru_add_drain(); ++ ++ blk_start_plug(&plug); ++ ++ set_mm_walk(lruvec_pgdat(lruvec)); ++ ++ while (true) { ++ int delta; ++ int swappiness; ++ unsigned long nr_to_scan; ++ ++ if (sc->may_swap) ++ swappiness = get_swappiness(lruvec, sc); ++ else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) ++ swappiness = 1; ++ else ++ swappiness = 0; ++ ++ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); ++ if (!nr_to_scan) ++ goto done; ++ ++ delta = evict_folios(lruvec, sc, swappiness, &need_swapping); ++ if (!delta) ++ goto done; ++ ++ scanned += delta; ++ if (scanned >= nr_to_scan) ++ break; ++ ++ if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) ++ break; ++ ++ cond_resched(); ++ } ++ ++ /* see the comment in lru_gen_age_node() */ ++ if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) ++ sc->memcgs_need_aging = false; ++done: ++ clear_mm_walk(); ++ ++ blk_finish_plug(&plug); ++} ++ ++/****************************************************************************** ++ * state change ++ ******************************************************************************/ ++ ++static bool __maybe_unused state_is_valid(struct lruvec *lruvec) ++{ ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ if (lrugen->enabled) { ++ enum lru_list lru; ++ ++ for_each_evictable_lru(lru) { ++ if (!list_empty(&lruvec->lists[lru])) ++ return false; ++ } ++ } else { ++ int gen, type, zone; ++ ++ for_each_gen_type_zone(gen, type, zone) { ++ if (!list_empty(&lrugen->lists[gen][type][zone])) ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++static bool fill_evictable(struct lruvec *lruvec) ++{ ++ enum lru_list lru; ++ int remaining = MAX_LRU_BATCH; ++ ++ for_each_evictable_lru(lru) { ++ int type = is_file_lru(lru); ++ bool active = is_active_lru(lru); ++ struct list_head *head = &lruvec->lists[lru]; ++ ++ while (!list_empty(head)) { ++ bool success; ++ struct folio *folio = lru_to_folio(head); ++ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); ++ ++ lruvec_del_folio(lruvec, folio); ++ success = lru_gen_add_folio(lruvec, folio, false); ++ VM_WARN_ON_ONCE(!success); ++ ++ if (!--remaining) ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++static bool drain_evictable(struct lruvec *lruvec) ++{ ++ int gen, type, zone; ++ int remaining = MAX_LRU_BATCH; ++ ++ for_each_gen_type_zone(gen, type, zone) { ++ struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; ++ ++ while (!list_empty(head)) { ++ bool success; ++ struct folio *folio = lru_to_folio(head); ++ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); ++ ++ success = lru_gen_del_folio(lruvec, folio, false); ++ VM_WARN_ON_ONCE(!success); ++ lruvec_add_folio(lruvec, folio); ++ ++ if (!--remaining) ++ return false; ++ } ++ } ++ ++ return true; ++} ++ ++static void lru_gen_change_state(bool enabled) ++{ ++ static DEFINE_MUTEX(state_mutex); ++ ++ struct mem_cgroup *memcg; ++ ++ cgroup_lock(); ++ cpus_read_lock(); ++ get_online_mems(); ++ mutex_lock(&state_mutex); ++ ++ if (enabled == lru_gen_enabled()) ++ goto unlock; ++ ++ if (enabled) ++ static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); ++ else ++ static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ int nid; ++ ++ for_each_node(nid) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ ++ if (!lruvec) ++ continue; ++ ++ spin_lock_irq(&lruvec->lru_lock); ++ ++ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); ++ VM_WARN_ON_ONCE(!state_is_valid(lruvec)); ++ ++ lruvec->lrugen.enabled = enabled; ++ ++ while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { ++ spin_unlock_irq(&lruvec->lru_lock); ++ cond_resched(); ++ spin_lock_irq(&lruvec->lru_lock); ++ } ++ ++ spin_unlock_irq(&lruvec->lru_lock); ++ } ++ ++ cond_resched(); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++unlock: ++ mutex_unlock(&state_mutex); ++ put_online_mems(); ++ cpus_read_unlock(); ++ cgroup_unlock(); ++} ++ ++/****************************************************************************** ++ * sysfs interface ++ ******************************************************************************/ ++ ++static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); ++} ++ ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, ++ const char *buf, size_t len) ++{ ++ unsigned int msecs; ++ ++ if (kstrtouint(buf, 0, &msecs)) ++ return -EINVAL; ++ ++ WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); ++ ++ return len; ++} ++ ++static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( ++ min_ttl_ms, 0644, show_min_ttl, store_min_ttl ++); ++ ++static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) ++{ ++ unsigned int caps = 0; ++ ++ if (get_cap(LRU_GEN_CORE)) ++ caps |= BIT(LRU_GEN_CORE); ++ ++ if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) ++ caps |= BIT(LRU_GEN_MM_WALK); ++ ++ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) ++ caps |= BIT(LRU_GEN_NONLEAF_YOUNG); ++ ++ return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); ++} ++ ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, ++ const char *buf, size_t len) ++{ ++ int i; ++ unsigned int caps; ++ ++ if (tolower(*buf) == 'n') ++ caps = 0; ++ else if (tolower(*buf) == 'y') ++ caps = -1; ++ else if (kstrtouint(buf, 0, &caps)) ++ return -EINVAL; ++ ++ for (i = 0; i < NR_LRU_GEN_CAPS; i++) { ++ bool enabled = caps & BIT(i); ++ ++ if (i == LRU_GEN_CORE) ++ lru_gen_change_state(enabled); ++ else if (enabled) ++ static_branch_enable(&lru_gen_caps[i]); ++ else ++ static_branch_disable(&lru_gen_caps[i]); ++ } ++ ++ return len; ++} ++ ++static struct kobj_attribute lru_gen_enabled_attr = __ATTR( ++ enabled, 0644, show_enabled, store_enabled ++); ++ ++static struct attribute *lru_gen_attrs[] = { ++ &lru_gen_min_ttl_attr.attr, ++ &lru_gen_enabled_attr.attr, ++ NULL ++}; ++ ++static struct attribute_group lru_gen_attr_group = { ++ .name = "lru_gen", ++ .attrs = lru_gen_attrs, ++}; ++ ++/****************************************************************************** ++ * debugfs interface ++ ******************************************************************************/ ++ ++static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) ++{ ++ struct mem_cgroup *memcg; ++ loff_t nr_to_skip = *pos; ++ ++ m->private = kvmalloc(PATH_MAX, GFP_KERNEL); ++ if (!m->private) ++ return ERR_PTR(-ENOMEM); ++ ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ int nid; ++ ++ for_each_node_state(nid, N_MEMORY) { ++ if (!nr_to_skip--) ++ return get_lruvec(memcg, nid); ++ } ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++ ++ return NULL; ++} ++ ++static void lru_gen_seq_stop(struct seq_file *m, void *v) ++{ ++ if (!IS_ERR_OR_NULL(v)) ++ mem_cgroup_iter_break(NULL, lruvec_memcg(v)); ++ ++ kvfree(m->private); ++ m->private = NULL; ++} ++ ++static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ int nid = lruvec_pgdat(v)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(v); ++ ++ ++*pos; ++ ++ nid = next_memory_node(nid); ++ if (nid == MAX_NUMNODES) { ++ memcg = mem_cgroup_iter(NULL, memcg, NULL); ++ if (!memcg) ++ return NULL; ++ ++ nid = first_memory_node; ++ } ++ ++ return get_lruvec(memcg, nid); ++} ++ ++static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, ++ unsigned long max_seq, unsigned long *min_seq, ++ unsigned long seq) ++{ ++ int i; ++ int type, tier; ++ int hist = lru_hist_from_seq(seq); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ for (tier = 0; tier < MAX_NR_TIERS; tier++) { ++ seq_printf(m, " %10d", tier); ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ const char *s = " "; ++ unsigned long n[3] = {}; ++ ++ if (seq == max_seq) { ++ s = "RT "; ++ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); ++ n[1] = READ_ONCE(lrugen->avg_total[type][tier]); ++ } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { ++ s = "rep"; ++ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); ++ } ++ ++ for (i = 0; i < 3; i++) ++ seq_printf(m, " %10lu%c", n[i], s[i]); ++ } ++ seq_putc(m, '\n'); ++ } ++ ++ seq_puts(m, " "); ++ for (i = 0; i < NR_MM_STATS; i++) { ++ const char *s = " "; ++ unsigned long n = 0; ++ ++ if (seq == max_seq && NR_HIST_GENS == 1) { ++ s = "LOYNFA"; ++ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); ++ } else if (seq != max_seq && NR_HIST_GENS > 1) { ++ s = "loynfa"; ++ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); ++ } ++ ++ seq_printf(m, " %10lu%c", n, s[i]); ++ } ++ seq_putc(m, '\n'); ++} ++ ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static int lru_gen_seq_show(struct seq_file *m, void *v) ++{ ++ unsigned long seq; ++ bool full = !debugfs_real_fops(m->file)->write; ++ struct lruvec *lruvec = v; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int nid = lruvec_pgdat(lruvec)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); ++ ++ if (nid == first_memory_node) { ++ const char *path = memcg ? m->private : ""; ++ ++#ifdef CONFIG_MEMCG ++ if (memcg) ++ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); ++#endif ++ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); ++ } ++ ++ seq_printf(m, " node %5d\n", nid); ++ ++ if (!full) ++ seq = min_seq[LRU_GEN_ANON]; ++ else if (max_seq >= MAX_NR_GENS) ++ seq = max_seq - MAX_NR_GENS + 1; ++ else ++ seq = 0; ++ ++ for (; seq <= max_seq; seq++) { ++ int type, zone; ++ int gen = lru_gen_from_seq(seq); ++ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); ++ ++ seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); ++ ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ unsigned long size = 0; ++ char mark = full && seq < min_seq[type] ? 'x' : ' '; ++ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) ++ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); ++ ++ seq_printf(m, " %10lu%c", size, mark); ++ } ++ ++ seq_putc(m, '\n'); ++ ++ if (full) ++ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); ++ } ++ ++ return 0; ++} ++ ++static const struct seq_operations lru_gen_seq_ops = { ++ .start = lru_gen_seq_start, ++ .stop = lru_gen_seq_stop, ++ .next = lru_gen_seq_next, ++ .show = lru_gen_seq_show, ++}; ++ ++static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, ++ bool can_swap, bool force_scan) ++{ ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); ++ ++ if (seq < max_seq) ++ return 0; ++ ++ if (seq > max_seq) ++ return -EINVAL; ++ ++ if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) ++ return -ERANGE; ++ ++ try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); ++ ++ return 0; ++} ++ ++static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, ++ int swappiness, unsigned long nr_to_reclaim) ++{ ++ DEFINE_MAX_SEQ(lruvec); ++ ++ if (seq + MIN_NR_GENS > max_seq) ++ return -EINVAL; ++ ++ sc->nr_reclaimed = 0; ++ ++ while (!signal_pending(current)) { ++ DEFINE_MIN_SEQ(lruvec); ++ ++ if (seq < min_seq[!swappiness]) ++ return 0; ++ ++ if (sc->nr_reclaimed >= nr_to_reclaim) ++ return 0; ++ ++ if (!evict_folios(lruvec, sc, swappiness, NULL)) ++ return 0; ++ ++ cond_resched(); ++ } ++ ++ return -EINTR; ++} ++ ++static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, ++ struct scan_control *sc, int swappiness, unsigned long opt) ++{ ++ struct lruvec *lruvec; ++ int err = -EINVAL; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) ++ return -EINVAL; ++ ++ if (!mem_cgroup_disabled()) { ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_id(memcg_id); ++#ifdef CONFIG_MEMCG ++ if (memcg && !css_tryget(&memcg->css)) ++ memcg = NULL; ++#endif ++ rcu_read_unlock(); ++ ++ if (!memcg) ++ return -EINVAL; ++ } ++ ++ if (memcg_id != mem_cgroup_id(memcg)) ++ goto done; ++ ++ lruvec = get_lruvec(memcg, nid); ++ ++ if (swappiness < 0) ++ swappiness = get_swappiness(lruvec, sc); ++ else if (swappiness > 200) ++ goto done; ++ ++ switch (cmd) { ++ case '+': ++ err = run_aging(lruvec, seq, sc, swappiness, opt); ++ break; ++ case '-': ++ err = run_eviction(lruvec, seq, sc, swappiness, opt); ++ break; ++ } ++done: ++ mem_cgroup_put(memcg); ++ ++ return err; ++} ++ ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, ++ size_t len, loff_t *pos) ++{ ++ void *buf; ++ char *cur, *next; ++ unsigned int flags; ++ struct blk_plug plug; ++ int err = -EINVAL; ++ struct scan_control sc = { ++ .may_writepage = true, ++ .may_unmap = true, ++ .may_swap = true, ++ .reclaim_idx = MAX_NR_ZONES - 1, ++ .gfp_mask = GFP_KERNEL, ++ }; ++ ++ buf = kvmalloc(len + 1, GFP_KERNEL); ++ if (!buf) ++ return -ENOMEM; ++ ++ if (copy_from_user(buf, src, len)) { ++ kvfree(buf); ++ return -EFAULT; ++ } ++ ++ set_task_reclaim_state(current, &sc.reclaim_state); ++ flags = memalloc_noreclaim_save(); ++ blk_start_plug(&plug); ++ if (!set_mm_walk(NULL)) { ++ err = -ENOMEM; ++ goto done; ++ } ++ ++ next = buf; ++ next[len] = '\0'; ++ ++ while ((cur = strsep(&next, ",;\n"))) { ++ int n; ++ int end; ++ char cmd; ++ unsigned int memcg_id; ++ unsigned int nid; ++ unsigned long seq; ++ unsigned int swappiness = -1; ++ unsigned long opt = -1; ++ ++ cur = skip_spaces(cur); ++ if (!*cur) ++ continue; ++ ++ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, ++ &seq, &end, &swappiness, &end, &opt, &end); ++ if (n < 4 || cur[end]) { ++ err = -EINVAL; ++ break; ++ } ++ ++ err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); ++ if (err) ++ break; ++ } ++done: ++ clear_mm_walk(); ++ blk_finish_plug(&plug); ++ memalloc_noreclaim_restore(flags); ++ set_task_reclaim_state(current, NULL); ++ ++ kvfree(buf); ++ ++ return err ? : len; ++} ++ ++static int lru_gen_seq_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &lru_gen_seq_ops); ++} ++ ++static const struct file_operations lru_gen_rw_fops = { ++ .open = lru_gen_seq_open, ++ .read = seq_read, ++ .write = lru_gen_seq_write, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++static const struct file_operations lru_gen_ro_fops = { ++ .open = lru_gen_seq_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++/****************************************************************************** ++ * initialization ++ ******************************************************************************/ ++ ++void lru_gen_init_lruvec(struct lruvec *lruvec) ++{ ++ int i; ++ int gen, type, zone; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ ++ lrugen->max_seq = MIN_NR_GENS + 1; ++ lrugen->enabled = lru_gen_enabled(); ++ ++ for (i = 0; i <= MIN_NR_GENS + 1; i++) ++ lrugen->timestamps[i] = jiffies; ++ ++ for_each_gen_type_zone(gen, type, zone) ++ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); ++ ++ lruvec->mm_state.seq = MIN_NR_GENS; ++ init_waitqueue_head(&lruvec->mm_state.wait); ++} ++ ++#ifdef CONFIG_MEMCG ++void lru_gen_init_memcg(struct mem_cgroup *memcg) ++{ ++ INIT_LIST_HEAD(&memcg->mm_list.fifo); ++ spin_lock_init(&memcg->mm_list.lock); ++} ++ ++void lru_gen_exit_memcg(struct mem_cgroup *memcg) ++{ ++ int i; ++ int nid; ++ ++ for_each_node(nid) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ ++ VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, ++ sizeof(lruvec->lrugen.nr_pages))); ++ ++ for (i = 0; i < NR_BLOOM_FILTERS; i++) { ++ bitmap_free(lruvec->mm_state.filters[i]); ++ lruvec->mm_state.filters[i] = NULL; ++ } ++ } ++} ++#endif ++ ++static int __init init_lru_gen(void) ++{ ++ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); ++ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); ++ ++ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) ++ pr_err("lru_gen: failed to create sysfs group\n"); ++ ++ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); ++ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); ++ ++ return 0; ++}; ++late_initcall(init_lru_gen); ++ ++#else /* !CONFIG_LRU_GEN */ ++ ++static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) ++{ ++} ++ ++static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++{ ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ ++static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++{ ++ unsigned long nr[NR_LRU_LISTS]; ++ unsigned long targets[NR_LRU_LISTS]; ++ unsigned long nr_to_scan; ++ enum lru_list lru; ++ unsigned long nr_reclaimed = 0; ++ unsigned long nr_to_reclaim = sc->nr_to_reclaim; ++ bool proportional_reclaim; ++ struct blk_plug plug; ++ ++ if (lru_gen_enabled()) { ++ lru_gen_shrink_lruvec(lruvec, sc); ++ return; ++ } ++ ++ get_scan_count(lruvec, sc, nr); ++ ++ /* Record the original scan target for proportional adjustments later */ ++ memcpy(targets, nr, sizeof(nr)); ++ ++ /* ++ * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal ++ * event that can occur when there is little memory pressure e.g. ++ * multiple streaming readers/writers. Hence, we do not abort scanning ++ * when the requested number of pages are reclaimed when scanning at ++ * DEF_PRIORITY on the assumption that the fact we are direct ++ * reclaiming implies that kswapd is not keeping up and it is best to ++ * do a batch of work at once. For memcg reclaim one check is made to ++ * abort proportional reclaim if either the file or anon lru has already ++ * dropped to zero at the first pass. ++ */ ++ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && ++ sc->priority == DEF_PRIORITY); ++ ++ blk_start_plug(&plug); ++ while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || ++ nr[LRU_INACTIVE_FILE]) { ++ unsigned long nr_anon, nr_file, percentage; ++ unsigned long nr_scanned; ++ ++ for_each_evictable_lru(lru) { ++ if (nr[lru]) { ++ nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); ++ nr[lru] -= nr_to_scan; ++ ++ nr_reclaimed += shrink_list(lru, nr_to_scan, ++ lruvec, sc); ++ } ++ } ++ ++ cond_resched(); ++ ++ if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) ++ continue; ++ ++ /* ++ * For kswapd and memcg, reclaim at least the number of pages ++ * requested. Ensure that the anon and file LRUs are scanned ++ * proportionally what was requested by get_scan_count(). We ++ * stop reclaiming one LRU and reduce the amount scanning ++ * proportional to the original scan target. ++ */ ++ nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; ++ nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; ++ ++ /* ++ * It's just vindictive to attack the larger once the smaller ++ * has gone to zero. And given the way we stop scanning the ++ * smaller below, this makes sure that we only make one nudge ++ * towards proportionality once we've got nr_to_reclaim. ++ */ ++ if (!nr_file || !nr_anon) ++ break; ++ ++ if (nr_file > nr_anon) { ++ unsigned long scan_target = targets[LRU_INACTIVE_ANON] + ++ targets[LRU_ACTIVE_ANON] + 1; ++ lru = LRU_BASE; ++ percentage = nr_anon * 100 / scan_target; ++ } else { ++ unsigned long scan_target = targets[LRU_INACTIVE_FILE] + ++ targets[LRU_ACTIVE_FILE] + 1; ++ lru = LRU_FILE; ++ percentage = nr_file * 100 / scan_target; ++ } ++ ++ /* Stop scanning the smaller of the LRU */ ++ nr[lru] = 0; ++ nr[lru + LRU_ACTIVE] = 0; ++ ++ /* ++ * Recalculate the other LRU scan count based on its original ++ * scan target and the percentage scanning already complete ++ */ ++ lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; ++ nr_scanned = targets[lru] - nr[lru]; ++ nr[lru] = targets[lru] * (100 - percentage) / 100; ++ nr[lru] -= min(nr[lru], nr_scanned); ++ ++ lru += LRU_ACTIVE; ++ nr_scanned = targets[lru] - nr[lru]; ++ nr[lru] = targets[lru] * (100 - percentage) / 100; ++ nr[lru] -= min(nr[lru], nr_scanned); ++ } ++ blk_finish_plug(&plug); ++ sc->nr_reclaimed += nr_reclaimed; ++ ++ /* ++ * Even if we did not try to evict anon pages at all, we want to ++ * rebalance the anon lru active/inactive ratio. ++ */ ++ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && ++ inactive_is_low(lruvec, LRU_INACTIVE_ANON)) ++ shrink_active_list(SWAP_CLUSTER_MAX, lruvec, ++ sc, LRU_ACTIVE_ANON); ++} ++ ++/* Use reclaim/compaction for costly allocs or under memory pressure */ ++static bool in_reclaim_compaction(struct scan_control *sc) ++{ ++ if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && ++ (sc->order > PAGE_ALLOC_COSTLY_ORDER || ++ sc->priority < DEF_PRIORITY - 2)) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * Reclaim/compaction is used for high-order allocation requests. It reclaims ++ * order-0 pages before compacting the zone. should_continue_reclaim() returns ++ * true if more pages should be reclaimed such that when the page allocator ++ * calls try_to_compact_pages() that it will have enough free pages to succeed. ++ * It will give up earlier than that if there is difficulty reclaiming pages. ++ */ ++static inline bool should_continue_reclaim(struct pglist_data *pgdat, ++ unsigned long nr_reclaimed, ++ struct scan_control *sc) ++{ ++ unsigned long pages_for_compaction; ++ unsigned long inactive_lru_pages; ++ int z; ++ ++ /* If not in reclaim/compaction mode, stop */ ++ if (!in_reclaim_compaction(sc)) ++ return false; ++ ++ /* ++ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX ++ * number of pages that were scanned. This will return to the caller ++ * with the risk reclaim/compaction and the resulting allocation attempt ++ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL ++ * allocations through requiring that the full LRU list has been scanned ++ * first, by assuming that zero delta of sc->nr_scanned means full LRU ++ * scan, but that approximation was wrong, and there were corner cases + * where always a non-zero amount of pages were scanned. + */ + if (!nr_reclaimed) +@@ -3201,109 +6074,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) + unsigned long nr_reclaimed, nr_scanned; + struct lruvec *target_lruvec; + bool reclaimable = false; +- unsigned long file; + + target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); + + again: +- /* +- * Flush the memory cgroup stats, so that we read accurate per-memcg +- * lruvec stats for heuristics. +- */ +- mem_cgroup_flush_stats(); +- + memset(&sc->nr, 0, sizeof(sc->nr)); + + nr_reclaimed = sc->nr_reclaimed; + nr_scanned = sc->nr_scanned; + +- /* +- * Determine the scan balance between anon and file LRUs. +- */ +- spin_lock_irq(&target_lruvec->lru_lock); +- sc->anon_cost = target_lruvec->anon_cost; +- sc->file_cost = target_lruvec->file_cost; +- spin_unlock_irq(&target_lruvec->lru_lock); +- +- /* +- * Target desirable inactive:active list ratios for the anon +- * and file LRU lists. +- */ +- if (!sc->force_deactivate) { +- unsigned long refaults; +- +- refaults = lruvec_page_state(target_lruvec, +- WORKINGSET_ACTIVATE_ANON); +- if (refaults != target_lruvec->refaults[0] || +- inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) +- sc->may_deactivate |= DEACTIVATE_ANON; +- else +- sc->may_deactivate &= ~DEACTIVATE_ANON; +- +- /* +- * When refaults are being observed, it means a new +- * workingset is being established. Deactivate to get +- * rid of any stale active pages quickly. +- */ +- refaults = lruvec_page_state(target_lruvec, +- WORKINGSET_ACTIVATE_FILE); +- if (refaults != target_lruvec->refaults[1] || +- inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) +- sc->may_deactivate |= DEACTIVATE_FILE; +- else +- sc->may_deactivate &= ~DEACTIVATE_FILE; +- } else +- sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; +- +- /* +- * If we have plenty of inactive file pages that aren't +- * thrashing, try to reclaim those first before touching +- * anonymous pages. +- */ +- file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); +- if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) +- sc->cache_trim_mode = 1; +- else +- sc->cache_trim_mode = 0; +- +- /* +- * Prevent the reclaimer from falling into the cache trap: as +- * cache pages start out inactive, every cache fault will tip +- * the scan balance towards the file LRU. And as the file LRU +- * shrinks, so does the window for rotation from references. +- * This means we have a runaway feedback loop where a tiny +- * thrashing file LRU becomes infinitely more attractive than +- * anon pages. Try to detect this based on file LRU size. +- */ +- if (!cgroup_reclaim(sc)) { +- unsigned long total_high_wmark = 0; +- unsigned long free, anon; +- int z; +- +- free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); +- file = node_page_state(pgdat, NR_ACTIVE_FILE) + +- node_page_state(pgdat, NR_INACTIVE_FILE); +- +- for (z = 0; z < MAX_NR_ZONES; z++) { +- struct zone *zone = &pgdat->node_zones[z]; +- if (!managed_zone(zone)) +- continue; +- +- total_high_wmark += high_wmark_pages(zone); +- } +- +- /* +- * Consider anon: if that's low too, this isn't a +- * runaway file reclaim problem, but rather just +- * extreme pressure. Reclaim as per usual then. +- */ +- anon = node_page_state(pgdat, NR_INACTIVE_ANON); +- +- sc->file_is_tiny = +- file + free <= total_high_wmark && +- !(sc->may_deactivate & DEACTIVATE_ANON) && +- anon >> sc->priority; +- } ++ prepare_scan_count(pgdat, sc); + + shrink_node_memcgs(pgdat, sc); + +@@ -3561,11 +6341,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) + struct lruvec *target_lruvec; + unsigned long refaults; + ++ if (lru_gen_enabled()) ++ return; ++ + target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); + refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); +- target_lruvec->refaults[0] = refaults; ++ target_lruvec->refaults[WORKINGSET_ANON] = refaults; + refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); +- target_lruvec->refaults[1] = refaults; ++ target_lruvec->refaults[WORKINGSET_FILE] = refaults; + } + + /* +@@ -3927,12 +6710,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + } + #endif + +-static void age_active_anon(struct pglist_data *pgdat, +- struct scan_control *sc) ++static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) + { + struct mem_cgroup *memcg; + struct lruvec *lruvec; + ++ if (lru_gen_enabled()) { ++ lru_gen_age_node(pgdat, sc); ++ return; ++ } ++ + if (!can_age_anon_pages(pgdat, sc)) + return; + +@@ -4252,12 +7039,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) + sc.may_swap = !nr_boost_reclaim; + + /* +- * Do some background aging of the anon list, to give +- * pages a chance to be referenced before reclaiming. All +- * pages are rotated regardless of classzone as this is +- * about consistent aging. ++ * Do some background aging, to give pages a chance to be ++ * referenced before reclaiming. All pages are rotated ++ * regardless of classzone as this is about consistent aging. + */ +- age_active_anon(pgdat, &sc); ++ kswapd_age_node(pgdat, &sc); + + /* + * If we're getting trouble reclaiming, start doing writepage +diff --git a/mm/workingset.c b/mm/workingset.c +index a5e84862fc86..ae7e984b23c6 100644 +--- a/mm/workingset.c ++++ b/mm/workingset.c +@@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; + static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, + bool workingset) + { +- eviction >>= bucket_order; + eviction &= EVICTION_MASK; + eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; + eviction = (eviction << NODES_SHIFT) | pgdat->node_id; +@@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, + + *memcgidp = memcgid; + *pgdat = NODE_DATA(nid); +- *evictionp = entry << bucket_order; ++ *evictionp = entry; + *workingsetp = workingset; + } + ++#ifdef CONFIG_LRU_GEN ++ ++static void *lru_gen_eviction(struct folio *folio) ++{ ++ int hist; ++ unsigned long token; ++ unsigned long min_seq; ++ struct lruvec *lruvec; ++ struct lru_gen_struct *lrugen; ++ int type = folio_is_file_lru(folio); ++ int delta = folio_nr_pages(folio); ++ int refs = folio_lru_refs(folio); ++ int tier = lru_tier_from_refs(refs); ++ struct mem_cgroup *memcg = folio_memcg(folio); ++ struct pglist_data *pgdat = folio_pgdat(folio); ++ ++ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); ++ ++ lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ lrugen = &lruvec->lrugen; ++ min_seq = READ_ONCE(lrugen->min_seq[type]); ++ token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); ++ ++ hist = lru_hist_from_seq(min_seq); ++ atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); ++ ++ return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); ++} ++ ++static void lru_gen_refault(struct folio *folio, void *shadow) ++{ ++ int hist, tier, refs; ++ int memcg_id; ++ bool workingset; ++ unsigned long token; ++ unsigned long min_seq; ++ struct lruvec *lruvec; ++ struct lru_gen_struct *lrugen; ++ struct mem_cgroup *memcg; ++ struct pglist_data *pgdat; ++ int type = folio_is_file_lru(folio); ++ int delta = folio_nr_pages(folio); ++ ++ unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); ++ ++ if (pgdat != folio_pgdat(folio)) ++ return; ++ ++ rcu_read_lock(); ++ ++ memcg = folio_memcg_rcu(folio); ++ if (memcg_id != mem_cgroup_id(memcg)) ++ goto unlock; ++ ++ lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ lrugen = &lruvec->lrugen; ++ ++ min_seq = READ_ONCE(lrugen->min_seq[type]); ++ if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) ++ goto unlock; ++ ++ hist = lru_hist_from_seq(min_seq); ++ /* see the comment in folio_lru_refs() */ ++ refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; ++ tier = lru_tier_from_refs(refs); ++ ++ atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); ++ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); ++ ++ /* ++ * Count the following two cases as stalls: ++ * 1. For pages accessed through page tables, hotter pages pushed out ++ * hot pages which refaulted immediately. ++ * 2. For pages accessed multiple times through file descriptors, ++ * numbers of accesses might have been out of the range. ++ */ ++ if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { ++ folio_set_workingset(folio); ++ mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); ++ } ++unlock: ++ rcu_read_unlock(); ++} ++ ++#else /* !CONFIG_LRU_GEN */ ++ ++static void *lru_gen_eviction(struct folio *folio) ++{ ++ return NULL; ++} ++ ++static void lru_gen_refault(struct folio *folio, void *shadow) ++{ ++} ++ ++#endif /* CONFIG_LRU_GEN */ ++ + /** + * workingset_age_nonresident - age non-resident entries as LRU ages + * @lruvec: the lruvec that was aged +@@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) + VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + ++ if (lru_gen_enabled()) ++ return lru_gen_eviction(folio); ++ + lruvec = mem_cgroup_lruvec(target_memcg, pgdat); + /* XXX: target_memcg can be NULL, go through lruvec */ + memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); + eviction = atomic_long_read(&lruvec->nonresident_age); ++ eviction >>= bucket_order; + workingset_age_nonresident(lruvec, folio_nr_pages(folio)); + return pack_shadow(memcgid, pgdat, eviction, + folio_test_workingset(folio)); +@@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) + int memcgid; + long nr; + ++ if (lru_gen_enabled()) { ++ lru_gen_refault(folio, shadow); ++ return; ++ } ++ + unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); ++ eviction <<= bucket_order; + + rcu_read_lock(); + /* +-- +2.38.0 + +From 386f84c5b5b2f5f427c8776d5735cf56f31e0618 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Sat, 8 Oct 2022 15:03:49 +0200 +Subject: [PATCH 06/17] Introducing the Maple Tree + +The maple tree is an RCU-safe range based B-tree designed to use modern +processor cache efficiently. There are a number of places in the kernel +that a non-overlapping range-based tree would be beneficial, especially +one with a simple interface. If you use an rbtree with other data +structures to improve performance or an interval tree to track +non-overlapping ranges, then this is for you. + +The tree has a branching factor of 10 for non-leaf nodes and 16 for leaf +nodes. With the increased branching factor, it is significantly shorter +than the rbtree so it has fewer cache misses. The removal of the linked +list between subsequent entries also reduces the cache misses and the need +to pull in the previous and next VMA during many tree alterations. + +The first user that is covered in this patch set is the vm_area_struct, +where three data structures are replaced by the maple tree: the augmented +rbtree, the vma cache, and the linked list of VMAs in the mm_struct. The +long term goal is to reduce or remove the mmap_lock contention. + +The plan is to get to the point where we use the maple tree in RCU mode. +Readers will not block for writers. A single write operation will be +allowed at a time. A reader re-walks if stale data is encountered. VMAs +would be RCU enabled and this mode would be entered once multiple tasks +are using the mm_struct. + +Davidlor said + +: Yes I like the maple tree, and at this stage I don't think we can ask for +: more from this series wrt the MM - albeit there seems to still be some +: folks reporting breakage. Fundamentally I see Liam's work to (re)move +: complexity out of the MM (not to say that the actual maple tree is not +: complex) by consolidating the three complimentary data structures very +: much worth it considering performance does not take a hit. This was very +: much a turn off with the range locking approach, which worst case scenario +: incurred in prohibitive overhead. Also as Liam and Matthew have +: mentioned, RCU opens up a lot of nice performance opportunities, and in +: addition academia[1] has shown outstanding scalability of address spaces +: with the foundation of replacing the locked rbtree with RCU aware trees. + +A similar work has been discovered in the academic press + + https://pdos.csail.mit.edu/papers/rcuvm:asplos12.pdf + +Sheer coincidence. We designed our tree with the intention of solving the +hardest problem first. Upon settling on a b-tree variant and a rough +outline, we researched ranged based b-trees and RCU b-trees and did find +that article. So it was nice to find reassurances that we were on the +right path, but our design choice of using ranges made that paper unusable +for us. + +This patch (of 70): + +The maple tree is an RCU-safe range based B-tree designed to use modern +processor cache efficiently. There are a number of places in the kernel +that a non-overlapping range-based tree would be beneficial, especially +one with a simple interface. If you use an rbtree with other data +structures to improve performance or an interval tree to track +non-overlapping ranges, then this is for you. + +The tree has a branching factor of 10 for non-leaf nodes and 16 for leaf +nodes. With the increased branching factor, it is significantly shorter +than the rbtree so it has fewer cache misses. The removal of the linked +list between subsequent entries also reduces the cache misses and the need +to pull in the previous and next VMA during many tree alterations. + +The first user that is covered in this patch set is the vm_area_struct, +where three data structures are replaced by the maple tree: the augmented +rbtree, the vma cache, and the linked list of VMAs in the mm_struct. The +long term goal is to reduce or remove the mmap_lock contention. + +The plan is to get to the point where we use the maple tree in RCU mode. +Readers will not block for writers. A single write operation will be +allowed at a time. A reader re-walks if stale data is encountered. VMAs +would be RCU enabled and this mode would be entered once multiple tasks +are using the mm_struct. + +There is additional BUG_ON() calls added within the tree, most of which +are in debug code. These will be replaced with a WARN_ON() call in the +future. There is also additional BUG_ON() calls within the code which +will also be reduced in number at a later date. These exist to catch +things such as out-of-range accesses which would crash anyways. + +Signed-off-by: Peter Jung +--- + Documentation/core-api/index.rst | 1 + + Documentation/core-api/maple_tree.rst | 217 + + MAINTAINERS | 12 + + arch/arm64/kernel/elfcore.c | 16 +- + arch/arm64/kernel/vdso.c | 3 +- + arch/parisc/kernel/cache.c | 9 +- + arch/powerpc/kernel/vdso.c | 6 +- + arch/powerpc/mm/book3s32/tlb.c | 11 +- + arch/powerpc/mm/book3s64/subpage_prot.c | 13 +- + arch/riscv/kernel/vdso.c | 3 +- + arch/s390/kernel/vdso.c | 3 +- + arch/s390/mm/gmap.c | 6 +- + arch/um/kernel/tlb.c | 14 +- + arch/x86/entry/vdso/vma.c | 9 +- + arch/x86/kernel/tboot.c | 2 +- + arch/xtensa/kernel/syscall.c | 18 +- + drivers/firmware/efi/efi.c | 2 +- + drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 14 +- + drivers/misc/cxl/fault.c | 45 +- + drivers/tee/optee/call.c | 18 +- + drivers/xen/privcmd.c | 2 +- + fs/coredump.c | 34 +- + fs/exec.c | 12 +- + fs/proc/base.c | 5 +- + fs/proc/internal.h | 2 +- + fs/proc/task_mmu.c | 74 +- + fs/proc/task_nommu.c | 45 +- + fs/userfaultfd.c | 62 +- + include/linux/maple_tree.h | 685 + + include/linux/mm.h | 78 +- + include/linux/mm_types.h | 43 +- + include/linux/mm_types_task.h | 12 - + include/linux/sched.h | 1 - + include/linux/userfaultfd_k.h | 7 +- + include/linux/vm_event_item.h | 4 - + include/linux/vmacache.h | 28 - + include/linux/vmstat.h | 6 - + include/trace/events/maple_tree.h | 123 + + include/trace/events/mmap.h | 73 + + init/main.c | 2 + + ipc/shm.c | 21 +- + kernel/acct.c | 11 +- + kernel/bpf/task_iter.c | 10 +- + kernel/debug/debug_core.c | 12 - + kernel/events/core.c | 3 +- + kernel/events/uprobes.c | 9 +- + kernel/fork.c | 62 +- + kernel/sched/fair.c | 10 +- + lib/Kconfig.debug | 17 +- + lib/Makefile | 2 +- + lib/maple_tree.c | 7130 +++ + lib/test_maple_tree.c | 38307 ++++++++++++++++ + mm/Makefile | 2 +- + mm/damon/vaddr-test.h | 36 +- + mm/damon/vaddr.c | 53 +- + mm/debug.c | 14 +- + mm/gup.c | 7 +- + mm/huge_memory.c | 4 +- + mm/init-mm.c | 4 +- + mm/internal.h | 8 +- + mm/khugepaged.c | 11 +- + mm/ksm.c | 18 +- + mm/madvise.c | 2 +- + mm/memcontrol.c | 6 +- + mm/memory.c | 33 +- + mm/mempolicy.c | 56 +- + mm/mlock.c | 35 +- + mm/mmap.c | 2154 +- + mm/mprotect.c | 8 +- + mm/mremap.c | 22 +- + mm/msync.c | 2 +- + mm/nommu.c | 260 +- + mm/oom_kill.c | 3 +- + mm/pagewalk.c | 2 +- + mm/swapfile.c | 4 +- + mm/util.c | 32 - + mm/vmacache.c | 117 - + mm/vmscan.c | 15 +- + mm/vmstat.c | 4 - + tools/include/linux/slab.h | 4 + + tools/testing/radix-tree/.gitignore | 2 + + tools/testing/radix-tree/Makefile | 9 +- + tools/testing/radix-tree/generated/autoconf.h | 1 + + tools/testing/radix-tree/linux.c | 160 +- + tools/testing/radix-tree/linux/kernel.h | 1 + + tools/testing/radix-tree/linux/lockdep.h | 2 + + tools/testing/radix-tree/linux/maple_tree.h | 7 + + tools/testing/radix-tree/maple.c | 59 + + .../radix-tree/trace/events/maple_tree.h | 5 + + 89 files changed, 48581 insertions(+), 1895 deletions(-) + create mode 100644 Documentation/core-api/maple_tree.rst + create mode 100644 include/linux/maple_tree.h + delete mode 100644 include/linux/vmacache.h + create mode 100644 include/trace/events/maple_tree.h + create mode 100644 lib/maple_tree.c + create mode 100644 lib/test_maple_tree.c + delete mode 100644 mm/vmacache.c + create mode 100644 tools/testing/radix-tree/linux/maple_tree.h + create mode 100644 tools/testing/radix-tree/maple.c + create mode 100644 tools/testing/radix-tree/trace/events/maple_tree.h + +diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst +index dc95df462eea..1da6a4fac664 100644 +--- a/Documentation/core-api/index.rst ++++ b/Documentation/core-api/index.rst +@@ -36,6 +36,7 @@ Library functionality that is used throughout the kernel. + kref + assoc_array + xarray ++ maple_tree + idr + circular-buffers + rbtree +diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst +new file mode 100644 +index 000000000000..45defcf15da7 +--- /dev/null ++++ b/Documentation/core-api/maple_tree.rst +@@ -0,0 +1,217 @@ ++.. SPDX-License-Identifier: GPL-2.0+ ++ ++ ++========== ++Maple Tree ++========== ++ ++:Author: Liam R. Howlett ++ ++Overview ++======== ++ ++The Maple Tree is a B-Tree data type which is optimized for storing ++non-overlapping ranges, including ranges of size 1. The tree was designed to ++be simple to use and does not require a user written search method. It ++supports iterating over a range of entries and going to the previous or next ++entry in a cache-efficient manner. The tree can also be put into an RCU-safe ++mode of operation which allows reading and writing concurrently. Writers must ++synchronize on a lock, which can be the default spinlock, or the user can set ++the lock to an external lock of a different type. ++ ++The Maple Tree maintains a small memory footprint and was designed to use ++modern processor cache efficiently. The majority of the users will be able to ++use the normal API. An :ref:`maple-tree-advanced-api` exists for more complex ++scenarios. The most important usage of the Maple Tree is the tracking of the ++virtual memory areas. ++ ++The Maple Tree can store values between ``0`` and ``ULONG_MAX``. The Maple ++Tree reserves values with the bottom two bits set to '10' which are below 4096 ++(ie 2, 6, 10 .. 4094) for internal use. If the entries may use reserved ++entries then the users can convert the entries using xa_mk_value() and convert ++them back by calling xa_to_value(). If the user needs to use a reserved ++value, then the user can convert the value when using the ++:ref:`maple-tree-advanced-api`, but are blocked by the normal API. ++ ++The Maple Tree can also be configured to support searching for a gap of a given ++size (or larger). ++ ++Pre-allocating of nodes is also supported using the ++:ref:`maple-tree-advanced-api`. This is useful for users who must guarantee a ++successful store operation within a given ++code segment when allocating cannot be done. Allocations of nodes are ++relatively small at around 256 bytes. ++ ++.. _maple-tree-normal-api: ++ ++Normal API ++========== ++ ++Start by initialising a maple tree, either with DEFINE_MTREE() for statically ++allocated maple trees or mt_init() for dynamically allocated ones. A ++freshly-initialised maple tree contains a ``NULL`` pointer for the range ``0`` ++- ``ULONG_MAX``. There are currently two types of maple trees supported: the ++allocation tree and the regular tree. The regular tree has a higher branching ++factor for internal nodes. The allocation tree has a lower branching factor ++but allows the user to search for a gap of a given size or larger from either ++``0`` upwards or ``ULONG_MAX`` down. An allocation tree can be used by ++passing in the ``MT_FLAGS_ALLOC_RANGE`` flag when initialising the tree. ++ ++You can then set entries using mtree_store() or mtree_store_range(). ++mtree_store() will overwrite any entry with the new entry and return 0 on ++success or an error code otherwise. mtree_store_range() works in the same way ++but takes a range. mtree_load() is used to retrieve the entry stored at a ++given index. You can use mtree_erase() to erase an entire range by only ++knowing one value within that range, or mtree_store() call with an entry of ++NULL may be used to partially erase a range or many ranges at once. ++ ++If you want to only store a new entry to a range (or index) if that range is ++currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which ++return -EEXIST if the range is not empty. ++ ++You can search for an entry from an index upwards by using mt_find(). ++ ++You can walk each entry within a range by calling mt_for_each(). You must ++provide a temporary variable to store a cursor. If you want to walk each ++element of the tree then ``0`` and ``ULONG_MAX`` may be used as the range. If ++the caller is going to hold the lock for the duration of the walk then it is ++worth looking at the mas_for_each() API in the :ref:`maple-tree-advanced-api` ++section. ++ ++Sometimes it is necessary to ensure the next call to store to a maple tree does ++not allocate memory, please see :ref:`maple-tree-advanced-api` for this use case. ++ ++Finally, you can remove all entries from a maple tree by calling ++mtree_destroy(). If the maple tree entries are pointers, you may wish to free ++the entries first. ++ ++Allocating Nodes ++---------------- ++ ++The allocations are handled by the internal tree code. See ++:ref:`maple-tree-advanced-alloc` for other options. ++ ++Locking ++------- ++ ++You do not have to worry about locking. See :ref:`maple-tree-advanced-locks` ++for other options. ++ ++The Maple Tree uses RCU and an internal spinlock to synchronise access: ++ ++Takes RCU read lock: ++ * mtree_load() ++ * mt_find() ++ * mt_for_each() ++ * mt_next() ++ * mt_prev() ++ ++Takes ma_lock internally: ++ * mtree_store() ++ * mtree_store_range() ++ * mtree_insert() ++ * mtree_insert_range() ++ * mtree_erase() ++ * mtree_destroy() ++ * mt_set_in_rcu() ++ * mt_clear_in_rcu() ++ ++If you want to take advantage of the internal lock to protect the data ++structures that you are storing in the Maple Tree, you can call mtree_lock() ++before calling mtree_load(), then take a reference count on the object you ++have found before calling mtree_unlock(). This will prevent stores from ++removing the object from the tree between looking up the object and ++incrementing the refcount. You can also use RCU to avoid dereferencing ++freed memory, but an explanation of that is beyond the scope of this ++document. ++ ++.. _maple-tree-advanced-api: ++ ++Advanced API ++============ ++ ++The advanced API offers more flexibility and better performance at the ++cost of an interface which can be harder to use and has fewer safeguards. ++You must take care of your own locking while using the advanced API. ++You can use the ma_lock, RCU or an external lock for protection. ++You can mix advanced and normal operations on the same array, as long ++as the locking is compatible. The :ref:`maple-tree-normal-api` is implemented ++in terms of the advanced API. ++ ++The advanced API is based around the ma_state, this is where the 'mas' ++prefix originates. The ma_state struct keeps track of tree operations to make ++life easier for both internal and external tree users. ++ ++Initialising the maple tree is the same as in the :ref:`maple-tree-normal-api`. ++Please see above. ++ ++The maple state keeps track of the range start and end in mas->index and ++mas->last, respectively. ++ ++mas_walk() will walk the tree to the location of mas->index and set the ++mas->index and mas->last according to the range for the entry. ++ ++You can set entries using mas_store(). mas_store() will overwrite any entry ++with the new entry and return the first existing entry that is overwritten. ++The range is passed in as members of the maple state: index and last. ++ ++You can use mas_erase() to erase an entire range by setting index and ++last of the maple state to the desired range to erase. This will erase ++the first range that is found in that range, set the maple state index ++and last as the range that was erased and return the entry that existed ++at that location. ++ ++You can walk each entry within a range by using mas_for_each(). If you want ++to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as ++the range. If the lock needs to be periodically dropped, see the locking ++section mas_pause(). ++ ++Using a maple state allows mas_next() and mas_prev() to function as if the ++tree was a linked list. With such a high branching factor the amortized ++performance penalty is outweighed by cache optimization. mas_next() will ++return the next entry which occurs after the entry at index. mas_prev() ++will return the previous entry which occurs before the entry at index. ++ ++mas_find() will find the first entry which exists at or above index on ++the first call, and the next entry from every subsequent calls. ++ ++mas_find_rev() will find the fist entry which exists at or below the last on ++the first call, and the previous entry from every subsequent calls. ++ ++If the user needs to yield the lock during an operation, then the maple state ++must be paused using mas_pause(). ++ ++There are a few extra interfaces provided when using an allocation tree. ++If you wish to search for a gap within a range, then mas_empty_area() ++or mas_empty_area_rev() can be used. mas_empty_area() searches for a gap ++starting at the lowest index given up to the maximum of the range. ++mas_empty_area_rev() searches for a gap starting at the highest index given ++and continues downward to the lower bound of the range. ++ ++.. _maple-tree-advanced-alloc: ++ ++Advanced Allocating Nodes ++------------------------- ++ ++Allocations are usually handled internally to the tree, however if allocations ++need to occur before a write occurs then calling mas_expected_entries() will ++allocate the worst-case number of needed nodes to insert the provided number of ++ranges. This also causes the tree to enter mass insertion mode. Once ++insertions are complete calling mas_destroy() on the maple state will free the ++unused allocations. ++ ++.. _maple-tree-advanced-locks: ++ ++Advanced Locking ++---------------- ++ ++The maple tree uses a spinlock by default, but external locks can be used for ++tree updates as well. To use an external lock, the tree must be initialized ++with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the ++MTREE_INIT_EXT() #define, which takes an external lock as an argument. ++ ++Functions and structures ++======================== ++ ++.. kernel-doc:: include/linux/maple_tree.h ++.. kernel-doc:: lib/maple_tree.c +diff --git a/MAINTAINERS b/MAINTAINERS +index 594e31ec15cb..9a5a422817af 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -12093,6 +12093,18 @@ L: linux-man@vger.kernel.org + S: Maintained + W: http://www.kernel.org/doc/man-pages + ++MAPLE TREE ++M: Liam R. Howlett ++L: linux-mm@kvack.org ++S: Supported ++F: Documentation/core-api/maple_tree.rst ++F: include/linux/maple_tree.h ++F: include/trace/events/maple_tree.h ++F: lib/maple_tree.c ++F: lib/test_maple_tree.c ++F: tools/testing/radix-tree/linux/maple_tree.h ++F: tools/testing/radix-tree/maple.c ++ + MARDUK (CREATOR CI40) DEVICE TREE SUPPORT + M: Rahul Bedarkar + L: linux-mips@vger.kernel.org +diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c +index 98d67444a5b6..27ef7ad3ffd2 100644 +--- a/arch/arm64/kernel/elfcore.c ++++ b/arch/arm64/kernel/elfcore.c +@@ -8,9 +8,9 @@ + #include + #include + +-#define for_each_mte_vma(tsk, vma) \ ++#define for_each_mte_vma(vmi, vma) \ + if (system_supports_mte()) \ +- for (vma = tsk->mm->mmap; vma; vma = vma->vm_next) \ ++ for_each_vma(vmi, vma) \ + if (vma->vm_flags & VM_MTE) + + static unsigned long mte_vma_tag_dump_size(struct vm_area_struct *vma) +@@ -81,8 +81,9 @@ Elf_Half elf_core_extra_phdrs(void) + { + struct vm_area_struct *vma; + int vma_count = 0; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) ++ for_each_mte_vma(vmi, vma) + vma_count++; + + return vma_count; +@@ -91,8 +92,9 @@ Elf_Half elf_core_extra_phdrs(void) + int elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) { ++ for_each_mte_vma(vmi, vma) { + struct elf_phdr phdr; + + phdr.p_type = PT_AARCH64_MEMTAG_MTE; +@@ -116,8 +118,9 @@ size_t elf_core_extra_data_size(void) + { + struct vm_area_struct *vma; + size_t data_size = 0; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) ++ for_each_mte_vma(vmi, vma) + data_size += mte_vma_tag_dump_size(vma); + + return data_size; +@@ -126,8 +129,9 @@ size_t elf_core_extra_data_size(void) + int elf_core_write_extra_data(struct coredump_params *cprm) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) { ++ for_each_mte_vma(vmi, vma) { + if (vma->vm_flags & VM_DONTDUMP) + continue; + +diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c +index a61fc4f989b3..a8388af62b99 100644 +--- a/arch/arm64/kernel/vdso.c ++++ b/arch/arm64/kernel/vdso.c +@@ -136,10 +136,11 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) +diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c +index 3feb7694e0ca..1d3b8bc8a623 100644 +--- a/arch/parisc/kernel/cache.c ++++ b/arch/parisc/kernel/cache.c +@@ -657,15 +657,20 @@ static inline unsigned long mm_total_size(struct mm_struct *mm) + { + struct vm_area_struct *vma; + unsigned long usize = 0; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma && usize < parisc_cache_flush_threshold; vma = vma->vm_next) ++ for_each_vma(vmi, vma) { ++ if (usize >= parisc_cache_flush_threshold) ++ break; + usize += vma->vm_end - vma->vm_start; ++ } + return usize; + } + + void flush_cache_mm(struct mm_struct *mm) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + /* + * Flushing the whole cache on each cpu takes forever on +@@ -685,7 +690,7 @@ void flush_cache_mm(struct mm_struct *mm) + } + + /* Flush mm */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) ++ for_each_vma(vmi, vma) + flush_cache_pages(vma, vma->vm_start, vma->vm_end); + } + +diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c +index 0da287544054..94a8fa5017c3 100644 +--- a/arch/powerpc/kernel/vdso.c ++++ b/arch/powerpc/kernel/vdso.c +@@ -113,18 +113,18 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page) + int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + + mmap_read_lock(mm); +- +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, &vvar_spec)) + zap_page_range(vma, vma->vm_start, size); + } +- + mmap_read_unlock(mm); ++ + return 0; + } + +diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c +index 19f0ef950d77..9ad6b56bfec9 100644 +--- a/arch/powerpc/mm/book3s32/tlb.c ++++ b/arch/powerpc/mm/book3s32/tlb.c +@@ -81,14 +81,15 @@ EXPORT_SYMBOL(hash__flush_range); + void hash__flush_tlb_mm(struct mm_struct *mm) + { + struct vm_area_struct *mp; ++ VMA_ITERATOR(vmi, mm, 0); + + /* +- * It is safe to go down the mm's list of vmas when called +- * from dup_mmap, holding mmap_lock. It would also be safe from +- * unmap_region or exit_mmap, but not from vmtruncate on SMP - +- * but it seems dup_mmap is the only SMP case which gets here. ++ * It is safe to iterate the vmas when called from dup_mmap, ++ * holding mmap_lock. It would also be safe from unmap_region ++ * or exit_mmap, but not from vmtruncate on SMP - but it seems ++ * dup_mmap is the only SMP case which gets here. + */ +- for (mp = mm->mmap; mp != NULL; mp = mp->vm_next) ++ for_each_vma(vmi, mp) + hash__flush_range(mp->vm_mm, mp->vm_start, mp->vm_end); + } + EXPORT_SYMBOL(hash__flush_tlb_mm); +diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c +index 60c6ea16a972..d73b3b4176e8 100644 +--- a/arch/powerpc/mm/book3s64/subpage_prot.c ++++ b/arch/powerpc/mm/book3s64/subpage_prot.c +@@ -149,24 +149,15 @@ static void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, + unsigned long len) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, addr); + + /* + * We don't try too hard, we just mark all the vma in that range + * VM_NOHUGEPAGE and split them. + */ +- vma = find_vma(mm, addr); +- /* +- * If the range is in unmapped range, just return +- */ +- if (vma && ((addr + len) <= vma->vm_start)) +- return; +- +- while (vma) { +- if (vma->vm_start >= (addr + len)) +- break; ++ for_each_vma_range(vmi, vma, addr + len) { + vma->vm_flags |= VM_NOHUGEPAGE; + walk_page_vma(vma, &subpage_walk_ops, NULL); +- vma = vma->vm_next; + } + } + #else +diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c +index 69b05b6c181b..692e7ae3dcb8 100644 +--- a/arch/riscv/kernel/vdso.c ++++ b/arch/riscv/kernel/vdso.c +@@ -114,11 +114,12 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + struct __vdso_info *vdso_info = mm->context.vdso_info; + + mmap_read_lock(mm); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, vdso_info->dm)) +diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c +index 5075cde77b29..535099f2736d 100644 +--- a/arch/s390/kernel/vdso.c ++++ b/arch/s390/kernel/vdso.c +@@ -69,10 +69,11 @@ static struct page *find_timens_vvar_page(struct vm_area_struct *vma) + int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (!vma_is_special_mapping(vma, &vvar_mapping)) +diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c +index 62758cb5872f..02d15c8dc92e 100644 +--- a/arch/s390/mm/gmap.c ++++ b/arch/s390/mm/gmap.c +@@ -2515,8 +2515,9 @@ static const struct mm_walk_ops thp_split_walk_ops = { + static inline void thp_split_mm(struct mm_struct *mm) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + vma->vm_flags &= ~VM_HUGEPAGE; + vma->vm_flags |= VM_NOHUGEPAGE; + walk_page_vma(vma, &thp_split_walk_ops, NULL); +@@ -2584,8 +2585,9 @@ int gmap_mark_unmergeable(void) + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + int ret; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, + MADV_UNMERGEABLE, &vma->vm_flags); + if (ret) +diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c +index bc38f79ca3a3..ad449173a1a1 100644 +--- a/arch/um/kernel/tlb.c ++++ b/arch/um/kernel/tlb.c +@@ -584,21 +584,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + + void flush_tlb_mm(struct mm_struct *mm) + { +- struct vm_area_struct *vma = mm->mmap; ++ struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- while (vma != NULL) { ++ for_each_vma(vmi, vma) + fix_range(mm, vma->vm_start, vma->vm_end, 0); +- vma = vma->vm_next; +- } + } + + void force_flush_all(void) + { + struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma = mm->mmap; ++ struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- while (vma != NULL) { ++ for_each_vma(vmi, vma) + fix_range(mm, vma->vm_start, vma->vm_end, 1); +- vma = vma->vm_next; +- } + } +diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c +index 1000d457c332..6292b960037b 100644 +--- a/arch/x86/entry/vdso/vma.c ++++ b/arch/x86/entry/vdso/vma.c +@@ -127,17 +127,17 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); +- +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, &vvar_mapping)) + zap_page_range(vma, vma->vm_start, size); + } +- + mmap_read_unlock(mm); ++ + return 0; + } + #else +@@ -354,6 +354,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) + { + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_write_lock(mm); + /* +@@ -363,7 +364,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) + * We could search vma near context.vdso, but it's a slowpath, + * so let's explicitly check all VMAs to be completely sure. + */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (vma_is_special_mapping(vma, &vdso_mapping) || + vma_is_special_mapping(vma, &vvar_mapping)) { + mmap_write_unlock(mm); +diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c +index 3bacd935f840..4c1bcb6053fc 100644 +--- a/arch/x86/kernel/tboot.c ++++ b/arch/x86/kernel/tboot.c +@@ -95,7 +95,7 @@ void __init tboot_probe(void) + + static pgd_t *tboot_pg_dir; + static struct mm_struct tboot_mm = { +- .mm_rb = RB_ROOT, ++ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, tboot_mm.mmap_lock), + .pgd = swapper_pg_dir, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), +diff --git a/arch/xtensa/kernel/syscall.c b/arch/xtensa/kernel/syscall.c +index 201356faa7e6..b3c2450d6f23 100644 +--- a/arch/xtensa/kernel/syscall.c ++++ b/arch/xtensa/kernel/syscall.c +@@ -58,6 +58,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) + { + struct vm_area_struct *vmm; ++ struct vma_iterator vmi; + + if (flags & MAP_FIXED) { + /* We do not accept a shared mapping if it would violate +@@ -79,15 +80,20 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, + else + addr = PAGE_ALIGN(addr); + +- for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { +- /* At this point: (!vmm || addr < vmm->vm_end). */ +- if (TASK_SIZE - len < addr) +- return -ENOMEM; +- if (!vmm || addr + len <= vm_start_gap(vmm)) +- return addr; ++ vma_iter_init(&vmi, current->mm, addr); ++ for_each_vma(vmi, vmm) { ++ /* At this point: (addr < vmm->vm_end). */ ++ if (addr + len <= vm_start_gap(vmm)) ++ break; ++ + addr = vmm->vm_end; + if (flags & MAP_SHARED) + addr = COLOUR_ALIGN(addr, pgoff); + } ++ ++ if (TASK_SIZE - len < addr) ++ return -ENOMEM; ++ ++ return addr; + } + #endif +diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c +index e4080ad96089..042a3ef4db1c 100644 +--- a/drivers/firmware/efi/efi.c ++++ b/drivers/firmware/efi/efi.c +@@ -57,7 +57,7 @@ static unsigned long __initdata mem_reserve = EFI_INVALID_TABLE_ADDR; + static unsigned long __initdata rt_prop = EFI_INVALID_TABLE_ADDR; + + struct mm_struct efi_mm = { +- .mm_rb = RB_ROOT, ++ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, efi_mm.mmap_lock), + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), + .write_protect_seq = SEQCNT_ZERO(efi_mm.write_protect_seq), +diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +index 8423df021b71..d4398948f016 100644 +--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c ++++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +@@ -426,12 +426,11 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { + static int + probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) + { +- const unsigned long end = addr + len; ++ VMA_ITERATOR(vmi, mm, addr); + struct vm_area_struct *vma; +- int ret = -EFAULT; + + mmap_read_lock(mm); +- for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { ++ for_each_vma_range(vmi, vma, addr + len) { + /* Check for holes, note that we also update the addr below */ + if (vma->vm_start > addr) + break; +@@ -439,16 +438,13 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) + if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + break; + +- if (vma->vm_end >= end) { +- ret = 0; +- break; +- } +- + addr = vma->vm_end; + } + mmap_read_unlock(mm); + +- return ret; ++ if (vma) ++ return -EFAULT; ++ return 0; + } + + /* +diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c +index 60c829113299..2c64f55cf01f 100644 +--- a/drivers/misc/cxl/fault.c ++++ b/drivers/misc/cxl/fault.c +@@ -280,22 +280,6 @@ void cxl_handle_fault(struct work_struct *fault_work) + mmput(mm); + } + +-static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) +-{ +- struct mm_struct *mm; +- +- mm = get_mem_context(ctx); +- if (mm == NULL) { +- pr_devel("cxl_prefault_one unable to get mm %i\n", +- pid_nr(ctx->pid)); +- return; +- } +- +- cxl_fault_segment(ctx, mm, ea); +- +- mmput(mm); +-} +- + static u64 next_segment(u64 ea, u64 vsid) + { + if (vsid & SLB_VSID_B_1T) +@@ -306,23 +290,16 @@ static u64 next_segment(u64 ea, u64 vsid) + return ea + 1; + } + +-static void cxl_prefault_vma(struct cxl_context *ctx) ++static void cxl_prefault_vma(struct cxl_context *ctx, struct mm_struct *mm) + { + u64 ea, last_esid = 0; + struct copro_slb slb; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + int rc; +- struct mm_struct *mm; +- +- mm = get_mem_context(ctx); +- if (mm == NULL) { +- pr_devel("cxl_prefault_vm unable to get mm %i\n", +- pid_nr(ctx->pid)); +- return; +- } + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + for (ea = vma->vm_start; ea < vma->vm_end; + ea = next_segment(ea, slb.vsid)) { + rc = copro_calculate_slb(mm, ea, &slb); +@@ -337,20 +314,28 @@ static void cxl_prefault_vma(struct cxl_context *ctx) + } + } + mmap_read_unlock(mm); +- +- mmput(mm); + } + + void cxl_prefault(struct cxl_context *ctx, u64 wed) + { ++ struct mm_struct *mm = get_mem_context(ctx); ++ ++ if (mm == NULL) { ++ pr_devel("cxl_prefault unable to get mm %i\n", ++ pid_nr(ctx->pid)); ++ return; ++ } ++ + switch (ctx->afu->prefault_mode) { + case CXL_PREFAULT_WED: +- cxl_prefault_one(ctx, wed); ++ cxl_fault_segment(ctx, mm, wed); + break; + case CXL_PREFAULT_ALL: +- cxl_prefault_vma(ctx); ++ cxl_prefault_vma(ctx, mm); + break; + default: + break; + } ++ ++ mmput(mm); + } +diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c +index 28f87cd8b3ed..290b1bb0e9cd 100644 +--- a/drivers/tee/optee/call.c ++++ b/drivers/tee/optee/call.c +@@ -492,15 +492,18 @@ static bool is_normal_memory(pgprot_t p) + #endif + } + +-static int __check_mem_type(struct vm_area_struct *vma, unsigned long end) ++static int __check_mem_type(struct mm_struct *mm, unsigned long start, ++ unsigned long end) + { +- while (vma && is_normal_memory(vma->vm_page_prot)) { +- if (vma->vm_end >= end) +- return 0; +- vma = vma->vm_next; ++ struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, start); ++ ++ for_each_vma_range(vmi, vma, end) { ++ if (!is_normal_memory(vma->vm_page_prot)) ++ return -EINVAL; + } + +- return -EINVAL; ++ return 0; + } + + int optee_check_mem_type(unsigned long start, size_t num_pages) +@@ -516,8 +519,7 @@ int optee_check_mem_type(unsigned long start, size_t num_pages) + return 0; + + mmap_read_lock(mm); +- rc = __check_mem_type(find_vma(mm, start), +- start + num_pages * PAGE_SIZE); ++ rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE); + mmap_read_unlock(mm); + + return rc; +diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c +index e88e8f6f0a33..fae50a24630b 100644 +--- a/drivers/xen/privcmd.c ++++ b/drivers/xen/privcmd.c +@@ -282,7 +282,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata) + struct page, lru); + struct privcmd_mmap_entry *msg = page_address(page); + +- vma = find_vma(mm, msg->va); ++ vma = vma_lookup(mm, msg->va); + rc = -EINVAL; + + if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data) +diff --git a/fs/coredump.c b/fs/coredump.c +index 1ab4f5b76a1e..debcebabcd73 100644 +--- a/fs/coredump.c ++++ b/fs/coredump.c +@@ -1100,30 +1100,20 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, + return vma->vm_end - vma->vm_start; + } + +-static struct vm_area_struct *first_vma(struct task_struct *tsk, +- struct vm_area_struct *gate_vma) +-{ +- struct vm_area_struct *ret = tsk->mm->mmap; +- +- if (ret) +- return ret; +- return gate_vma; +-} +- + /* + * Helper function for iterating across a vma list. It ensures that the caller + * will visit `gate_vma' prior to terminating the search. + */ +-static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, ++static struct vm_area_struct *coredump_next_vma(struct ma_state *mas, ++ struct vm_area_struct *vma, + struct vm_area_struct *gate_vma) + { +- struct vm_area_struct *ret; +- +- ret = this_vma->vm_next; +- if (ret) +- return ret; +- if (this_vma == gate_vma) ++ if (gate_vma && (vma == gate_vma)) + return NULL; ++ ++ vma = mas_next(mas, ULONG_MAX); ++ if (vma) ++ return vma; + return gate_vma; + } + +@@ -1147,9 +1137,10 @@ static void free_vma_snapshot(struct coredump_params *cprm) + */ + static bool dump_vma_snapshot(struct coredump_params *cprm) + { +- struct vm_area_struct *vma, *gate_vma; ++ struct vm_area_struct *gate_vma, *vma = NULL; + struct mm_struct *mm = current->mm; +- int i; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ int i = 0; + + /* + * Once the stack expansion code is fixed to not change VMA bounds +@@ -1169,8 +1160,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) + return false; + } + +- for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; +- vma = next_vma(vma, gate_vma), i++) { ++ while ((vma = coredump_next_vma(&mas, vma, gate_vma)) != NULL) { + struct core_vma_metadata *m = cprm->vma_meta + i; + + m->start = vma->vm_start; +@@ -1178,10 +1168,10 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) + m->flags = vma->vm_flags; + m->dump_size = vma_dump_size(vma, cprm->mm_flags); + m->pgoff = vma->vm_pgoff; +- + m->file = vma->vm_file; + if (m->file) + get_file(m->file); ++ i++; + } + + mmap_write_unlock(mm); +diff --git a/fs/exec.c b/fs/exec.c +index c67b12f0f577..3a7bae4e16e5 100644 +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -28,7 +28,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -683,6 +682,8 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + unsigned long length = old_end - old_start; + unsigned long new_start = old_start - shift; + unsigned long new_end = old_end - shift; ++ VMA_ITERATOR(vmi, mm, new_start); ++ struct vm_area_struct *next; + struct mmu_gather tlb; + + BUG_ON(new_start > new_end); +@@ -691,7 +692,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + * ensure there are no vmas between where we want to go + * and where we are + */ +- if (vma != find_vma(mm, new_start)) ++ if (vma != vma_next(&vmi)) + return -EFAULT; + + /* +@@ -710,12 +711,13 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + + lru_add_drain(); + tlb_gather_mmu(&tlb, mm); ++ next = vma_next(&vmi); + if (new_end > old_start) { + /* + * when the old and new regions overlap clear from new_end. + */ + free_pgd_range(&tlb, new_end, old_end, new_end, +- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); ++ next ? next->vm_start : USER_PGTABLES_CEILING); + } else { + /* + * otherwise, clean from old_start; this is done to not touch +@@ -724,7 +726,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + * for the others its just a little faster. + */ + free_pgd_range(&tlb, old_start, old_end, new_end, +- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); ++ next ? next->vm_start : USER_PGTABLES_CEILING); + } + tlb_finish_mmu(&tlb); + +@@ -1024,8 +1026,6 @@ static int exec_mmap(struct mm_struct *mm) + activate_mm(active_mm, mm); + if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); +- tsk->mm->vmacache_seqnum = 0; +- vmacache_flush(tsk); + task_unlock(tsk); + lru_gen_use_mm(mm); + if (old_mm) { +diff --git a/fs/proc/base.c b/fs/proc/base.c +index 93f7e3d971e4..12885a75913f 100644 +--- a/fs/proc/base.c ++++ b/fs/proc/base.c +@@ -2350,6 +2350,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) + GENRADIX(struct map_files_info) fa; + struct map_files_info *p; + int ret; ++ struct vma_iterator vmi; + + genradix_init(&fa); + +@@ -2388,7 +2389,9 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) + * routine might require mmap_lock taken in might_fault(). + */ + +- for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { ++ pos = 2; ++ vma_iter_init(&vmi, mm, 0); ++ for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + if (++pos <= ctx->pos) +diff --git a/fs/proc/internal.h b/fs/proc/internal.h +index 06a80f78433d..f03000764ce5 100644 +--- a/fs/proc/internal.h ++++ b/fs/proc/internal.h +@@ -285,7 +285,7 @@ struct proc_maps_private { + struct task_struct *task; + struct mm_struct *mm; + #ifdef CONFIG_MMU +- struct vm_area_struct *tail_vma; ++ struct vma_iterator iter; + #endif + #ifdef CONFIG_NUMA + struct mempolicy *task_mempolicy; +diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c +index 4e0023643f8b..72a02b563e64 100644 +--- a/fs/proc/task_mmu.c ++++ b/fs/proc/task_mmu.c +@@ -1,6 +1,5 @@ + // SPDX-License-Identifier: GPL-2.0 + #include +-#include + #include + #include + #include +@@ -124,12 +123,26 @@ static void release_task_mempolicy(struct proc_maps_private *priv) + } + #endif + ++static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv, ++ loff_t *ppos) ++{ ++ struct vm_area_struct *vma = vma_next(&priv->iter); ++ ++ if (vma) { ++ *ppos = vma->vm_start; ++ } else { ++ *ppos = -2UL; ++ vma = get_gate_vma(priv->mm); ++ } ++ ++ return vma; ++} ++ + static void *m_start(struct seq_file *m, loff_t *ppos) + { + struct proc_maps_private *priv = m->private; + unsigned long last_addr = *ppos; + struct mm_struct *mm; +- struct vm_area_struct *vma; + + /* See m_next(). Zero at the start or after lseek. */ + if (last_addr == -1UL) +@@ -153,31 +166,21 @@ static void *m_start(struct seq_file *m, loff_t *ppos) + return ERR_PTR(-EINTR); + } + ++ vma_iter_init(&priv->iter, mm, last_addr); + hold_task_mempolicy(priv); +- priv->tail_vma = get_gate_vma(mm); +- +- vma = find_vma(mm, last_addr); +- if (vma) +- return vma; ++ if (last_addr == -2UL) ++ return get_gate_vma(mm); + +- return priv->tail_vma; ++ return proc_get_vma(priv, ppos); + } + + static void *m_next(struct seq_file *m, void *v, loff_t *ppos) + { +- struct proc_maps_private *priv = m->private; +- struct vm_area_struct *next, *vma = v; +- +- if (vma == priv->tail_vma) +- next = NULL; +- else if (vma->vm_next) +- next = vma->vm_next; +- else +- next = priv->tail_vma; +- +- *ppos = next ? next->vm_start : -1UL; +- +- return next; ++ if (*ppos == -2UL) { ++ *ppos = -1UL; ++ return NULL; ++ } ++ return proc_get_vma(m->private, ppos); + } + + static void m_stop(struct seq_file *m, void *v) +@@ -877,16 +880,16 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + { + struct proc_maps_private *priv = m->private; + struct mem_size_stats mss; +- struct mm_struct *mm; ++ struct mm_struct *mm = priv->mm; + struct vm_area_struct *vma; +- unsigned long last_vma_end = 0; ++ unsigned long vma_start = 0, last_vma_end = 0; + int ret = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + priv->task = get_proc_task(priv->inode); + if (!priv->task) + return -ESRCH; + +- mm = priv->mm; + if (!mm || !mmget_not_zero(mm)) { + ret = -ESRCH; + goto out_put_task; +@@ -899,8 +902,13 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + goto out_put_mm; + + hold_task_mempolicy(priv); ++ vma = mas_find(&mas, 0); ++ ++ if (unlikely(!vma)) ++ goto empty_set; + +- for (vma = priv->mm->mmap; vma;) { ++ vma_start = vma->vm_start; ++ do { + smap_gather_stats(vma, &mss, 0); + last_vma_end = vma->vm_end; + +@@ -909,6 +917,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + * access it for write request. + */ + if (mmap_lock_is_contended(mm)) { ++ mas_pause(&mas); + mmap_read_unlock(mm); + ret = mmap_read_lock_killable(mm); + if (ret) { +@@ -952,7 +961,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + * contains last_vma_end. + * Iterate VMA' from last_vma_end. + */ +- vma = find_vma(mm, last_vma_end - 1); ++ vma = mas_find(&mas, ULONG_MAX); + /* Case 3 above */ + if (!vma) + break; +@@ -966,11 +975,10 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + smap_gather_stats(vma, &mss, last_vma_end); + } + /* Case 2 above */ +- vma = vma->vm_next; +- } ++ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); + +- show_vma_header_prefix(m, priv->mm->mmap->vm_start, +- last_vma_end, 0, 0, 0, 0); ++empty_set: ++ show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); + seq_pad(m, ' '); + seq_puts(m, "[rollup]\n"); + +@@ -1263,6 +1271,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, + return -ESRCH; + mm = get_task_mm(task); + if (mm) { ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + struct mmu_notifier_range range; + struct clear_refs_private cp = { + .type = type, +@@ -1282,7 +1291,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, + } + + if (type == CLEAR_REFS_SOFT_DIRTY) { +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (!(vma->vm_flags & VM_SOFTDIRTY)) + continue; + vma->vm_flags &= ~VM_SOFTDIRTY; +@@ -1294,8 +1303,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, + 0, NULL, mm, 0, -1UL); + mmu_notifier_invalidate_range_start(&range); + } +- walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, +- &cp); ++ walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); + if (type == CLEAR_REFS_SOFT_DIRTY) { + mmu_notifier_invalidate_range_end(&range); + flush_tlb_mm(mm); +diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c +index a6d21fc0033c..2fd06f52b6a4 100644 +--- a/fs/proc/task_nommu.c ++++ b/fs/proc/task_nommu.c +@@ -20,15 +20,13 @@ + */ + void task_mem(struct seq_file *m, struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + struct vm_region *region; +- struct rb_node *p; + unsigned long bytes = 0, sbytes = 0, slack = 0, size; +- +- mmap_read_lock(mm); +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { +- vma = rb_entry(p, struct vm_area_struct, vm_rb); + ++ mmap_read_lock(mm); ++ for_each_vma(vmi, vma) { + bytes += kobjsize(vma); + + region = vma->vm_region; +@@ -82,15 +80,13 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) + + unsigned long task_vsize(struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; +- struct rb_node *p; + unsigned long vsize = 0; + + mmap_read_lock(mm); +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { +- vma = rb_entry(p, struct vm_area_struct, vm_rb); ++ for_each_vma(vmi, vma) + vsize += vma->vm_end - vma->vm_start; +- } + mmap_read_unlock(mm); + return vsize; + } +@@ -99,14 +95,13 @@ unsigned long task_statm(struct mm_struct *mm, + unsigned long *shared, unsigned long *text, + unsigned long *data, unsigned long *resident) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + struct vm_region *region; +- struct rb_node *p; + unsigned long size = kobjsize(mm); + + mmap_read_lock(mm); +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { +- vma = rb_entry(p, struct vm_area_struct, vm_rb); ++ for_each_vma(vmi, vma) { + size += kobjsize(vma); + region = vma->vm_region; + if (region) { +@@ -190,17 +185,19 @@ static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) + */ + static int show_map(struct seq_file *m, void *_p) + { +- struct rb_node *p = _p; +- +- return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb)); ++ return nommu_vma_show(m, _p); + } + + static void *m_start(struct seq_file *m, loff_t *pos) + { + struct proc_maps_private *priv = m->private; + struct mm_struct *mm; +- struct rb_node *p; +- loff_t n = *pos; ++ struct vm_area_struct *vma; ++ unsigned long addr = *pos; ++ ++ /* See m_next(). Zero at the start or after lseek. */ ++ if (addr == -1UL) ++ return NULL; + + /* pin the task and mm whilst we play with them */ + priv->task = get_proc_task(priv->inode); +@@ -216,10 +213,10 @@ static void *m_start(struct seq_file *m, loff_t *pos) + return ERR_PTR(-EINTR); + } + +- /* start from the Nth VMA */ +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) +- if (n-- == 0) +- return p; ++ /* start the next element from addr */ ++ vma = find_vma(mm, addr); ++ if (vma) ++ return vma; + + mmap_read_unlock(mm); + mmput(mm); +@@ -242,10 +239,10 @@ static void m_stop(struct seq_file *m, void *_vml) + + static void *m_next(struct seq_file *m, void *_p, loff_t *pos) + { +- struct rb_node *p = _p; ++ struct vm_area_struct *vma = _p; + +- (*pos)++; +- return p ? rb_next(p) : NULL; ++ *pos = vma->vm_end; ++ return find_vma(vma->vm_mm, vma->vm_end); + } + + static const struct seq_operations proc_pid_maps_ops = { +diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c +index 175de70e3adf..f662cede4488 100644 +--- a/fs/userfaultfd.c ++++ b/fs/userfaultfd.c +@@ -615,14 +615,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, + if (release_new_ctx) { + struct vm_area_struct *vma; + struct mm_struct *mm = release_new_ctx->mm; ++ VMA_ITERATOR(vmi, mm, 0); + + /* the various vma->vm_userfaultfd_ctx still points to it */ + mmap_write_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) ++ for_each_vma(vmi, vma) { + if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; + vma->vm_flags &= ~__VM_UFFD_FLAGS; + } ++ } + mmap_write_unlock(mm); + + userfaultfd_ctx_put(release_new_ctx); +@@ -803,11 +805,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, + return false; + } + +-int userfaultfd_unmap_prep(struct vm_area_struct *vma, +- unsigned long start, unsigned long end, +- struct list_head *unmaps) ++int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, ++ unsigned long end, struct list_head *unmaps) + { +- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { ++ VMA_ITERATOR(vmi, mm, start); ++ struct vm_area_struct *vma; ++ ++ for_each_vma_range(vmi, vma, end) { + struct userfaultfd_unmap_ctx *unmap_ctx; + struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; + +@@ -857,6 +861,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) + /* len == 0 means wake all */ + struct userfaultfd_wake_range range = { .len = 0, }; + unsigned long new_flags; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + WRITE_ONCE(ctx->released, true); + +@@ -873,7 +878,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) + */ + mmap_write_lock(mm); + prev = NULL; +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + cond_resched(); + BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ + !!(vma->vm_flags & __VM_UFFD_FLAGS)); +@@ -887,10 +892,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file) + vma->vm_file, vma->vm_pgoff, + vma_policy(vma), + NULL_VM_UFFD_CTX, anon_vma_name(vma)); +- if (prev) ++ if (prev) { ++ mas_pause(&mas); + vma = prev; +- else ++ } else { + prev = vma; ++ } ++ + vma->vm_flags = new_flags; + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; + } +@@ -1272,6 +1280,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + bool found; + bool basic_ioctls; + unsigned long start, end, vma_end; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + user_uffdio_register = (struct uffdio_register __user *) arg; + +@@ -1314,7 +1323,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + goto out; + + mmap_write_lock(mm); +- vma = find_vma_prev(mm, start, &prev); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) + goto out_unlock; + +@@ -1339,7 +1349,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + */ + found = false; + basic_ioctls = false; +- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { ++ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { + cond_resched(); + + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ +@@ -1399,8 +1409,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + } + BUG_ON(!found); + +- if (vma->vm_start < start) +- prev = vma; ++ mas_set(&mas, start); ++ prev = mas_prev(&mas, 0); ++ if (prev != vma) ++ mas_next(&mas, ULONG_MAX); + + ret = 0; + do { +@@ -1430,6 +1442,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + ((struct vm_userfaultfd_ctx){ ctx }), + anon_vma_name(vma)); + if (prev) { ++ /* vma_merge() invalidated the mas */ ++ mas_pause(&mas); + vma = prev; + goto next; + } +@@ -1437,11 +1451,15 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + ret = split_vma(mm, vma, start, 1); + if (ret) + break; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } + if (vma->vm_end > end) { + ret = split_vma(mm, vma, end, 0); + if (ret) + break; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } + next: + /* +@@ -1458,8 +1476,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + skip: + prev = vma; + start = vma->vm_end; +- vma = vma->vm_next; +- } while (vma && vma->vm_start < end); ++ vma = mas_next(&mas, end - 1); ++ } while (vma); + out_unlock: + mmap_write_unlock(mm); + mmput(mm); +@@ -1503,6 +1521,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + bool found; + unsigned long start, end, vma_end; + const void __user *buf = (void __user *)arg; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + ret = -EFAULT; + if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) +@@ -1521,7 +1540,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + goto out; + + mmap_write_lock(mm); +- vma = find_vma_prev(mm, start, &prev); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) + goto out_unlock; + +@@ -1546,7 +1566,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + */ + found = false; + ret = -EINVAL; +- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { ++ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { + cond_resched(); + + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ +@@ -1566,8 +1586,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + } + BUG_ON(!found); + +- if (vma->vm_start < start) +- prev = vma; ++ mas_set(&mas, start); ++ prev = mas_prev(&mas, 0); ++ if (prev != vma) ++ mas_next(&mas, ULONG_MAX); + + ret = 0; + do { +@@ -1636,8 +1658,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + skip: + prev = vma; + start = vma->vm_end; +- vma = vma->vm_next; +- } while (vma && vma->vm_start < end); ++ vma = mas_next(&mas, end - 1); ++ } while (vma); + out_unlock: + mmap_write_unlock(mm); + mmput(mm); +diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h +new file mode 100644 +index 000000000000..2effab72add1 +--- /dev/null ++++ b/include/linux/maple_tree.h +@@ -0,0 +1,685 @@ ++/* SPDX-License-Identifier: GPL-2.0+ */ ++#ifndef _LINUX_MAPLE_TREE_H ++#define _LINUX_MAPLE_TREE_H ++/* ++ * Maple Tree - An RCU-safe adaptive tree for storing ranges ++ * Copyright (c) 2018-2022 Oracle ++ * Authors: Liam R. Howlett ++ * Matthew Wilcox ++ */ ++ ++#include ++#include ++#include ++/* #define CONFIG_MAPLE_RCU_DISABLED */ ++/* #define CONFIG_DEBUG_MAPLE_TREE_VERBOSE */ ++ ++/* ++ * Allocated nodes are mutable until they have been inserted into the tree, ++ * at which time they cannot change their type until they have been removed ++ * from the tree and an RCU grace period has passed. ++ * ++ * Removed nodes have their ->parent set to point to themselves. RCU readers ++ * check ->parent before relying on the value that they loaded from the ++ * slots array. This lets us reuse the slots array for the RCU head. ++ * ++ * Nodes in the tree point to their parent unless bit 0 is set. ++ */ ++#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) ++/* 64bit sizes */ ++#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ ++#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ ++#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ ++#define MAPLE_ARANGE64_META_MAX 15 /* Out of range for metadata */ ++#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) ++#else ++/* 32bit sizes */ ++#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ ++#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ ++#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ ++#define MAPLE_ARANGE64_META_MAX 31 /* Out of range for metadata */ ++#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) ++#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ ++ ++#define MAPLE_NODE_MASK 255UL ++ ++/* ++ * The node->parent of the root node has bit 0 set and the rest of the pointer ++ * is a pointer to the tree itself. No more bits are available in this pointer ++ * (on m68k, the data structure may only be 2-byte aligned). ++ * ++ * Internal non-root nodes can only have maple_range_* nodes as parents. The ++ * parent pointer is 256B aligned like all other tree nodes. When storing a 32 ++ * or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an ++ * extra bit to store the offset. This extra bit comes from a reuse of the last ++ * bit in the node type. This is possible by using bit 1 to indicate if bit 2 ++ * is part of the type or the slot. ++ * ++ * Once the type is decided, the decision of an allocation range type or a range ++ * type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE ++ * flag. ++ * ++ * Node types: ++ * 0x??1 = Root ++ * 0x?00 = 16 bit nodes ++ * 0x010 = 32 bit nodes ++ * 0x110 = 64 bit nodes ++ * ++ * Slot size and location in the parent pointer: ++ * type : slot location ++ * 0x??1 : Root ++ * 0x?00 : 16 bit values, type in 0-1, slot in 2-6 ++ * 0x010 : 32 bit values, type in 0-2, slot in 3-6 ++ * 0x110 : 64 bit values, type in 0-2, slot in 3-6 ++ */ ++ ++/* ++ * This metadata is used to optimize the gap updating code and in reverse ++ * searching for gaps or any other code that needs to find the end of the data. ++ */ ++struct maple_metadata { ++ unsigned char end; ++ unsigned char gap; ++}; ++ ++/* ++ * Leaf nodes do not store pointers to nodes, they store user data. Users may ++ * store almost any bit pattern. As noted above, the optimisation of storing an ++ * entry at 0 in the root pointer cannot be done for data which have the bottom ++ * two bits set to '10'. We also reserve values with the bottom two bits set to ++ * '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs ++ * return errnos as a negative errno shifted right by two bits and the bottom ++ * two bits set to '10', and while choosing to store these values in the array ++ * is not an error, it may lead to confusion if you're testing for an error with ++ * mas_is_err(). ++ * ++ * Non-leaf nodes store the type of the node pointed to (enum maple_type in bits ++ * 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now. ++ * ++ * In regular B-Tree terms, pivots are called keys. The term pivot is used to ++ * indicate that the tree is specifying ranges, Pivots may appear in the ++ * subtree with an entry attached to the value whereas keys are unique to a ++ * specific position of a B-tree. Pivot values are inclusive of the slot with ++ * the same index. ++ */ ++ ++struct maple_range_64 { ++ struct maple_pnode *parent; ++ unsigned long pivot[MAPLE_RANGE64_SLOTS - 1]; ++ union { ++ void __rcu *slot[MAPLE_RANGE64_SLOTS]; ++ struct { ++ void __rcu *pad[MAPLE_RANGE64_SLOTS - 1]; ++ struct maple_metadata meta; ++ }; ++ }; ++}; ++ ++/* ++ * At tree creation time, the user can specify that they're willing to trade off ++ * storing fewer entries in a tree in return for storing more information in ++ * each node. ++ * ++ * The maple tree supports recording the largest range of NULL entries available ++ * in this node, also called gaps. This optimises the tree for allocating a ++ * range. ++ */ ++struct maple_arange_64 { ++ struct maple_pnode *parent; ++ unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1]; ++ void __rcu *slot[MAPLE_ARANGE64_SLOTS]; ++ unsigned long gap[MAPLE_ARANGE64_SLOTS]; ++ struct maple_metadata meta; ++}; ++ ++struct maple_alloc { ++ unsigned long total; ++ unsigned char node_count; ++ unsigned int request_count; ++ struct maple_alloc *slot[MAPLE_ALLOC_SLOTS]; ++}; ++ ++struct maple_topiary { ++ struct maple_pnode *parent; ++ struct maple_enode *next; /* Overlaps the pivot */ ++}; ++ ++enum maple_type { ++ maple_dense, ++ maple_leaf_64, ++ maple_range_64, ++ maple_arange_64, ++}; ++ ++ ++/** ++ * DOC: Maple tree flags ++ * ++ * * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree ++ * * MT_FLAGS_USE_RCU - Operate in RCU mode ++ * * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags ++ * * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value ++ * * MT_FLAGS_LOCK_MASK - How the mt_lock is used ++ * * MT_FLAGS_LOCK_IRQ - Acquired irq-safe ++ * * MT_FLAGS_LOCK_BH - Acquired bh-safe ++ * * MT_FLAGS_LOCK_EXTERN - mt_lock is not used ++ * ++ * MAPLE_HEIGHT_MAX The largest height that can be stored ++ */ ++#define MT_FLAGS_ALLOC_RANGE 0x01 ++#define MT_FLAGS_USE_RCU 0x02 ++#define MT_FLAGS_HEIGHT_OFFSET 0x02 ++#define MT_FLAGS_HEIGHT_MASK 0x7C ++#define MT_FLAGS_LOCK_MASK 0x300 ++#define MT_FLAGS_LOCK_IRQ 0x100 ++#define MT_FLAGS_LOCK_BH 0x200 ++#define MT_FLAGS_LOCK_EXTERN 0x300 ++ ++#define MAPLE_HEIGHT_MAX 31 ++ ++ ++#define MAPLE_NODE_TYPE_MASK 0x0F ++#define MAPLE_NODE_TYPE_SHIFT 0x03 ++ ++#define MAPLE_RESERVED_RANGE 4096 ++ ++#ifdef CONFIG_LOCKDEP ++typedef struct lockdep_map *lockdep_map_p; ++#define mt_lock_is_held(mt) lock_is_held(mt->ma_external_lock) ++#define mt_set_external_lock(mt, lock) \ ++ (mt)->ma_external_lock = &(lock)->dep_map ++#else ++typedef struct { /* nothing */ } lockdep_map_p; ++#define mt_lock_is_held(mt) 1 ++#define mt_set_external_lock(mt, lock) do { } while (0) ++#endif ++ ++/* ++ * If the tree contains a single entry at index 0, it is usually stored in ++ * tree->ma_root. To optimise for the page cache, an entry which ends in '00', ++ * '01' or '11' is stored in the root, but an entry which ends in '10' will be ++ * stored in a node. Bits 3-6 are used to store enum maple_type. ++ * ++ * The flags are used both to store some immutable information about this tree ++ * (set at tree creation time) and dynamic information set under the spinlock. ++ * ++ * Another use of flags are to indicate global states of the tree. This is the ++ * case with the MAPLE_USE_RCU flag, which indicates the tree is currently in ++ * RCU mode. This mode was added to allow the tree to reuse nodes instead of ++ * re-allocating and RCU freeing nodes when there is a single user. ++ */ ++struct maple_tree { ++ union { ++ spinlock_t ma_lock; ++ lockdep_map_p ma_external_lock; ++ }; ++ void __rcu *ma_root; ++ unsigned int ma_flags; ++}; ++ ++/** ++ * MTREE_INIT() - Initialize a maple tree ++ * @name: The maple tree name ++ * @__flags: The maple tree flags ++ * ++ */ ++#define MTREE_INIT(name, __flags) { \ ++ .ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), \ ++ .ma_flags = __flags, \ ++ .ma_root = NULL, \ ++} ++ ++/** ++ * MTREE_INIT_EXT() - Initialize a maple tree with an external lock. ++ * @name: The tree name ++ * @__flags: The maple tree flags ++ * @__lock: The external lock ++ */ ++#ifdef CONFIG_LOCKDEP ++#define MTREE_INIT_EXT(name, __flags, __lock) { \ ++ .ma_external_lock = &(__lock).dep_map, \ ++ .ma_flags = (__flags), \ ++ .ma_root = NULL, \ ++} ++#else ++#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags) ++#endif ++ ++#define DEFINE_MTREE(name) \ ++ struct maple_tree name = MTREE_INIT(name, 0) ++ ++#define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) ++#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock)) ++ ++/* ++ * The Maple Tree squeezes various bits in at various points which aren't ++ * necessarily obvious. Usually, this is done by observing that pointers are ++ * N-byte aligned and thus the bottom log_2(N) bits are available for use. We ++ * don't use the high bits of pointers to store additional information because ++ * we don't know what bits are unused on any given architecture. ++ * ++ * Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8 ++ * low bits for our own purposes. Nodes are currently of 4 types: ++ * 1. Single pointer (Range is 0-0) ++ * 2. Non-leaf Allocation Range nodes ++ * 3. Non-leaf Range nodes ++ * 4. Leaf Range nodes All nodes consist of a number of node slots, ++ * pivots, and a parent pointer. ++ */ ++ ++struct maple_node { ++ union { ++ struct { ++ struct maple_pnode *parent; ++ void __rcu *slot[MAPLE_NODE_SLOTS]; ++ }; ++ struct { ++ void *pad; ++ struct rcu_head rcu; ++ struct maple_enode *piv_parent; ++ unsigned char parent_slot; ++ enum maple_type type; ++ unsigned char slot_len; ++ unsigned int ma_flags; ++ }; ++ struct maple_range_64 mr64; ++ struct maple_arange_64 ma64; ++ struct maple_alloc alloc; ++ }; ++}; ++ ++/* ++ * More complicated stores can cause two nodes to become one or three and ++ * potentially alter the height of the tree. Either half of the tree may need ++ * to be rebalanced against the other. The ma_topiary struct is used to track ++ * which nodes have been 'cut' from the tree so that the change can be done ++ * safely at a later date. This is done to support RCU. ++ */ ++struct ma_topiary { ++ struct maple_enode *head; ++ struct maple_enode *tail; ++ struct maple_tree *mtree; ++}; ++ ++void *mtree_load(struct maple_tree *mt, unsigned long index); ++ ++int mtree_insert(struct maple_tree *mt, unsigned long index, ++ void *entry, gfp_t gfp); ++int mtree_insert_range(struct maple_tree *mt, unsigned long first, ++ unsigned long last, void *entry, gfp_t gfp); ++int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp); ++int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp); ++ ++int mtree_store_range(struct maple_tree *mt, unsigned long first, ++ unsigned long last, void *entry, gfp_t gfp); ++int mtree_store(struct maple_tree *mt, unsigned long index, ++ void *entry, gfp_t gfp); ++void *mtree_erase(struct maple_tree *mt, unsigned long index); ++ ++void mtree_destroy(struct maple_tree *mt); ++void __mt_destroy(struct maple_tree *mt); ++ ++/** ++ * mtree_empty() - Determine if a tree has any present entries. ++ * @mt: Maple Tree. ++ * ++ * Context: Any context. ++ * Return: %true if the tree contains only NULL pointers. ++ */ ++static inline bool mtree_empty(const struct maple_tree *mt) ++{ ++ return mt->ma_root == NULL; ++} ++ ++/* Advanced API */ ++ ++/* ++ * The maple state is defined in the struct ma_state and is used to keep track ++ * of information during operations, and even between operations when using the ++ * advanced API. ++ * ++ * If state->node has bit 0 set then it references a tree location which is not ++ * a node (eg the root). If bit 1 is set, the rest of the bits are a negative ++ * errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the ++ * node type. ++ * ++ * state->alloc either has a request number of nodes or an allocated node. If ++ * stat->alloc has a requested number of nodes, the first bit will be set (0x1) ++ * and the remaining bits are the value. If state->alloc is a node, then the ++ * node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for ++ * storing more allocated nodes, a total number of nodes allocated, and the ++ * node_count in this node. node_count is the number of allocated nodes in this ++ * node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further ++ * nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc ++ * by removing a node from the state->alloc node until state->alloc->node_count ++ * is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted ++ * to state->alloc. Nodes are pushed onto state->alloc by putting the current ++ * state->alloc into the pushed node's slot[0]. ++ * ++ * The state also contains the implied min/max of the state->node, the depth of ++ * this search, and the offset. The implied min/max are either from the parent ++ * node or are 0-oo for the root node. The depth is incremented or decremented ++ * every time a node is walked down or up. The offset is the slot/pivot of ++ * interest in the node - either for reading or writing. ++ * ++ * When returning a value the maple state index and last respectively contain ++ * the start and end of the range for the entry. Ranges are inclusive in the ++ * Maple Tree. ++ */ ++struct ma_state { ++ struct maple_tree *tree; /* The tree we're operating in */ ++ unsigned long index; /* The index we're operating on - range start */ ++ unsigned long last; /* The last index we're operating on - range end */ ++ struct maple_enode *node; /* The node containing this entry */ ++ unsigned long min; /* The minimum index of this node - implied pivot min */ ++ unsigned long max; /* The maximum index of this node - implied pivot max */ ++ struct maple_alloc *alloc; /* Allocated nodes for this operation */ ++ unsigned char depth; /* depth of tree descent during write */ ++ unsigned char offset; ++ unsigned char mas_flags; ++}; ++ ++struct ma_wr_state { ++ struct ma_state *mas; ++ struct maple_node *node; /* Decoded mas->node */ ++ unsigned long r_min; /* range min */ ++ unsigned long r_max; /* range max */ ++ enum maple_type type; /* mas->node type */ ++ unsigned char offset_end; /* The offset where the write ends */ ++ unsigned char node_end; /* mas->node end */ ++ unsigned long *pivots; /* mas->node->pivots pointer */ ++ unsigned long end_piv; /* The pivot at the offset end */ ++ void __rcu **slots; /* mas->node->slots pointer */ ++ void *entry; /* The entry to write */ ++ void *content; /* The existing entry that is being overwritten */ ++}; ++ ++#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock)) ++#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock)) ++ ++ ++/* ++ * Special values for ma_state.node. ++ * MAS_START means we have not searched the tree. ++ * MAS_ROOT means we have searched the tree and the entry we found lives in ++ * the root of the tree (ie it has index 0, length 1 and is the only entry in ++ * the tree). ++ * MAS_NONE means we have searched the tree and there is no node in the ++ * tree for this entry. For example, we searched for index 1 in an empty ++ * tree. Or we have a tree which points to a full leaf node and we ++ * searched for an entry which is larger than can be contained in that ++ * leaf node. ++ * MA_ERROR represents an errno. After dropping the lock and attempting ++ * to resolve the error, the walk would have to be restarted from the ++ * top of the tree as the tree may have been modified. ++ */ ++#define MAS_START ((struct maple_enode *)1UL) ++#define MAS_ROOT ((struct maple_enode *)5UL) ++#define MAS_NONE ((struct maple_enode *)9UL) ++#define MAS_PAUSE ((struct maple_enode *)17UL) ++#define MA_ERROR(err) \ ++ ((struct maple_enode *)(((unsigned long)err << 2) | 2UL)) ++ ++#define MA_STATE(name, mt, first, end) \ ++ struct ma_state name = { \ ++ .tree = mt, \ ++ .index = first, \ ++ .last = end, \ ++ .node = MAS_START, \ ++ .min = 0, \ ++ .max = ULONG_MAX, \ ++ .alloc = NULL, \ ++ } ++ ++#define MA_WR_STATE(name, ma_state, wr_entry) \ ++ struct ma_wr_state name = { \ ++ .mas = ma_state, \ ++ .content = NULL, \ ++ .entry = wr_entry, \ ++ } ++ ++#define MA_TOPIARY(name, tree) \ ++ struct ma_topiary name = { \ ++ .head = NULL, \ ++ .tail = NULL, \ ++ .mtree = tree, \ ++ } ++ ++void *mas_walk(struct ma_state *mas); ++void *mas_store(struct ma_state *mas, void *entry); ++void *mas_erase(struct ma_state *mas); ++int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp); ++void mas_store_prealloc(struct ma_state *mas, void *entry); ++void *mas_find(struct ma_state *mas, unsigned long max); ++void *mas_find_rev(struct ma_state *mas, unsigned long min); ++int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp); ++bool mas_is_err(struct ma_state *mas); ++ ++bool mas_nomem(struct ma_state *mas, gfp_t gfp); ++void mas_pause(struct ma_state *mas); ++void maple_tree_init(void); ++void mas_destroy(struct ma_state *mas); ++int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); ++ ++void *mas_prev(struct ma_state *mas, unsigned long min); ++void *mas_next(struct ma_state *mas, unsigned long max); ++ ++int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, ++ unsigned long size); ++ ++/* Checks if a mas has not found anything */ ++static inline bool mas_is_none(struct ma_state *mas) ++{ ++ return mas->node == MAS_NONE; ++} ++ ++/* Checks if a mas has been paused */ ++static inline bool mas_is_paused(struct ma_state *mas) ++{ ++ return mas->node == MAS_PAUSE; ++} ++ ++void mas_dup_tree(struct ma_state *oldmas, struct ma_state *mas); ++void mas_dup_store(struct ma_state *mas, void *entry); ++ ++/* ++ * This finds an empty area from the highest address to the lowest. ++ * AKA "Topdown" version, ++ */ ++int mas_empty_area_rev(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size); ++/** ++ * mas_reset() - Reset a Maple Tree operation state. ++ * @mas: Maple Tree operation state. ++ * ++ * Resets the error or walk state of the @mas so future walks of the ++ * array will start from the root. Use this if you have dropped the ++ * lock and want to reuse the ma_state. ++ * ++ * Context: Any context. ++ */ ++static inline void mas_reset(struct ma_state *mas) ++{ ++ mas->node = MAS_START; ++} ++ ++/** ++ * mas_for_each() - Iterate over a range of the maple tree. ++ * @__mas: Maple Tree operation state (maple_state) ++ * @__entry: Entry retrieved from the tree ++ * @__max: maximum index to retrieve from the tree ++ * ++ * When returned, mas->index and mas->last will hold the entire range for the ++ * entry. ++ * ++ * Note: may return the zero entry. ++ * ++ */ ++#define mas_for_each(__mas, __entry, __max) \ ++ while (((__entry) = mas_find((__mas), (__max))) != NULL) ++ ++ ++/** ++ * mas_set_range() - Set up Maple Tree operation state for a different index. ++ * @mas: Maple Tree operation state. ++ * @start: New start of range in the Maple Tree. ++ * @last: New end of range in the Maple Tree. ++ * ++ * Move the operation state to refer to a different range. This will ++ * have the effect of starting a walk from the top; see mas_next() ++ * to move to an adjacent index. ++ */ ++static inline ++void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last) ++{ ++ mas->index = start; ++ mas->last = last; ++ mas->node = MAS_START; ++} ++ ++/** ++ * mas_set() - Set up Maple Tree operation state for a different index. ++ * @mas: Maple Tree operation state. ++ * @index: New index into the Maple Tree. ++ * ++ * Move the operation state to refer to a different index. This will ++ * have the effect of starting a walk from the top; see mas_next() ++ * to move to an adjacent index. ++ */ ++static inline void mas_set(struct ma_state *mas, unsigned long index) ++{ ++ ++ mas_set_range(mas, index, index); ++} ++ ++static inline bool mt_external_lock(const struct maple_tree *mt) ++{ ++ return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN; ++} ++ ++/** ++ * mt_init_flags() - Initialise an empty maple tree with flags. ++ * @mt: Maple Tree ++ * @flags: maple tree flags. ++ * ++ * If you need to initialise a Maple Tree with special flags (eg, an ++ * allocation tree), use this function. ++ * ++ * Context: Any context. ++ */ ++static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags) ++{ ++ mt->ma_flags = flags; ++ if (!mt_external_lock(mt)) ++ spin_lock_init(&mt->ma_lock); ++ rcu_assign_pointer(mt->ma_root, NULL); ++} ++ ++/** ++ * mt_init() - Initialise an empty maple tree. ++ * @mt: Maple Tree ++ * ++ * An empty Maple Tree. ++ * ++ * Context: Any context. ++ */ ++static inline void mt_init(struct maple_tree *mt) ++{ ++ mt_init_flags(mt, 0); ++} ++ ++static inline bool mt_in_rcu(struct maple_tree *mt) ++{ ++#ifdef CONFIG_MAPLE_RCU_DISABLED ++ return false; ++#endif ++ return mt->ma_flags & MT_FLAGS_USE_RCU; ++} ++ ++/** ++ * mt_clear_in_rcu() - Switch the tree to non-RCU mode. ++ * @mt: The Maple Tree ++ */ ++static inline void mt_clear_in_rcu(struct maple_tree *mt) ++{ ++ if (!mt_in_rcu(mt)) ++ return; ++ ++ if (mt_external_lock(mt)) { ++ BUG_ON(!mt_lock_is_held(mt)); ++ mt->ma_flags &= ~MT_FLAGS_USE_RCU; ++ } else { ++ mtree_lock(mt); ++ mt->ma_flags &= ~MT_FLAGS_USE_RCU; ++ mtree_unlock(mt); ++ } ++} ++ ++/** ++ * mt_set_in_rcu() - Switch the tree to RCU safe mode. ++ * @mt: The Maple Tree ++ */ ++static inline void mt_set_in_rcu(struct maple_tree *mt) ++{ ++ if (mt_in_rcu(mt)) ++ return; ++ ++ if (mt_external_lock(mt)) { ++ BUG_ON(!mt_lock_is_held(mt)); ++ mt->ma_flags |= MT_FLAGS_USE_RCU; ++ } else { ++ mtree_lock(mt); ++ mt->ma_flags |= MT_FLAGS_USE_RCU; ++ mtree_unlock(mt); ++ } ++} ++ ++void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max); ++void *mt_find_after(struct maple_tree *mt, unsigned long *index, ++ unsigned long max); ++void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min); ++void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max); ++ ++/** ++ * mt_for_each - Iterate over each entry starting at index until max. ++ * @__tree: The Maple Tree ++ * @__entry: The current entry ++ * @__index: The index to update to track the location in the tree ++ * @__max: The maximum limit for @index ++ * ++ * Note: Will not return the zero entry. ++ */ ++#define mt_for_each(__tree, __entry, __index, __max) \ ++ for (__entry = mt_find(__tree, &(__index), __max); \ ++ __entry; __entry = mt_find_after(__tree, &(__index), __max)) ++ ++ ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++extern atomic_t maple_tree_tests_run; ++extern atomic_t maple_tree_tests_passed; ++ ++void mt_dump(const struct maple_tree *mt); ++void mt_validate(struct maple_tree *mt); ++#define MT_BUG_ON(__tree, __x) do { \ ++ atomic_inc(&maple_tree_tests_run); \ ++ if (__x) { \ ++ pr_info("BUG at %s:%d (%u)\n", \ ++ __func__, __LINE__, __x); \ ++ mt_dump(__tree); \ ++ pr_info("Pass: %u Run:%u\n", \ ++ atomic_read(&maple_tree_tests_passed), \ ++ atomic_read(&maple_tree_tests_run)); \ ++ dump_stack(); \ ++ } else { \ ++ atomic_inc(&maple_tree_tests_passed); \ ++ } \ ++} while (0) ++#else ++#define MT_BUG_ON(__tree, __x) BUG_ON(__x) ++#endif /* CONFIG_DEBUG_MAPLE_TREE */ ++ ++#endif /*_LINUX_MAPLE_TREE_H */ +diff --git a/include/linux/mm.h b/include/linux/mm.h +index 88976a521ef5..9ac0e02e2238 100644 +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -661,6 +661,38 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) + return vma->vm_flags & VM_ACCESS_FLAGS; + } + ++static inline ++struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) ++{ ++ return mas_find(&vmi->mas, max); ++} ++ ++static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) ++{ ++ /* ++ * Uses vma_find() to get the first VMA when the iterator starts. ++ * Calling mas_next() could skip the first entry. ++ */ ++ return vma_find(vmi, ULONG_MAX); ++} ++ ++static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) ++{ ++ return mas_prev(&vmi->mas, 0); ++} ++ ++static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) ++{ ++ return vmi->mas.index; ++} ++ ++#define for_each_vma(__vmi, __vma) \ ++ while (((__vma) = vma_next(&(__vmi))) != NULL) ++ ++/* The MM code likes to work with exclusive end addresses */ ++#define for_each_vma_range(__vmi, __vma, __end) \ ++ while (((__vma) = vma_find(&(__vmi), (__end) - 1)) != NULL) ++ + #ifdef CONFIG_SHMEM + /* + * The vma_is_shmem is not inline because it is used only by slow +@@ -1800,8 +1832,9 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, + unsigned long size); + void zap_page_range(struct vm_area_struct *vma, unsigned long address, + unsigned long size); +-void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, +- unsigned long start, unsigned long end); ++void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, ++ struct vm_area_struct *start_vma, unsigned long start, ++ unsigned long end); + + struct mmu_notifier_range; + +@@ -2598,14 +2631,15 @@ extern int __split_vma(struct mm_struct *, struct vm_area_struct *, + extern int split_vma(struct mm_struct *, struct vm_area_struct *, + unsigned long addr, int new_below); + extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); +-extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, +- struct rb_node **, struct rb_node *); + extern void unlink_file_vma(struct vm_area_struct *); + extern struct vm_area_struct *copy_vma(struct vm_area_struct **, + unsigned long addr, unsigned long len, pgoff_t pgoff, + bool *need_rmap_locks); + extern void exit_mmap(struct mm_struct *); + ++void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas); ++void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas); ++ + static inline int check_data_rlimit(unsigned long rlim, + unsigned long new, + unsigned long start, +@@ -2653,8 +2687,9 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, + extern unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + unsigned long pgoff, unsigned long *populate, struct list_head *uf); +-extern int __do_munmap(struct mm_struct *, unsigned long, size_t, +- struct list_head *uf, bool downgrade); ++extern int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, ++ unsigned long start, size_t len, struct list_head *uf, ++ bool downgrade); + extern int do_munmap(struct mm_struct *, unsigned long, size_t, + struct list_head *uf); + extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); +@@ -2721,26 +2756,12 @@ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long add + extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, + struct vm_area_struct **pprev); + +-/** +- * find_vma_intersection() - Look up the first VMA which intersects the interval +- * @mm: The process address space. +- * @start_addr: The inclusive start user address. +- * @end_addr: The exclusive end user address. +- * +- * Returns: The first VMA within the provided range, %NULL otherwise. Assumes +- * start_addr < end_addr. ++/* ++ * Look up the first VMA which intersects the interval [start_addr, end_addr) ++ * NULL if none. Assume start_addr < end_addr. + */ +-static inline + struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, +- unsigned long start_addr, +- unsigned long end_addr) +-{ +- struct vm_area_struct *vma = find_vma(mm, start_addr); +- +- if (vma && end_addr <= vma->vm_start) +- vma = NULL; +- return vma; +-} ++ unsigned long start_addr, unsigned long end_addr); + + /** + * vma_lookup() - Find a VMA at a specific address +@@ -2752,12 +2773,7 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, + static inline + struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) + { +- struct vm_area_struct *vma = find_vma(mm, addr); +- +- if (vma && addr < vma->vm_start) +- vma = NULL; +- +- return vma; ++ return mtree_load(&mm->mm_mt, addr); + } + + static inline unsigned long vm_start_gap(struct vm_area_struct *vma) +@@ -2793,7 +2809,7 @@ static inline unsigned long vma_pages(struct vm_area_struct *vma) + static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, + unsigned long vm_start, unsigned long vm_end) + { +- struct vm_area_struct *vma = find_vma(mm, vm_start); ++ struct vm_area_struct *vma = vma_lookup(mm, vm_start); + + if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) + vma = NULL; +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index e1797813cc2c..5e32211cb5a9 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -407,21 +408,6 @@ struct vm_area_struct { + unsigned long vm_end; /* The first byte after our end address + within vm_mm. */ + +- /* linked list of VM areas per task, sorted by address */ +- struct vm_area_struct *vm_next, *vm_prev; +- +- struct rb_node vm_rb; +- +- /* +- * Largest free memory gap in bytes to the left of this VMA. +- * Either between this VMA and vma->vm_prev, or between one of the +- * VMAs below us in the VMA rbtree and its ->vm_prev. This helps +- * get_unmapped_area find a free area of the right size. +- */ +- unsigned long rb_subtree_gap; +- +- /* Second cache line starts here. */ +- + struct mm_struct *vm_mm; /* The address space we belong to. */ + + /* +@@ -485,9 +471,7 @@ struct vm_area_struct { + struct kioctx_table; + struct mm_struct { + struct { +- struct vm_area_struct *mmap; /* list of VMAs */ +- struct rb_root mm_rb; +- u64 vmacache_seqnum; /* per-thread vmacache */ ++ struct maple_tree mm_mt; + #ifdef CONFIG_MMU + unsigned long (*get_unmapped_area) (struct file *filp, + unsigned long addr, unsigned long len, +@@ -501,7 +485,6 @@ struct mm_struct { + unsigned long mmap_compat_legacy_base; + #endif + unsigned long task_size; /* size of task vm space */ +- unsigned long highest_vm_end; /* highest vma end address */ + pgd_t * pgd; + + #ifdef CONFIG_MEMBARRIER +@@ -697,6 +680,7 @@ struct mm_struct { + unsigned long cpu_bitmap[]; + }; + ++#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN) + extern struct mm_struct init_mm; + + /* Pointer magic because the dynamic array size confuses some compilers. */ +@@ -774,6 +758,27 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) + + #endif /* CONFIG_LRU_GEN */ + ++struct vma_iterator { ++ struct ma_state mas; ++}; ++ ++#define VMA_ITERATOR(name, __mm, __addr) \ ++ struct vma_iterator name = { \ ++ .mas = { \ ++ .tree = &(__mm)->mm_mt, \ ++ .index = __addr, \ ++ .node = MAS_START, \ ++ }, \ ++ } ++ ++static inline void vma_iter_init(struct vma_iterator *vmi, ++ struct mm_struct *mm, unsigned long addr) ++{ ++ vmi->mas.tree = &mm->mm_mt; ++ vmi->mas.index = addr; ++ vmi->mas.node = MAS_START; ++} ++ + struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); +diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h +index c1bc6731125c..0bb4b6da9993 100644 +--- a/include/linux/mm_types_task.h ++++ b/include/linux/mm_types_task.h +@@ -24,18 +24,6 @@ + IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) + #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) + +-/* +- * The per task VMA cache array: +- */ +-#define VMACACHE_BITS 2 +-#define VMACACHE_SIZE (1U << VMACACHE_BITS) +-#define VMACACHE_MASK (VMACACHE_SIZE - 1) +- +-struct vmacache { +- u64 seqnum; +- struct vm_area_struct *vmas[VMACACHE_SIZE]; +-}; +- + /* + * When updating this, please also update struct resident_page_types[] in + * kernel/fork.c +diff --git a/include/linux/sched.h b/include/linux/sched.h +index 8cc46a789193..3dc4e14c4df4 100644 +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -861,7 +861,6 @@ struct task_struct { + struct mm_struct *active_mm; + + /* Per-thread vma caching: */ +- struct vmacache vmacache; + + #ifdef SPLIT_RSS_COUNTING + struct task_rss_stat rss_stat; +diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h +index e1b8a915e9e9..f07e6998bb68 100644 +--- a/include/linux/userfaultfd_k.h ++++ b/include/linux/userfaultfd_k.h +@@ -175,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma, + unsigned long start, + unsigned long end); + +-extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, +- unsigned long start, unsigned long end, +- struct list_head *uf); ++extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, ++ unsigned long end, struct list_head *uf); + extern void userfaultfd_unmap_complete(struct mm_struct *mm, + struct list_head *uf); + +@@ -258,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma, + return true; + } + +-static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, ++static inline int userfaultfd_unmap_prep(struct mm_struct *mm, + unsigned long start, unsigned long end, + struct list_head *uf) + { +diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h +index f3fc36cd2276..3518dba1e02f 100644 +--- a/include/linux/vm_event_item.h ++++ b/include/linux/vm_event_item.h +@@ -129,10 +129,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, + NR_TLB_LOCAL_FLUSH_ALL, + NR_TLB_LOCAL_FLUSH_ONE, + #endif /* CONFIG_DEBUG_TLBFLUSH */ +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- VMACACHE_FIND_CALLS, +- VMACACHE_FIND_HITS, +-#endif + #ifdef CONFIG_SWAP + SWAP_RA, + SWAP_RA_HIT, +diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h +deleted file mode 100644 +index 6fce268a4588..000000000000 +--- a/include/linux/vmacache.h ++++ /dev/null +@@ -1,28 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0 */ +-#ifndef __LINUX_VMACACHE_H +-#define __LINUX_VMACACHE_H +- +-#include +-#include +- +-static inline void vmacache_flush(struct task_struct *tsk) +-{ +- memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas)); +-} +- +-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma); +-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm, +- unsigned long addr); +- +-#ifndef CONFIG_MMU +-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, +- unsigned long start, +- unsigned long end); +-#endif +- +-static inline void vmacache_invalidate(struct mm_struct *mm) +-{ +- mm->vmacache_seqnum++; +-} +- +-#endif /* __LINUX_VMACACHE_H */ +diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h +index bfe38869498d..19cf5b6892ce 100644 +--- a/include/linux/vmstat.h ++++ b/include/linux/vmstat.h +@@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu) + #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) + #endif + +-#ifdef CONFIG_DEBUG_VM_VMACACHE +-#define count_vm_vmacache_event(x) count_vm_event(x) +-#else +-#define count_vm_vmacache_event(x) do {} while (0) +-#endif +- + #define __count_zid_vm_events(item, zid, delta) \ + __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) + +diff --git a/include/trace/events/maple_tree.h b/include/trace/events/maple_tree.h +new file mode 100644 +index 000000000000..2be403bdc2bd +--- /dev/null ++++ b/include/trace/events/maple_tree.h +@@ -0,0 +1,123 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM maple_tree ++ ++#if !defined(_TRACE_MM_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MM_H ++ ++ ++#include ++ ++struct ma_state; ++ ++TRACE_EVENT(ma_op, ++ ++ TP_PROTO(const char *fn, struct ma_state *mas), ++ ++ TP_ARGS(fn, mas), ++ ++ TP_STRUCT__entry( ++ __field(const char *, fn) ++ __field(unsigned long, min) ++ __field(unsigned long, max) ++ __field(unsigned long, index) ++ __field(unsigned long, last) ++ __field(void *, node) ++ ), ++ ++ TP_fast_assign( ++ __entry->fn = fn; ++ __entry->min = mas->min; ++ __entry->max = mas->max; ++ __entry->index = mas->index; ++ __entry->last = mas->last; ++ __entry->node = mas->node; ++ ), ++ ++ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", ++ __entry->fn, ++ (void *) __entry->node, ++ (unsigned long) __entry->min, ++ (unsigned long) __entry->max, ++ (unsigned long) __entry->index, ++ (unsigned long) __entry->last ++ ) ++) ++TRACE_EVENT(ma_read, ++ ++ TP_PROTO(const char *fn, struct ma_state *mas), ++ ++ TP_ARGS(fn, mas), ++ ++ TP_STRUCT__entry( ++ __field(const char *, fn) ++ __field(unsigned long, min) ++ __field(unsigned long, max) ++ __field(unsigned long, index) ++ __field(unsigned long, last) ++ __field(void *, node) ++ ), ++ ++ TP_fast_assign( ++ __entry->fn = fn; ++ __entry->min = mas->min; ++ __entry->max = mas->max; ++ __entry->index = mas->index; ++ __entry->last = mas->last; ++ __entry->node = mas->node; ++ ), ++ ++ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", ++ __entry->fn, ++ (void *) __entry->node, ++ (unsigned long) __entry->min, ++ (unsigned long) __entry->max, ++ (unsigned long) __entry->index, ++ (unsigned long) __entry->last ++ ) ++) ++ ++TRACE_EVENT(ma_write, ++ ++ TP_PROTO(const char *fn, struct ma_state *mas, unsigned long piv, ++ void *val), ++ ++ TP_ARGS(fn, mas, piv, val), ++ ++ TP_STRUCT__entry( ++ __field(const char *, fn) ++ __field(unsigned long, min) ++ __field(unsigned long, max) ++ __field(unsigned long, index) ++ __field(unsigned long, last) ++ __field(unsigned long, piv) ++ __field(void *, val) ++ __field(void *, node) ++ ), ++ ++ TP_fast_assign( ++ __entry->fn = fn; ++ __entry->min = mas->min; ++ __entry->max = mas->max; ++ __entry->index = mas->index; ++ __entry->last = mas->last; ++ __entry->piv = piv; ++ __entry->val = val; ++ __entry->node = mas->node; ++ ), ++ ++ TP_printk("%s\tNode %p (%lu %lu) range:%lu-%lu piv (%lu) val %p", ++ __entry->fn, ++ (void *) __entry->node, ++ (unsigned long) __entry->min, ++ (unsigned long) __entry->max, ++ (unsigned long) __entry->index, ++ (unsigned long) __entry->last, ++ (unsigned long) __entry->piv, ++ (void *) __entry->val ++ ) ++) ++#endif /* _TRACE_MM_H */ ++ ++/* This part must be outside protection */ ++#include +diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h +index 4661f7ba07c0..216de5f03621 100644 +--- a/include/trace/events/mmap.h ++++ b/include/trace/events/mmap.h +@@ -42,6 +42,79 @@ TRACE_EVENT(vm_unmapped_area, + __entry->low_limit, __entry->high_limit, __entry->align_mask, + __entry->align_offset) + ); ++ ++TRACE_EVENT(vma_mas_szero, ++ TP_PROTO(struct maple_tree *mt, unsigned long start, ++ unsigned long end), ++ ++ TP_ARGS(mt, start, end), ++ ++ TP_STRUCT__entry( ++ __field(struct maple_tree *, mt) ++ __field(unsigned long, start) ++ __field(unsigned long, end) ++ ), ++ ++ TP_fast_assign( ++ __entry->mt = mt; ++ __entry->start = start; ++ __entry->end = end; ++ ), ++ ++ TP_printk("mt_mod %p, (NULL), SNULL, %lu, %lu,", ++ __entry->mt, ++ (unsigned long) __entry->start, ++ (unsigned long) __entry->end ++ ) ++); ++ ++TRACE_EVENT(vma_store, ++ TP_PROTO(struct maple_tree *mt, struct vm_area_struct *vma), ++ ++ TP_ARGS(mt, vma), ++ ++ TP_STRUCT__entry( ++ __field(struct maple_tree *, mt) ++ __field(struct vm_area_struct *, vma) ++ __field(unsigned long, vm_start) ++ __field(unsigned long, vm_end) ++ ), ++ ++ TP_fast_assign( ++ __entry->mt = mt; ++ __entry->vma = vma; ++ __entry->vm_start = vma->vm_start; ++ __entry->vm_end = vma->vm_end - 1; ++ ), ++ ++ TP_printk("mt_mod %p, (%p), STORE, %lu, %lu,", ++ __entry->mt, __entry->vma, ++ (unsigned long) __entry->vm_start, ++ (unsigned long) __entry->vm_end ++ ) ++); ++ ++ ++TRACE_EVENT(exit_mmap, ++ TP_PROTO(struct mm_struct *mm), ++ ++ TP_ARGS(mm), ++ ++ TP_STRUCT__entry( ++ __field(struct mm_struct *, mm) ++ __field(struct maple_tree *, mt) ++ ), ++ ++ TP_fast_assign( ++ __entry->mm = mm; ++ __entry->mt = &mm->mm_mt; ++ ), ++ ++ TP_printk("mt_mod %p, DESTROY\n", ++ __entry->mt ++ ) ++); ++ + #endif + + /* This part must be outside protection */ +diff --git a/init/main.c b/init/main.c +index 1fe7942f5d4a..df800fc61b2a 100644 +--- a/init/main.c ++++ b/init/main.c +@@ -117,6 +117,7 @@ static int kernel_init(void *); + + extern void init_IRQ(void); + extern void radix_tree_init(void); ++extern void maple_tree_init(void); + + /* + * Debug helper: via this flag we know that we are in 'early bootup code' +@@ -1002,6 +1003,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) + "Interrupts were enabled *very* early, fixing it\n")) + local_irq_disable(); + radix_tree_init(); ++ maple_tree_init(); + + /* + * Set up housekeeping before setting up workqueues to allow the unbound +diff --git a/ipc/shm.c b/ipc/shm.c +index b3048ebd5c31..7d86f058fb86 100644 +--- a/ipc/shm.c ++++ b/ipc/shm.c +@@ -1721,7 +1721,7 @@ long ksys_shmdt(char __user *shmaddr) + #ifdef CONFIG_MMU + loff_t size = 0; + struct file *file; +- struct vm_area_struct *next; ++ VMA_ITERATOR(vmi, mm, addr); + #endif + + if (addr & ~PAGE_MASK) +@@ -1751,12 +1751,9 @@ long ksys_shmdt(char __user *shmaddr) + * match the usual checks anyway. So assume all vma's are + * above the starting address given. + */ +- vma = find_vma(mm, addr); + + #ifdef CONFIG_MMU +- while (vma) { +- next = vma->vm_next; +- ++ for_each_vma(vmi, vma) { + /* + * Check if the starting address would match, i.e. it's + * a fragment created by mprotect() and/or munmap(), or it +@@ -1774,6 +1771,7 @@ long ksys_shmdt(char __user *shmaddr) + file = vma->vm_file; + size = i_size_read(file_inode(vma->vm_file)); + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); ++ mas_pause(&vmi.mas); + /* + * We discovered the size of the shm segment, so + * break out of here and fall through to the next +@@ -1781,10 +1779,9 @@ long ksys_shmdt(char __user *shmaddr) + * searching for matching vma's. + */ + retval = 0; +- vma = next; ++ vma = vma_next(&vmi); + break; + } +- vma = next; + } + + /* +@@ -1794,17 +1791,19 @@ long ksys_shmdt(char __user *shmaddr) + */ + size = PAGE_ALIGN(size); + while (vma && (loff_t)(vma->vm_end - addr) <= size) { +- next = vma->vm_next; +- + /* finding a matching vma now does not alter retval */ + if ((vma->vm_ops == &shm_vm_ops) && + ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && +- (vma->vm_file == file)) ++ (vma->vm_file == file)) { + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); +- vma = next; ++ mas_pause(&vmi.mas); ++ } ++ ++ vma = vma_next(&vmi); + } + + #else /* CONFIG_MMU */ ++ vma = vma_lookup(mm, addr); + /* under NOMMU conditions, the exact address to be destroyed must be + * given + */ +diff --git a/kernel/acct.c b/kernel/acct.c +index 13706356ec54..62200d799b9b 100644 +--- a/kernel/acct.c ++++ b/kernel/acct.c +@@ -555,15 +555,14 @@ void acct_collect(long exitcode, int group_dead) + unsigned long vsize = 0; + + if (group_dead && current->mm) { ++ struct mm_struct *mm = current->mm; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + +- mmap_read_lock(current->mm); +- vma = current->mm->mmap; +- while (vma) { ++ mmap_read_lock(mm); ++ for_each_vma(vmi, vma) + vsize += vma->vm_end - vma->vm_start; +- vma = vma->vm_next; +- } +- mmap_read_unlock(current->mm); ++ mmap_read_unlock(mm); + } + + spin_lock_irq(¤t->sighand->siglock); +diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c +index 8c921799def4..1c8debd42dc9 100644 +--- a/kernel/bpf/task_iter.c ++++ b/kernel/bpf/task_iter.c +@@ -299,8 +299,8 @@ struct bpf_iter_seq_task_vma_info { + }; + + enum bpf_task_vma_iter_find_op { +- task_vma_iter_first_vma, /* use mm->mmap */ +- task_vma_iter_next_vma, /* use curr_vma->vm_next */ ++ task_vma_iter_first_vma, /* use find_vma() with addr 0 */ ++ task_vma_iter_next_vma, /* use vma_next() with curr_vma */ + task_vma_iter_find_vma, /* use find_vma() to find next vma */ + }; + +@@ -400,10 +400,10 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) + + switch (op) { + case task_vma_iter_first_vma: +- curr_vma = curr_task->mm->mmap; ++ curr_vma = find_vma(curr_task->mm, 0); + break; + case task_vma_iter_next_vma: +- curr_vma = curr_vma->vm_next; ++ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); + break; + case task_vma_iter_find_vma: + /* We dropped mmap_lock so it is necessary to use find_vma +@@ -417,7 +417,7 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) + if (curr_vma && + curr_vma->vm_start == info->prev_vm_start && + curr_vma->vm_end == info->prev_vm_end) +- curr_vma = curr_vma->vm_next; ++ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); + break; + } + if (!curr_vma) { +diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c +index 7beceb447211..d5e9ccde3ab8 100644 +--- a/kernel/debug/debug_core.c ++++ b/kernel/debug/debug_core.c +@@ -50,7 +50,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -283,17 +282,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr) + if (!CACHE_FLUSH_IS_SAFE) + return; + +- if (current->mm) { +- int i; +- +- for (i = 0; i < VMACACHE_SIZE; i++) { +- if (!current->vmacache.vmas[i]) +- continue; +- flush_cache_range(current->vmacache.vmas[i], +- addr, addr + BREAK_INSTR_SIZE); +- } +- } +- + /* Force flush instruction cache if it was outside the mm */ + flush_icache_range(addr, addr + BREAK_INSTR_SIZE); + } +diff --git a/kernel/events/core.c b/kernel/events/core.c +index ff4bffc502c6..7a23df62d2e4 100644 +--- a/kernel/events/core.c ++++ b/kernel/events/core.c +@@ -10238,8 +10238,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, + struct perf_addr_filter_range *fr) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + +diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c +index 2eaa327f8158..401bc2d24ce0 100644 +--- a/kernel/events/uprobes.c ++++ b/kernel/events/uprobes.c +@@ -349,9 +349,10 @@ static bool valid_ref_ctr_vma(struct uprobe *uprobe, + static struct vm_area_struct * + find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *tmp; + +- for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) ++ for_each_vma(vmi, tmp) + if (valid_ref_ctr_vma(uprobe, tmp)) + return tmp; + +@@ -1231,11 +1232,12 @@ int uprobe_apply(struct inode *inode, loff_t offset, + + static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + int err = 0; + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long vaddr; + loff_t offset; + +@@ -1983,9 +1985,10 @@ bool uprobe_deny_signal(void) + + static void mmf_recalc_uprobes(struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!valid_vma(vma, false)) + continue; + /* +diff --git a/kernel/fork.c b/kernel/fork.c +index b15aaae04403..1add76edca04 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -43,7 +43,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -479,7 +478,6 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) + */ + *new = data_race(*orig); + INIT_LIST_HEAD(&new->anon_vma_chain); +- new->vm_next = new->vm_prev = NULL; + dup_anon_vma_name(orig, new); + } + return new; +@@ -584,11 +582,12 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) + static __latent_entropy int dup_mmap(struct mm_struct *mm, + struct mm_struct *oldmm) + { +- struct vm_area_struct *mpnt, *tmp, *prev, **pprev; +- struct rb_node **rb_link, *rb_parent; ++ struct vm_area_struct *mpnt, *tmp; + int retval; +- unsigned long charge; ++ unsigned long charge = 0; + LIST_HEAD(uf); ++ MA_STATE(old_mas, &oldmm->mm_mt, 0, 0); ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + uprobe_start_dup_mmap(); + if (mmap_write_lock_killable(oldmm)) { +@@ -610,16 +609,16 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + mm->exec_vm = oldmm->exec_vm; + mm->stack_vm = oldmm->stack_vm; + +- rb_link = &mm->mm_rb.rb_node; +- rb_parent = NULL; +- pprev = &mm->mmap; + retval = ksm_fork(mm, oldmm); + if (retval) + goto out; + khugepaged_fork(mm, oldmm); + +- prev = NULL; +- for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { ++ retval = mas_expected_entries(&mas, oldmm->map_count); ++ if (retval) ++ goto out; ++ ++ mas_for_each(&old_mas, mpnt, ULONG_MAX) { + struct file *file; + + if (mpnt->vm_flags & VM_DONTCOPY) { +@@ -633,7 +632,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + */ + if (fatal_signal_pending(current)) { + retval = -EINTR; +- goto out; ++ goto loop_out; + } + if (mpnt->vm_flags & VM_ACCOUNT) { + unsigned long len = vma_pages(mpnt); +@@ -686,17 +685,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + if (is_vm_hugetlb_page(tmp)) + reset_vma_resv_huge_pages(tmp); + +- /* +- * Link in the new vma and copy the page table entries. +- */ +- *pprev = tmp; +- pprev = &tmp->vm_next; +- tmp->vm_prev = prev; +- prev = tmp; +- +- __vma_link_rb(mm, tmp, rb_link, rb_parent); +- rb_link = &tmp->vm_rb.rb_right; +- rb_parent = &tmp->vm_rb; ++ /* Link the vma into the MT */ ++ mas.index = tmp->vm_start; ++ mas.last = tmp->vm_end - 1; ++ mas_store(&mas, tmp); ++ if (mas_is_err(&mas)) ++ goto fail_nomem_mas_store; + + mm->map_count++; + if (!(tmp->vm_flags & VM_WIPEONFORK)) +@@ -706,10 +700,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + tmp->vm_ops->open(tmp); + + if (retval) +- goto out; ++ goto loop_out; + } + /* a new mm has just been created */ + retval = arch_dup_mmap(oldmm, mm); ++loop_out: ++ mas_destroy(&mas); + out: + mmap_write_unlock(mm); + flush_tlb_mm(oldmm); +@@ -718,6 +714,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + fail_uprobe_end: + uprobe_end_dup_mmap(); + return retval; ++ ++fail_nomem_mas_store: ++ unlink_anon_vmas(tmp); + fail_nomem_anon_vma_fork: + mpol_put(vma_policy(tmp)); + fail_nomem_policy: +@@ -725,7 +724,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + fail_nomem: + retval = -ENOMEM; + vm_unacct_memory(charge); +- goto out; ++ goto loop_out; + } + + static inline int mm_alloc_pgd(struct mm_struct *mm) +@@ -1113,9 +1112,8 @@ static void mm_init_uprobes_state(struct mm_struct *mm) + static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, + struct user_namespace *user_ns) + { +- mm->mmap = NULL; +- mm->mm_rb = RB_ROOT; +- mm->vmacache_seqnum = 0; ++ mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); ++ mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); + atomic_set(&mm->mm_users, 1); + atomic_set(&mm->mm_count, 1); + seqcount_init(&mm->write_protect_seq); +@@ -1291,13 +1289,16 @@ int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) + /* Forbid mm->exe_file change if old file still mapped. */ + old_exe_file = get_mm_exe_file(mm); + if (old_exe_file) { ++ VMA_ITERATOR(vmi, mm, 0); + mmap_read_lock(mm); +- for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + if (path_equal(&vma->vm_file->f_path, +- &old_exe_file->f_path)) ++ &old_exe_file->f_path)) { + ret = -EBUSY; ++ break; ++ } + } + mmap_read_unlock(mm); + fput(old_exe_file); +@@ -1573,9 +1574,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) + if (!oldmm) + return 0; + +- /* initialize the new vmacache entries */ +- vmacache_flush(tsk); +- + if (clone_flags & CLONE_VM) { + mmget(oldmm); + mm = oldmm; +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index c4fd77e7e8f3..67b8e93ea742 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -2783,6 +2783,7 @@ static void task_numa_work(struct callback_head *work) + struct task_struct *p = current; + struct mm_struct *mm = p->mm; + u64 runtime = p->se.sum_exec_runtime; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + struct vm_area_struct *vma; + unsigned long start, end; + unsigned long nr_pte_updates = 0; +@@ -2839,13 +2840,16 @@ static void task_numa_work(struct callback_head *work) + + if (!mmap_read_trylock(mm)) + return; +- vma = find_vma(mm, start); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) { + reset_ptenuma_scan(p); + start = 0; +- vma = mm->mmap; ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + } +- for (; vma; vma = vma->vm_next) { ++ ++ for (; vma; vma = mas_find(&mas, ULONG_MAX)) { + if (!vma_migratable(vma) || !vma_policy_mof(vma) || + is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + continue; +diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug +index d3e5f36bb01e..72cef567aae6 100644 +--- a/lib/Kconfig.debug ++++ b/lib/Kconfig.debug +@@ -814,13 +814,12 @@ config DEBUG_VM + + If unsure, say N. + +-config DEBUG_VM_VMACACHE +- bool "Debug VMA caching" ++config DEBUG_VM_MAPLE_TREE ++ bool "Debug VM maple trees" + depends on DEBUG_VM ++ select DEBUG_MAPLE_TREE + help +- Enable this to turn on VMA caching debug information. Doing so +- can cause significant overhead, so only enable it in non-production +- environments. ++ Enable VM maple tree debugging information and extra validations. + + If unsure, say N. + +@@ -1637,6 +1636,14 @@ config BUG_ON_DATA_CORRUPTION + + If unsure, say N. + ++config DEBUG_MAPLE_TREE ++ bool "Debug maple trees" ++ depends on DEBUG_KERNEL ++ help ++ Enable maple tree debugging information and extra validations. ++ ++ If unsure, say N. ++ + endmenu + + config DEBUG_CREDENTIALS +diff --git a/lib/Makefile b/lib/Makefile +index ffabc30a27d4..6dc0d6f8e57d 100644 +--- a/lib/Makefile ++++ b/lib/Makefile +@@ -29,7 +29,7 @@ endif + + lib-y := ctype.o string.o vsprintf.o cmdline.o \ + rbtree.o radix-tree.o timerqueue.o xarray.o \ +- idr.o extable.o irq_regs.o argv_split.o \ ++ maple_tree.o idr.o extable.o irq_regs.o argv_split.o \ + flex_proportions.o ratelimit.o show_mem.o \ + is_single_threaded.o plist.o decompress.o kobject_uevent.o \ + earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ +diff --git a/lib/maple_tree.c b/lib/maple_tree.c +new file mode 100644 +index 000000000000..e1743803c851 +--- /dev/null ++++ b/lib/maple_tree.c +@@ -0,0 +1,7130 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++/* ++ * Maple Tree implementation ++ * Copyright (c) 2018-2022 Oracle Corporation ++ * Authors: Liam R. Howlett ++ * Matthew Wilcox ++ */ ++ ++/* ++ * DOC: Interesting implementation details of the Maple Tree ++ * ++ * Each node type has a number of slots for entries and a number of slots for ++ * pivots. In the case of dense nodes, the pivots are implied by the position ++ * and are simply the slot index + the minimum of the node. ++ * ++ * In regular B-Tree terms, pivots are called keys. The term pivot is used to ++ * indicate that the tree is specifying ranges, Pivots may appear in the ++ * subtree with an entry attached to the value where as keys are unique to a ++ * specific position of a B-tree. Pivot values are inclusive of the slot with ++ * the same index. ++ * ++ * ++ * The following illustrates the layout of a range64 nodes slots and pivots. ++ * ++ * ++ * Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 | ++ * ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ++ * │ │ │ │ │ │ │ │ └─ Implied maximum ++ * │ │ │ │ │ │ │ └─ Pivot 14 ++ * │ │ │ │ │ │ └─ Pivot 13 ++ * │ │ │ │ │ └─ Pivot 12 ++ * │ │ │ │ └─ Pivot 11 ++ * │ │ │ └─ Pivot 2 ++ * │ │ └─ Pivot 1 ++ * │ └─ Pivot 0 ++ * └─ Implied minimum ++ * ++ * Slot contents: ++ * Internal (non-leaf) nodes contain pointers to other nodes. ++ * Leaf nodes contain entries. ++ * ++ * The location of interest is often referred to as an offset. All offsets have ++ * a slot, but the last offset has an implied pivot from the node above (or ++ * UINT_MAX for the root node. ++ * ++ * Ranges complicate certain write activities. When modifying any of ++ * the B-tree variants, it is known that one entry will either be added or ++ * deleted. When modifying the Maple Tree, one store operation may overwrite ++ * the entire data set, or one half of the tree, or the middle half of the tree. ++ * ++ */ ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define CREATE_TRACE_POINTS ++#include ++ ++#define MA_ROOT_PARENT 1 ++ ++/* ++ * Maple state flags ++ * * MA_STATE_BULK - Bulk insert mode ++ * * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert ++ * * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation ++ */ ++#define MA_STATE_BULK 1 ++#define MA_STATE_REBALANCE 2 ++#define MA_STATE_PREALLOC 4 ++ ++#define ma_parent_ptr(x) ((struct maple_pnode *)(x)) ++#define ma_mnode_ptr(x) ((struct maple_node *)(x)) ++#define ma_enode_ptr(x) ((struct maple_enode *)(x)) ++static struct kmem_cache *maple_node_cache; ++ ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++static const unsigned long mt_max[] = { ++ [maple_dense] = MAPLE_NODE_SLOTS, ++ [maple_leaf_64] = ULONG_MAX, ++ [maple_range_64] = ULONG_MAX, ++ [maple_arange_64] = ULONG_MAX, ++}; ++#define mt_node_max(x) mt_max[mte_node_type(x)] ++#endif ++ ++static const unsigned char mt_slots[] = { ++ [maple_dense] = MAPLE_NODE_SLOTS, ++ [maple_leaf_64] = MAPLE_RANGE64_SLOTS, ++ [maple_range_64] = MAPLE_RANGE64_SLOTS, ++ [maple_arange_64] = MAPLE_ARANGE64_SLOTS, ++}; ++#define mt_slot_count(x) mt_slots[mte_node_type(x)] ++ ++static const unsigned char mt_pivots[] = { ++ [maple_dense] = 0, ++ [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1, ++ [maple_range_64] = MAPLE_RANGE64_SLOTS - 1, ++ [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1, ++}; ++#define mt_pivot_count(x) mt_pivots[mte_node_type(x)] ++ ++static const unsigned char mt_min_slots[] = { ++ [maple_dense] = MAPLE_NODE_SLOTS / 2, ++ [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, ++ [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, ++ [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1, ++}; ++#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] ++ ++#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) ++#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) ++ ++struct maple_big_node { ++ struct maple_pnode *parent; ++ unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; ++ union { ++ struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; ++ struct { ++ unsigned long padding[MAPLE_BIG_NODE_GAPS]; ++ unsigned long gap[MAPLE_BIG_NODE_GAPS]; ++ }; ++ }; ++ unsigned char b_end; ++ enum maple_type type; ++}; ++ ++/* ++ * The maple_subtree_state is used to build a tree to replace a segment of an ++ * existing tree in a more atomic way. Any walkers of the older tree will hit a ++ * dead node and restart on updates. ++ */ ++struct maple_subtree_state { ++ struct ma_state *orig_l; /* Original left side of subtree */ ++ struct ma_state *orig_r; /* Original right side of subtree */ ++ struct ma_state *l; /* New left side of subtree */ ++ struct ma_state *m; /* New middle of subtree (rare) */ ++ struct ma_state *r; /* New right side of subtree */ ++ struct ma_topiary *free; /* nodes to be freed */ ++ struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ ++ struct maple_big_node *bn; ++}; ++ ++/* Functions */ ++static inline struct maple_node *mt_alloc_one(gfp_t gfp) ++{ ++ return kmem_cache_alloc(maple_node_cache, gfp | __GFP_ZERO); ++} ++ ++static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes) ++{ ++ return kmem_cache_alloc_bulk(maple_node_cache, gfp | __GFP_ZERO, size, ++ nodes); ++} ++ ++static inline void mt_free_bulk(size_t size, void __rcu **nodes) ++{ ++ kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes); ++} ++ ++static void mt_free_rcu(struct rcu_head *head) ++{ ++ struct maple_node *node = container_of(head, struct maple_node, rcu); ++ ++ kmem_cache_free(maple_node_cache, node); ++} ++ ++/* ++ * ma_free_rcu() - Use rcu callback to free a maple node ++ * @node: The node to free ++ * ++ * The maple tree uses the parent pointer to indicate this node is no longer in ++ * use and will be freed. ++ */ ++static void ma_free_rcu(struct maple_node *node) ++{ ++ node->parent = ma_parent_ptr(node); ++ call_rcu(&node->rcu, mt_free_rcu); ++} ++ ++static unsigned int mt_height(const struct maple_tree *mt) ++{ ++ return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; ++} ++ ++static void mas_set_height(struct ma_state *mas) ++{ ++ unsigned int new_flags = mas->tree->ma_flags; ++ ++ new_flags &= ~MT_FLAGS_HEIGHT_MASK; ++ BUG_ON(mas->depth > MAPLE_HEIGHT_MAX); ++ new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET; ++ mas->tree->ma_flags = new_flags; ++} ++ ++static unsigned int mas_mt_height(struct ma_state *mas) ++{ ++ return mt_height(mas->tree); ++} ++ ++static inline enum maple_type mte_node_type(const struct maple_enode *entry) ++{ ++ return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) & ++ MAPLE_NODE_TYPE_MASK; ++} ++ ++static inline bool ma_is_dense(const enum maple_type type) ++{ ++ return type < maple_leaf_64; ++} ++ ++static inline bool ma_is_leaf(const enum maple_type type) ++{ ++ return type < maple_range_64; ++} ++ ++static inline bool mte_is_leaf(const struct maple_enode *entry) ++{ ++ return ma_is_leaf(mte_node_type(entry)); ++} ++ ++/* ++ * We also reserve values with the bottom two bits set to '10' which are ++ * below 4096 ++ */ ++static inline bool mt_is_reserved(const void *entry) ++{ ++ return ((unsigned long)entry < MAPLE_RESERVED_RANGE) && ++ xa_is_internal(entry); ++} ++ ++static inline void mas_set_err(struct ma_state *mas, long err) ++{ ++ mas->node = MA_ERROR(err); ++} ++ ++static inline bool mas_is_ptr(struct ma_state *mas) ++{ ++ return mas->node == MAS_ROOT; ++} ++ ++static inline bool mas_is_start(struct ma_state *mas) ++{ ++ return mas->node == MAS_START; ++} ++ ++bool mas_is_err(struct ma_state *mas) ++{ ++ return xa_is_err(mas->node); ++} ++ ++static inline bool mas_searchable(struct ma_state *mas) ++{ ++ if (mas_is_none(mas)) ++ return false; ++ ++ if (mas_is_ptr(mas)) ++ return false; ++ ++ return true; ++} ++ ++static inline struct maple_node *mte_to_node(const struct maple_enode *entry) ++{ ++ return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK); ++} ++ ++/* ++ * mte_to_mat() - Convert a maple encoded node to a maple topiary node. ++ * @entry: The maple encoded node ++ * ++ * Return: a maple topiary pointer ++ */ ++static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) ++{ ++ return (struct maple_topiary *) ++ ((unsigned long)entry & ~MAPLE_NODE_MASK); ++} ++ ++/* ++ * mas_mn() - Get the maple state node. ++ * @mas: The maple state ++ * ++ * Return: the maple node (not encoded - bare pointer). ++ */ ++static inline struct maple_node *mas_mn(const struct ma_state *mas) ++{ ++ return mte_to_node(mas->node); ++} ++ ++/* ++ * mte_set_node_dead() - Set a maple encoded node as dead. ++ * @mn: The maple encoded node. ++ */ ++static inline void mte_set_node_dead(struct maple_enode *mn) ++{ ++ mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn)); ++ smp_wmb(); /* Needed for RCU */ ++} ++ ++/* Bit 1 indicates the root is a node */ ++#define MAPLE_ROOT_NODE 0x02 ++/* maple_type stored bit 3-6 */ ++#define MAPLE_ENODE_TYPE_SHIFT 0x03 ++/* Bit 2 means a NULL somewhere below */ ++#define MAPLE_ENODE_NULL 0x04 ++ ++static inline struct maple_enode *mt_mk_node(const struct maple_node *node, ++ enum maple_type type) ++{ ++ return (void *)((unsigned long)node | ++ (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); ++} ++ ++static inline void *mte_mk_root(const struct maple_enode *node) ++{ ++ return (void *)((unsigned long)node | MAPLE_ROOT_NODE); ++} ++ ++static inline void *mte_safe_root(const struct maple_enode *node) ++{ ++ return (void *)((unsigned long)node & ~MAPLE_ROOT_NODE); ++} ++ ++static inline void mte_set_full(const struct maple_enode *node) ++{ ++ node = (void *)((unsigned long)node & ~MAPLE_ENODE_NULL); ++} ++ ++static inline void mte_clear_full(const struct maple_enode *node) ++{ ++ node = (void *)((unsigned long)node | MAPLE_ENODE_NULL); ++} ++ ++static inline bool ma_is_root(struct maple_node *node) ++{ ++ return ((unsigned long)node->parent & MA_ROOT_PARENT); ++} ++ ++static inline bool mte_is_root(const struct maple_enode *node) ++{ ++ return ma_is_root(mte_to_node(node)); ++} ++ ++static inline bool mas_is_root_limits(const struct ma_state *mas) ++{ ++ return !mas->min && mas->max == ULONG_MAX; ++} ++ ++static inline bool mt_is_alloc(struct maple_tree *mt) ++{ ++ return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); ++} ++ ++/* ++ * The Parent Pointer ++ * Excluding root, the parent pointer is 256B aligned like all other tree nodes. ++ * When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16 ++ * bit values need an extra bit to store the offset. This extra bit comes from ++ * a reuse of the last bit in the node type. This is possible by using bit 1 to ++ * indicate if bit 2 is part of the type or the slot. ++ * ++ * Note types: ++ * 0x??1 = Root ++ * 0x?00 = 16 bit nodes ++ * 0x010 = 32 bit nodes ++ * 0x110 = 64 bit nodes ++ * ++ * Slot size and alignment ++ * 0b??1 : Root ++ * 0b?00 : 16 bit values, type in 0-1, slot in 2-7 ++ * 0b010 : 32 bit values, type in 0-2, slot in 3-7 ++ * 0b110 : 64 bit values, type in 0-2, slot in 3-7 ++ */ ++ ++#define MAPLE_PARENT_ROOT 0x01 ++ ++#define MAPLE_PARENT_SLOT_SHIFT 0x03 ++#define MAPLE_PARENT_SLOT_MASK 0xF8 ++ ++#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02 ++#define MAPLE_PARENT_16B_SLOT_MASK 0xFC ++ ++#define MAPLE_PARENT_RANGE64 0x06 ++#define MAPLE_PARENT_RANGE32 0x04 ++#define MAPLE_PARENT_NOT_RANGE16 0x02 ++ ++/* ++ * mte_parent_shift() - Get the parent shift for the slot storage. ++ * @parent: The parent pointer cast as an unsigned long ++ * Return: The shift into that pointer to the star to of the slot ++ */ ++static inline unsigned long mte_parent_shift(unsigned long parent) ++{ ++ /* Note bit 1 == 0 means 16B */ ++ if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) ++ return MAPLE_PARENT_SLOT_SHIFT; ++ ++ return MAPLE_PARENT_16B_SLOT_SHIFT; ++} ++ ++/* ++ * mte_parent_slot_mask() - Get the slot mask for the parent. ++ * @parent: The parent pointer cast as an unsigned long. ++ * Return: The slot mask for that parent. ++ */ ++static inline unsigned long mte_parent_slot_mask(unsigned long parent) ++{ ++ /* Note bit 1 == 0 means 16B */ ++ if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) ++ return MAPLE_PARENT_SLOT_MASK; ++ ++ return MAPLE_PARENT_16B_SLOT_MASK; ++} ++ ++/* ++ * mas_parent_enum() - Return the maple_type of the parent from the stored ++ * parent type. ++ * @mas: The maple state ++ * @node: The maple_enode to extract the parent's enum ++ * Return: The node->parent maple_type ++ */ ++static inline ++enum maple_type mte_parent_enum(struct maple_enode *p_enode, ++ struct maple_tree *mt) ++{ ++ unsigned long p_type; ++ ++ p_type = (unsigned long)p_enode; ++ if (p_type & MAPLE_PARENT_ROOT) ++ return 0; /* Validated in the caller. */ ++ ++ p_type &= MAPLE_NODE_MASK; ++ p_type = p_type & ~(MAPLE_PARENT_ROOT | mte_parent_slot_mask(p_type)); ++ ++ switch (p_type) { ++ case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */ ++ if (mt_is_alloc(mt)) ++ return maple_arange_64; ++ return maple_range_64; ++ } ++ ++ return 0; ++} ++ ++static inline ++enum maple_type mas_parent_enum(struct ma_state *mas, struct maple_enode *enode) ++{ ++ return mte_parent_enum(ma_enode_ptr(mte_to_node(enode)->parent), mas->tree); ++} ++ ++/* ++ * mte_set_parent() - Set the parent node and encode the slot ++ * @enode: The encoded maple node. ++ * @parent: The encoded maple node that is the parent of @enode. ++ * @slot: The slot that @enode resides in @parent. ++ * ++ * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the ++ * parent type. ++ */ ++static inline ++void mte_set_parent(struct maple_enode *enode, const struct maple_enode *parent, ++ unsigned char slot) ++{ ++ unsigned long val = (unsigned long) parent; ++ unsigned long shift; ++ unsigned long type; ++ enum maple_type p_type = mte_node_type(parent); ++ ++ BUG_ON(p_type == maple_dense); ++ BUG_ON(p_type == maple_leaf_64); ++ ++ switch (p_type) { ++ case maple_range_64: ++ case maple_arange_64: ++ shift = MAPLE_PARENT_SLOT_SHIFT; ++ type = MAPLE_PARENT_RANGE64; ++ break; ++ default: ++ case maple_dense: ++ case maple_leaf_64: ++ shift = type = 0; ++ break; ++ } ++ ++ val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ ++ val |= (slot << shift) | type; ++ mte_to_node(enode)->parent = ma_parent_ptr(val); ++} ++ ++/* ++ * mte_parent_slot() - get the parent slot of @enode. ++ * @enode: The encoded maple node. ++ * ++ * Return: The slot in the parent node where @enode resides. ++ */ ++static inline unsigned int mte_parent_slot(const struct maple_enode *enode) ++{ ++ unsigned long val = (unsigned long) mte_to_node(enode)->parent; ++ ++ /* Root. */ ++ if (val & 1) ++ return 0; ++ ++ /* ++ * Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost ++ * by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT ++ */ ++ return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val); ++} ++ ++/* ++ * mte_parent() - Get the parent of @node. ++ * @node: The encoded maple node. ++ * ++ * Return: The parent maple node. ++ */ ++static inline struct maple_node *mte_parent(const struct maple_enode *enode) ++{ ++ return (void *)((unsigned long) ++ (mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK); ++} ++ ++/* ++ * ma_dead_node() - check if the @enode is dead. ++ * @enode: The encoded maple node ++ * ++ * Return: true if dead, false otherwise. ++ */ ++static inline bool ma_dead_node(const struct maple_node *node) ++{ ++ struct maple_node *parent = (void *)((unsigned long) ++ node->parent & ~MAPLE_NODE_MASK); ++ ++ return (parent == node); ++} ++/* ++ * mte_dead_node() - check if the @enode is dead. ++ * @enode: The encoded maple node ++ * ++ * Return: true if dead, false otherwise. ++ */ ++static inline bool mte_dead_node(const struct maple_enode *enode) ++{ ++ struct maple_node *parent, *node; ++ ++ node = mte_to_node(enode); ++ parent = mte_parent(enode); ++ return (parent == node); ++} ++ ++/* ++ * mas_allocated() - Get the number of nodes allocated in a maple state. ++ * @mas: The maple state ++ * ++ * The ma_state alloc member is overloaded to hold a pointer to the first ++ * allocated node or to the number of requested nodes to allocate. If bit 0 is ++ * set, then the alloc contains the number of requested nodes. If there is an ++ * allocated node, then the total allocated nodes is in that node. ++ * ++ * Return: The total number of nodes allocated ++ */ ++static inline unsigned long mas_allocated(const struct ma_state *mas) ++{ ++ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) ++ return 0; ++ ++ return mas->alloc->total; ++} ++ ++/* ++ * mas_set_alloc_req() - Set the requested number of allocations. ++ * @mas: the maple state ++ * @count: the number of allocations. ++ * ++ * The requested number of allocations is either in the first allocated node, ++ * located in @mas->alloc->request_count, or directly in @mas->alloc if there is ++ * no allocated node. Set the request either in the node or do the necessary ++ * encoding to store in @mas->alloc directly. ++ */ ++static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) ++{ ++ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) { ++ if (!count) ++ mas->alloc = NULL; ++ else ++ mas->alloc = (struct maple_alloc *)(((count) << 1U) | 1U); ++ return; ++ } ++ ++ mas->alloc->request_count = count; ++} ++ ++/* ++ * mas_alloc_req() - get the requested number of allocations. ++ * @mas: The maple state ++ * ++ * The alloc count is either stored directly in @mas, or in ++ * @mas->alloc->request_count if there is at least one node allocated. Decode ++ * the request count if it's stored directly in @mas->alloc. ++ * ++ * Return: The allocation request count. ++ */ ++static inline unsigned int mas_alloc_req(const struct ma_state *mas) ++{ ++ if ((unsigned long)mas->alloc & 0x1) ++ return (unsigned long)(mas->alloc) >> 1; ++ else if (mas->alloc) ++ return mas->alloc->request_count; ++ return 0; ++} ++ ++/* ++ * ma_pivots() - Get a pointer to the maple node pivots. ++ * @node - the maple node ++ * @type - the node type ++ * ++ * Return: A pointer to the maple node pivots ++ */ ++static inline unsigned long *ma_pivots(struct maple_node *node, ++ enum maple_type type) ++{ ++ switch (type) { ++ case maple_arange_64: ++ return node->ma64.pivot; ++ case maple_range_64: ++ case maple_leaf_64: ++ return node->mr64.pivot; ++ case maple_dense: ++ return NULL; ++ } ++ return NULL; ++} ++ ++/* ++ * ma_gaps() - Get a pointer to the maple node gaps. ++ * @node - the maple node ++ * @type - the node type ++ * ++ * Return: A pointer to the maple node gaps ++ */ ++static inline unsigned long *ma_gaps(struct maple_node *node, ++ enum maple_type type) ++{ ++ switch (type) { ++ case maple_arange_64: ++ return node->ma64.gap; ++ case maple_range_64: ++ case maple_leaf_64: ++ case maple_dense: ++ return NULL; ++ } ++ return NULL; ++} ++ ++/* ++ * mte_pivot() - Get the pivot at @piv of the maple encoded node. ++ * @mn: The maple encoded node. ++ * @piv: The pivot. ++ * ++ * Return: the pivot at @piv of @mn. ++ */ ++static inline unsigned long mte_pivot(const struct maple_enode *mn, ++ unsigned char piv) ++{ ++ struct maple_node *node = mte_to_node(mn); ++ ++ if (piv >= mt_pivots[piv]) { ++ WARN_ON(1); ++ return 0; ++ } ++ switch (mte_node_type(mn)) { ++ case maple_arange_64: ++ return node->ma64.pivot[piv]; ++ case maple_range_64: ++ case maple_leaf_64: ++ return node->mr64.pivot[piv]; ++ case maple_dense: ++ return 0; ++ } ++ return 0; ++} ++ ++/* ++ * mas_safe_pivot() - get the pivot at @piv or mas->max. ++ * @mas: The maple state ++ * @pivots: The pointer to the maple node pivots ++ * @piv: The pivot to fetch ++ * @type: The maple node type ++ * ++ * Return: The pivot at @piv within the limit of the @pivots array, @mas->max ++ * otherwise. ++ */ ++static inline unsigned long ++mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, ++ unsigned char piv, enum maple_type type) ++{ ++ if (piv >= mt_pivots[type]) ++ return mas->max; ++ ++ return pivots[piv]; ++} ++ ++/* ++ * mas_safe_min() - Return the minimum for a given offset. ++ * @mas: The maple state ++ * @pivots: The pointer to the maple node pivots ++ * @offset: The offset into the pivot array ++ * ++ * Return: The minimum range value that is contained in @offset. ++ */ ++static inline unsigned long ++mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset) ++{ ++ if (likely(offset)) ++ return pivots[offset - 1] + 1; ++ ++ return mas->min; ++} ++ ++/* ++ * mas_logical_pivot() - Get the logical pivot of a given offset. ++ * @mas: The maple state ++ * @pivots: The pointer to the maple node pivots ++ * @offset: The offset into the pivot array ++ * @type: The maple node type ++ * ++ * When there is no value at a pivot (beyond the end of the data), then the ++ * pivot is actually @mas->max. ++ * ++ * Return: the logical pivot of a given @offset. ++ */ ++static inline unsigned long ++mas_logical_pivot(struct ma_state *mas, unsigned long *pivots, ++ unsigned char offset, enum maple_type type) ++{ ++ unsigned long lpiv = mas_safe_pivot(mas, pivots, offset, type); ++ ++ if (likely(lpiv)) ++ return lpiv; ++ ++ if (likely(offset)) ++ return mas->max; ++ ++ return lpiv; ++} ++ ++/* ++ * mte_set_pivot() - Set a pivot to a value in an encoded maple node. ++ * @mn: The encoded maple node ++ * @piv: The pivot offset ++ * @val: The value of the pivot ++ */ ++static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, ++ unsigned long val) ++{ ++ struct maple_node *node = mte_to_node(mn); ++ enum maple_type type = mte_node_type(mn); ++ ++ BUG_ON(piv >= mt_pivots[type]); ++ switch (type) { ++ default: ++ case maple_range_64: ++ case maple_leaf_64: ++ node->mr64.pivot[piv] = val; ++ break; ++ case maple_arange_64: ++ node->ma64.pivot[piv] = val; ++ break; ++ case maple_dense: ++ break; ++ } ++ ++} ++ ++/* ++ * ma_slots() - Get a pointer to the maple node slots. ++ * @mn: The maple node ++ * @mt: The maple node type ++ * ++ * Return: A pointer to the maple node slots ++ */ ++static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) ++{ ++ switch (mt) { ++ default: ++ case maple_arange_64: ++ return mn->ma64.slot; ++ case maple_range_64: ++ case maple_leaf_64: ++ return mn->mr64.slot; ++ case maple_dense: ++ return mn->slot; ++ } ++} ++ ++static inline bool mt_locked(const struct maple_tree *mt) ++{ ++ return mt_external_lock(mt) ? mt_lock_is_held(mt) : ++ lockdep_is_held(&mt->ma_lock); ++} ++ ++static inline void *mt_slot(const struct maple_tree *mt, ++ void __rcu **slots, unsigned char offset) ++{ ++ return rcu_dereference_check(slots[offset], mt_locked(mt)); ++} ++ ++/* ++ * mas_slot_locked() - Get the slot value when holding the maple tree lock. ++ * @mas: The maple state ++ * @slots: The pointer to the slots ++ * @offset: The offset into the slots array to fetch ++ * ++ * Return: The entry stored in @slots at the @offset. ++ */ ++static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots, ++ unsigned char offset) ++{ ++ return rcu_dereference_protected(slots[offset], mt_locked(mas->tree)); ++} ++ ++/* ++ * mas_slot() - Get the slot value when not holding the maple tree lock. ++ * @mas: The maple state ++ * @slots: The pointer to the slots ++ * @offset: The offset into the slots array to fetch ++ * ++ * Return: The entry stored in @slots at the @offset ++ */ ++static inline void *mas_slot(struct ma_state *mas, void __rcu **slots, ++ unsigned char offset) ++{ ++ return mt_slot(mas->tree, slots, offset); ++} ++ ++/* ++ * mas_root() - Get the maple tree root. ++ * @mas: The maple state. ++ * ++ * Return: The pointer to the root of the tree ++ */ ++static inline void *mas_root(struct ma_state *mas) ++{ ++ return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree)); ++} ++ ++static inline void *mt_root_locked(struct maple_tree *mt) ++{ ++ return rcu_dereference_protected(mt->ma_root, mt_locked(mt)); ++} ++ ++/* ++ * mas_root_locked() - Get the maple tree root when holding the maple tree lock. ++ * @mas: The maple state. ++ * ++ * Return: The pointer to the root of the tree ++ */ ++static inline void *mas_root_locked(struct ma_state *mas) ++{ ++ return mt_root_locked(mas->tree); ++} ++ ++static inline struct maple_metadata *ma_meta(struct maple_node *mn, ++ enum maple_type mt) ++{ ++ switch (mt) { ++ case maple_arange_64: ++ return &mn->ma64.meta; ++ default: ++ return &mn->mr64.meta; ++ } ++} ++ ++/* ++ * ma_set_meta() - Set the metadata information of a node. ++ * @mn: The maple node ++ * @mt: The maple node type ++ * @offset: The offset of the highest sub-gap in this node. ++ * @end: The end of the data in this node. ++ */ ++static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt, ++ unsigned char offset, unsigned char end) ++{ ++ struct maple_metadata *meta = ma_meta(mn, mt); ++ ++ meta->gap = offset; ++ meta->end = end; ++} ++ ++/* ++ * ma_meta_end() - Get the data end of a node from the metadata ++ * @mn: The maple node ++ * @mt: The maple node type ++ */ ++static inline unsigned char ma_meta_end(struct maple_node *mn, ++ enum maple_type mt) ++{ ++ struct maple_metadata *meta = ma_meta(mn, mt); ++ ++ return meta->end; ++} ++ ++/* ++ * ma_meta_gap() - Get the largest gap location of a node from the metadata ++ * @mn: The maple node ++ * @mt: The maple node type ++ */ ++static inline unsigned char ma_meta_gap(struct maple_node *mn, ++ enum maple_type mt) ++{ ++ BUG_ON(mt != maple_arange_64); ++ ++ return mn->ma64.meta.gap; ++} ++ ++/* ++ * ma_set_meta_gap() - Set the largest gap location in a nodes metadata ++ * @mn: The maple node ++ * @mn: The maple node type ++ * @offset: The location of the largest gap. ++ */ ++static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, ++ unsigned char offset) ++{ ++ ++ struct maple_metadata *meta = ma_meta(mn, mt); ++ ++ meta->gap = offset; ++} ++ ++/* ++ * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. ++ * @mat - the ma_topiary, a linked list of dead nodes. ++ * @dead_enode - the node to be marked as dead and added to the tail of the list ++ * ++ * Add the @dead_enode to the linked list in @mat. ++ */ ++static inline void mat_add(struct ma_topiary *mat, ++ struct maple_enode *dead_enode) ++{ ++ mte_set_node_dead(dead_enode); ++ mte_to_mat(dead_enode)->next = NULL; ++ if (!mat->tail) { ++ mat->tail = mat->head = dead_enode; ++ return; ++ } ++ ++ mte_to_mat(mat->tail)->next = dead_enode; ++ mat->tail = dead_enode; ++} ++ ++static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); ++static inline void mas_free(struct ma_state *mas, struct maple_enode *used); ++ ++/* ++ * mas_mat_free() - Free all nodes in a dead list. ++ * @mas - the maple state ++ * @mat - the ma_topiary linked list of dead nodes to free. ++ * ++ * Free walk a dead list. ++ */ ++static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat) ++{ ++ struct maple_enode *next; ++ ++ while (mat->head) { ++ next = mte_to_mat(mat->head)->next; ++ mas_free(mas, mat->head); ++ mat->head = next; ++ } ++} ++ ++/* ++ * mas_mat_destroy() - Free all nodes and subtrees in a dead list. ++ * @mas - the maple state ++ * @mat - the ma_topiary linked list of dead nodes to free. ++ * ++ * Destroy walk a dead list. ++ */ ++static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat) ++{ ++ struct maple_enode *next; ++ ++ while (mat->head) { ++ next = mte_to_mat(mat->head)->next; ++ mte_destroy_walk(mat->head, mat->mtree); ++ mat->head = next; ++ } ++} ++/* ++ * mas_descend() - Descend into the slot stored in the ma_state. ++ * @mas - the maple state. ++ * ++ * Note: Not RCU safe, only use in write side or debug code. ++ */ ++static inline void mas_descend(struct ma_state *mas) ++{ ++ enum maple_type type; ++ unsigned long *pivots; ++ struct maple_node *node; ++ void __rcu **slots; ++ ++ node = mas_mn(mas); ++ type = mte_node_type(mas->node); ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ ++ if (mas->offset) ++ mas->min = pivots[mas->offset - 1] + 1; ++ mas->max = mas_safe_pivot(mas, pivots, mas->offset, type); ++ mas->node = mas_slot(mas, slots, mas->offset); ++} ++ ++/* ++ * mte_set_gap() - Set a maple node gap. ++ * @mn: The encoded maple node ++ * @gap: The offset of the gap to set ++ * @val: The gap value ++ */ ++static inline void mte_set_gap(const struct maple_enode *mn, ++ unsigned char gap, unsigned long val) ++{ ++ switch (mte_node_type(mn)) { ++ default: ++ break; ++ case maple_arange_64: ++ mte_to_node(mn)->ma64.gap[gap] = val; ++ break; ++ } ++} ++ ++/* ++ * mas_ascend() - Walk up a level of the tree. ++ * @mas: The maple state ++ * ++ * Sets the @mas->max and @mas->min to the correct values when walking up. This ++ * may cause several levels of walking up to find the correct min and max. ++ * May find a dead node which will cause a premature return. ++ * Return: 1 on dead node, 0 otherwise ++ */ ++static int mas_ascend(struct ma_state *mas) ++{ ++ struct maple_enode *p_enode; /* parent enode. */ ++ struct maple_enode *a_enode; /* ancestor enode. */ ++ struct maple_node *a_node; /* ancestor node. */ ++ struct maple_node *p_node; /* parent node. */ ++ unsigned char a_slot; ++ enum maple_type a_type; ++ unsigned long min, max; ++ unsigned long *pivots; ++ unsigned char offset; ++ bool set_max = false, set_min = false; ++ ++ a_node = mas_mn(mas); ++ if (ma_is_root(a_node)) { ++ mas->offset = 0; ++ return 0; ++ } ++ ++ p_node = mte_parent(mas->node); ++ if (unlikely(a_node == p_node)) ++ return 1; ++ a_type = mas_parent_enum(mas, mas->node); ++ offset = mte_parent_slot(mas->node); ++ a_enode = mt_mk_node(p_node, a_type); ++ ++ /* Check to make sure all parent information is still accurate */ ++ if (p_node != mte_parent(mas->node)) ++ return 1; ++ ++ mas->node = a_enode; ++ mas->offset = offset; ++ ++ if (mte_is_root(a_enode)) { ++ mas->max = ULONG_MAX; ++ mas->min = 0; ++ return 0; ++ } ++ ++ min = 0; ++ max = ULONG_MAX; ++ do { ++ p_enode = a_enode; ++ a_type = mas_parent_enum(mas, p_enode); ++ a_node = mte_parent(p_enode); ++ a_slot = mte_parent_slot(p_enode); ++ pivots = ma_pivots(a_node, a_type); ++ a_enode = mt_mk_node(a_node, a_type); ++ ++ if (!set_min && a_slot) { ++ set_min = true; ++ min = pivots[a_slot - 1] + 1; ++ } ++ ++ if (!set_max && a_slot < mt_pivots[a_type]) { ++ set_max = true; ++ max = pivots[a_slot]; ++ } ++ ++ if (unlikely(ma_dead_node(a_node))) ++ return 1; ++ ++ if (unlikely(ma_is_root(a_node))) ++ break; ++ ++ } while (!set_min || !set_max); ++ ++ mas->max = max; ++ mas->min = min; ++ return 0; ++} ++ ++/* ++ * mas_pop_node() - Get a previously allocated maple node from the maple state. ++ * @mas: The maple state ++ * ++ * Return: A pointer to a maple node. ++ */ ++static inline struct maple_node *mas_pop_node(struct ma_state *mas) ++{ ++ struct maple_alloc *ret, *node = mas->alloc; ++ unsigned long total = mas_allocated(mas); ++ ++ /* nothing or a request pending. */ ++ if (unlikely(!total)) ++ return NULL; ++ ++ if (total == 1) { ++ /* single allocation in this ma_state */ ++ mas->alloc = NULL; ++ ret = node; ++ goto single_node; ++ } ++ ++ if (!node->node_count) { ++ /* Single allocation in this node. */ ++ mas->alloc = node->slot[0]; ++ node->slot[0] = NULL; ++ mas->alloc->total = node->total - 1; ++ ret = node; ++ goto new_head; ++ } ++ ++ node->total--; ++ ret = node->slot[node->node_count]; ++ node->slot[node->node_count--] = NULL; ++ ++single_node: ++new_head: ++ ret->total = 0; ++ ret->node_count = 0; ++ if (ret->request_count) { ++ mas_set_alloc_req(mas, ret->request_count + 1); ++ ret->request_count = 0; ++ } ++ return (struct maple_node *)ret; ++} ++ ++/* ++ * mas_push_node() - Push a node back on the maple state allocation. ++ * @mas: The maple state ++ * @used: The used maple node ++ * ++ * Stores the maple node back into @mas->alloc for reuse. Updates allocated and ++ * requested node count as necessary. ++ */ ++static inline void mas_push_node(struct ma_state *mas, struct maple_node *used) ++{ ++ struct maple_alloc *reuse = (struct maple_alloc *)used; ++ struct maple_alloc *head = mas->alloc; ++ unsigned long count; ++ unsigned int requested = mas_alloc_req(mas); ++ ++ memset(reuse, 0, sizeof(*reuse)); ++ count = mas_allocated(mas); ++ ++ if (count && (head->node_count < MAPLE_ALLOC_SLOTS - 1)) { ++ if (head->slot[0]) ++ head->node_count++; ++ head->slot[head->node_count] = reuse; ++ head->total++; ++ goto done; ++ } ++ ++ reuse->total = 1; ++ if ((head) && !((unsigned long)head & 0x1)) { ++ head->request_count = 0; ++ reuse->slot[0] = head; ++ reuse->total += head->total; ++ } ++ ++ mas->alloc = reuse; ++done: ++ if (requested > 1) ++ mas_set_alloc_req(mas, requested - 1); ++} ++ ++/* ++ * mas_alloc_nodes() - Allocate nodes into a maple state ++ * @mas: The maple state ++ * @gfp: The GFP Flags ++ */ ++static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) ++{ ++ struct maple_alloc *node; ++ struct maple_alloc **nodep = &mas->alloc; ++ unsigned long allocated = mas_allocated(mas); ++ unsigned long success = allocated; ++ unsigned int requested = mas_alloc_req(mas); ++ unsigned int count; ++ void **slots = NULL; ++ unsigned int max_req = 0; ++ ++ if (!requested) ++ return; ++ ++ mas_set_alloc_req(mas, 0); ++ if (mas->mas_flags & MA_STATE_PREALLOC) { ++ if (allocated) ++ return; ++ WARN_ON(!allocated); ++ } ++ ++ if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS - 1) { ++ node = (struct maple_alloc *)mt_alloc_one(gfp); ++ if (!node) ++ goto nomem_one; ++ ++ if (allocated) ++ node->slot[0] = mas->alloc; ++ ++ success++; ++ mas->alloc = node; ++ requested--; ++ } ++ ++ node = mas->alloc; ++ while (requested) { ++ max_req = MAPLE_ALLOC_SLOTS; ++ if (node->slot[0]) { ++ unsigned int offset = node->node_count + 1; ++ ++ slots = (void **)&node->slot[offset]; ++ max_req -= offset; ++ } else { ++ slots = (void **)&node->slot; ++ } ++ ++ max_req = min(requested, max_req); ++ count = mt_alloc_bulk(gfp, max_req, slots); ++ if (!count) ++ goto nomem_bulk; ++ ++ node->node_count += count; ++ /* zero indexed. */ ++ if (slots == (void **)&node->slot) ++ node->node_count--; ++ ++ success += count; ++ nodep = &node->slot[0]; ++ node = *nodep; ++ requested -= count; ++ } ++ mas->alloc->total = success; ++ return; ++ ++nomem_bulk: ++ /* Clean up potential freed allocations on bulk failure */ ++ memset(slots, 0, max_req * sizeof(unsigned long)); ++nomem_one: ++ mas_set_alloc_req(mas, requested); ++ if (mas->alloc && !(((unsigned long)mas->alloc & 0x1))) ++ mas->alloc->total = success; ++ mas_set_err(mas, -ENOMEM); ++ return; ++ ++} ++ ++/* ++ * mas_free() - Free an encoded maple node ++ * @mas: The maple state ++ * @used: The encoded maple node to free. ++ * ++ * Uses rcu free if necessary, pushes @used back on the maple state allocations ++ * otherwise. ++ */ ++static inline void mas_free(struct ma_state *mas, struct maple_enode *used) ++{ ++ struct maple_node *tmp = mte_to_node(used); ++ ++ if (mt_in_rcu(mas->tree)) ++ ma_free_rcu(tmp); ++ else ++ mas_push_node(mas, tmp); ++} ++ ++/* ++ * mas_node_count() - Check if enough nodes are allocated and request more if ++ * there is not enough nodes. ++ * @mas: The maple state ++ * @count: The number of nodes needed ++ * @gfp: the gfp flags ++ */ ++static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp) ++{ ++ unsigned long allocated = mas_allocated(mas); ++ ++ if (allocated < count) { ++ mas_set_alloc_req(mas, count - allocated); ++ mas_alloc_nodes(mas, gfp); ++ } ++} ++ ++/* ++ * mas_node_count() - Check if enough nodes are allocated and request more if ++ * there is not enough nodes. ++ * @mas: The maple state ++ * @count: The number of nodes needed ++ * ++ * Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags. ++ */ ++static void mas_node_count(struct ma_state *mas, int count) ++{ ++ return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN); ++} ++ ++/* ++ * mas_start() - Sets up maple state for operations. ++ * @mas: The maple state. ++ * ++ * If mas->node == MAS_START, then set the min, max, depth, and offset to ++ * defaults. ++ * ++ * Return: ++ * - If mas->node is an error or not MAS_START, return NULL. ++ * - If it's an empty tree: NULL & mas->node == MAS_NONE ++ * - If it's a single entry: The entry & mas->node == MAS_ROOT ++ * - If it's a tree: NULL & mas->node == safe root node. ++ */ ++static inline struct maple_enode *mas_start(struct ma_state *mas) ++{ ++ if (likely(mas_is_start(mas))) { ++ struct maple_enode *root; ++ ++ mas->node = MAS_NONE; ++ mas->min = 0; ++ mas->max = ULONG_MAX; ++ mas->depth = 0; ++ mas->offset = 0; ++ ++ root = mas_root(mas); ++ /* Tree with nodes */ ++ if (likely(xa_is_node(root))) { ++ mas->node = mte_safe_root(root); ++ return NULL; ++ } ++ ++ /* empty tree */ ++ if (unlikely(!root)) { ++ mas->offset = MAPLE_NODE_SLOTS; ++ return NULL; ++ } ++ ++ /* Single entry tree */ ++ mas->node = MAS_ROOT; ++ mas->offset = MAPLE_NODE_SLOTS; ++ ++ /* Single entry tree. */ ++ if (mas->index > 0) ++ return NULL; ++ ++ return root; ++ } ++ ++ return NULL; ++} ++ ++/* ++ * ma_data_end() - Find the end of the data in a node. ++ * @node: The maple node ++ * @type: The maple node type ++ * @pivots: The array of pivots in the node ++ * @max: The maximum value in the node ++ * ++ * Uses metadata to find the end of the data when possible. ++ * Return: The zero indexed last slot with data (may be null). ++ */ ++static inline unsigned char ma_data_end(struct maple_node *node, ++ enum maple_type type, ++ unsigned long *pivots, ++ unsigned long max) ++{ ++ unsigned char offset; ++ ++ if (type == maple_arange_64) ++ return ma_meta_end(node, type); ++ ++ offset = mt_pivots[type] - 1; ++ if (likely(!pivots[offset])) ++ return ma_meta_end(node, type); ++ ++ if (likely(pivots[offset] == max)) ++ return offset; ++ ++ return mt_pivots[type]; ++} ++ ++/* ++ * mas_data_end() - Find the end of the data (slot). ++ * @mas: the maple state ++ * ++ * This method is optimized to check the metadata of a node if the node type ++ * supports data end metadata. ++ * ++ * Return: The zero indexed last slot with data (may be null). ++ */ ++static inline unsigned char mas_data_end(struct ma_state *mas) ++{ ++ enum maple_type type; ++ struct maple_node *node; ++ unsigned char offset; ++ unsigned long *pivots; ++ ++ type = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ if (type == maple_arange_64) ++ return ma_meta_end(node, type); ++ ++ pivots = ma_pivots(node, type); ++ offset = mt_pivots[type] - 1; ++ if (likely(!pivots[offset])) ++ return ma_meta_end(node, type); ++ ++ if (likely(pivots[offset] == mas->max)) ++ return offset; ++ ++ return mt_pivots[type]; ++} ++ ++/* ++ * mas_leaf_max_gap() - Returns the largest gap in a leaf node ++ * @mas - the maple state ++ * ++ * Return: The maximum gap in the leaf. ++ */ ++static unsigned long mas_leaf_max_gap(struct ma_state *mas) ++{ ++ enum maple_type mt; ++ unsigned long pstart, gap, max_gap; ++ struct maple_node *mn; ++ unsigned long *pivots; ++ void __rcu **slots; ++ unsigned char i; ++ unsigned char max_piv; ++ ++ mt = mte_node_type(mas->node); ++ mn = mas_mn(mas); ++ slots = ma_slots(mn, mt); ++ max_gap = 0; ++ if (unlikely(ma_is_dense(mt))) { ++ gap = 0; ++ for (i = 0; i < mt_slots[mt]; i++) { ++ if (slots[i]) { ++ if (gap > max_gap) ++ max_gap = gap; ++ gap = 0; ++ } else { ++ gap++; ++ } ++ } ++ if (gap > max_gap) ++ max_gap = gap; ++ return max_gap; ++ } ++ ++ /* ++ * Check the first implied pivot optimizes the loop below and slot 1 may ++ * be skipped if there is a gap in slot 0. ++ */ ++ pivots = ma_pivots(mn, mt); ++ if (likely(!slots[0])) { ++ max_gap = pivots[0] - mas->min + 1; ++ i = 2; ++ } else { ++ i = 1; ++ } ++ ++ /* reduce max_piv as the special case is checked before the loop */ ++ max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1; ++ /* ++ * Check end implied pivot which can only be a gap on the right most ++ * node. ++ */ ++ if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) { ++ gap = ULONG_MAX - pivots[max_piv]; ++ if (gap > max_gap) ++ max_gap = gap; ++ } ++ ++ for (; i <= max_piv; i++) { ++ /* data == no gap. */ ++ if (likely(slots[i])) ++ continue; ++ ++ pstart = pivots[i - 1]; ++ gap = pivots[i] - pstart; ++ if (gap > max_gap) ++ max_gap = gap; ++ ++ /* There cannot be two gaps in a row. */ ++ i++; ++ } ++ return max_gap; ++} ++ ++/* ++ * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) ++ * @node: The maple node ++ * @gaps: The pointer to the gaps ++ * @mt: The maple node type ++ * @*off: Pointer to store the offset location of the gap. ++ * ++ * Uses the metadata data end to scan backwards across set gaps. ++ * ++ * Return: The maximum gap value ++ */ ++static inline unsigned long ++ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, ++ unsigned char *off) ++{ ++ unsigned char offset, i; ++ unsigned long max_gap = 0; ++ ++ i = offset = ma_meta_end(node, mt); ++ do { ++ if (gaps[i] > max_gap) { ++ max_gap = gaps[i]; ++ offset = i; ++ } ++ } while (i--); ++ ++ *off = offset; ++ return max_gap; ++} ++ ++/* ++ * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. ++ * @mas: The maple state. ++ * ++ * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap. ++ * ++ * Return: The gap value. ++ */ ++static inline unsigned long mas_max_gap(struct ma_state *mas) ++{ ++ unsigned long *gaps; ++ unsigned char offset; ++ enum maple_type mt; ++ struct maple_node *node; ++ ++ mt = mte_node_type(mas->node); ++ if (ma_is_leaf(mt)) ++ return mas_leaf_max_gap(mas); ++ ++ node = mas_mn(mas); ++ offset = ma_meta_gap(node, mt); ++ if (offset == MAPLE_ARANGE64_META_MAX) ++ return 0; ++ ++ gaps = ma_gaps(node, mt); ++ return gaps[offset]; ++} ++ ++/* ++ * mas_parent_gap() - Set the parent gap and any gaps above, as needed ++ * @mas: The maple state ++ * @offset: The gap offset in the parent to set ++ * @new: The new gap value. ++ * ++ * Set the parent gap then continue to set the gap upwards, using the metadata ++ * of the parent to see if it is necessary to check the node above. ++ */ ++static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, ++ unsigned long new) ++{ ++ unsigned long meta_gap = 0; ++ struct maple_node *pnode; ++ struct maple_enode *penode; ++ unsigned long *pgaps; ++ unsigned char meta_offset; ++ enum maple_type pmt; ++ ++ pnode = mte_parent(mas->node); ++ pmt = mas_parent_enum(mas, mas->node); ++ penode = mt_mk_node(pnode, pmt); ++ pgaps = ma_gaps(pnode, pmt); ++ ++ascend: ++ meta_offset = ma_meta_gap(pnode, pmt); ++ if (meta_offset == MAPLE_ARANGE64_META_MAX) ++ meta_gap = 0; ++ else ++ meta_gap = pgaps[meta_offset]; ++ ++ pgaps[offset] = new; ++ ++ if (meta_gap == new) ++ return; ++ ++ if (offset != meta_offset) { ++ if (meta_gap > new) ++ return; ++ ++ ma_set_meta_gap(pnode, pmt, offset); ++ } else if (new < meta_gap) { ++ meta_offset = 15; ++ new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); ++ ma_set_meta_gap(pnode, pmt, meta_offset); ++ } ++ ++ if (ma_is_root(pnode)) ++ return; ++ ++ /* Go to the parent node. */ ++ pnode = mte_parent(penode); ++ pmt = mas_parent_enum(mas, penode); ++ pgaps = ma_gaps(pnode, pmt); ++ offset = mte_parent_slot(penode); ++ penode = mt_mk_node(pnode, pmt); ++ goto ascend; ++} ++ ++/* ++ * mas_update_gap() - Update a nodes gaps and propagate up if necessary. ++ * @mas - the maple state. ++ */ ++static inline void mas_update_gap(struct ma_state *mas) ++{ ++ unsigned char pslot; ++ unsigned long p_gap; ++ unsigned long max_gap; ++ ++ if (!mt_is_alloc(mas->tree)) ++ return; ++ ++ if (mte_is_root(mas->node)) ++ return; ++ ++ max_gap = mas_max_gap(mas); ++ ++ pslot = mte_parent_slot(mas->node); ++ p_gap = ma_gaps(mte_parent(mas->node), ++ mas_parent_enum(mas, mas->node))[pslot]; ++ ++ if (p_gap != max_gap) ++ mas_parent_gap(mas, pslot, max_gap); ++} ++ ++/* ++ * mas_adopt_children() - Set the parent pointer of all nodes in @parent to ++ * @parent with the slot encoded. ++ * @mas - the maple state (for the tree) ++ * @parent - the maple encoded node containing the children. ++ */ ++static inline void mas_adopt_children(struct ma_state *mas, ++ struct maple_enode *parent) ++{ ++ enum maple_type type = mte_node_type(parent); ++ struct maple_node *node = mas_mn(mas); ++ void __rcu **slots = ma_slots(node, type); ++ unsigned long *pivots = ma_pivots(node, type); ++ struct maple_enode *child; ++ unsigned char offset; ++ ++ offset = ma_data_end(node, type, pivots, mas->max); ++ do { ++ child = mas_slot_locked(mas, slots, offset); ++ mte_set_parent(child, parent, offset); ++ } while (offset--); ++} ++ ++/* ++ * mas_replace() - Replace a maple node in the tree with mas->node. Uses the ++ * parent encoding to locate the maple node in the tree. ++ * @mas - the ma_state to use for operations. ++ * @advanced - boolean to adopt the child nodes and free the old node (false) or ++ * leave the node (true) and handle the adoption and free elsewhere. ++ */ ++static inline void mas_replace(struct ma_state *mas, bool advanced) ++ __must_hold(mas->tree->lock) ++{ ++ struct maple_node *mn = mas_mn(mas); ++ struct maple_enode *old_enode; ++ unsigned char offset = 0; ++ void __rcu **slots = NULL; ++ ++ if (ma_is_root(mn)) { ++ old_enode = mas_root_locked(mas); ++ } else { ++ offset = mte_parent_slot(mas->node); ++ slots = ma_slots(mte_parent(mas->node), ++ mas_parent_enum(mas, mas->node)); ++ old_enode = mas_slot_locked(mas, slots, offset); ++ } ++ ++ if (!advanced && !mte_is_leaf(mas->node)) ++ mas_adopt_children(mas, mas->node); ++ ++ if (mte_is_root(mas->node)) { ++ mn->parent = ma_parent_ptr( ++ ((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); ++ mas_set_height(mas); ++ } else { ++ rcu_assign_pointer(slots[offset], mas->node); ++ } ++ ++ if (!advanced) ++ mas_free(mas, old_enode); ++} ++ ++/* ++ * mas_new_child() - Find the new child of a node. ++ * @mas: the maple state ++ * @child: the maple state to store the child. ++ */ ++static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) ++ __must_hold(mas->tree->lock) ++{ ++ enum maple_type mt; ++ unsigned char offset; ++ unsigned char end; ++ unsigned long *pivots; ++ struct maple_enode *entry; ++ struct maple_node *node; ++ void __rcu **slots; ++ ++ mt = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ end = ma_data_end(node, mt, pivots, mas->max); ++ for (offset = mas->offset; offset <= end; offset++) { ++ entry = mas_slot_locked(mas, slots, offset); ++ if (mte_parent(entry) == node) { ++ *child = *mas; ++ mas->offset = offset + 1; ++ child->offset = offset; ++ mas_descend(child); ++ child->offset = 0; ++ return true; ++ } ++ } ++ return false; ++} ++ ++/* ++ * mab_shift_right() - Shift the data in mab right. Note, does not clean out the ++ * old data or set b_node->b_end. ++ * @b_node: the maple_big_node ++ * @shift: the shift count ++ */ ++static inline void mab_shift_right(struct maple_big_node *b_node, ++ unsigned char shift) ++{ ++ unsigned long size = b_node->b_end * sizeof(unsigned long); ++ ++ memmove(b_node->pivot + shift, b_node->pivot, size); ++ memmove(b_node->slot + shift, b_node->slot, size); ++ if (b_node->type == maple_arange_64) ++ memmove(b_node->gap + shift, b_node->gap, size); ++} ++ ++/* ++ * mab_middle_node() - Check if a middle node is needed (unlikely) ++ * @b_node: the maple_big_node that contains the data. ++ * @size: the amount of data in the b_node ++ * @split: the potential split location ++ * @slot_count: the size that can be stored in a single node being considered. ++ * ++ * Return: true if a middle node is required. ++ */ ++static inline bool mab_middle_node(struct maple_big_node *b_node, int split, ++ unsigned char slot_count) ++{ ++ unsigned char size = b_node->b_end; ++ ++ if (size >= 2 * slot_count) ++ return true; ++ ++ if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * mab_no_null_split() - ensure the split doesn't fall on a NULL ++ * @b_node: the maple_big_node with the data ++ * @split: the suggested split location ++ * @slot_count: the number of slots in the node being considered. ++ * ++ * Return: the split location. ++ */ ++static inline int mab_no_null_split(struct maple_big_node *b_node, ++ unsigned char split, unsigned char slot_count) ++{ ++ if (!b_node->slot[split]) { ++ /* ++ * If the split is less than the max slot && the right side will ++ * still be sufficient, then increment the split on NULL. ++ */ ++ if ((split < slot_count - 1) && ++ (b_node->b_end - split) > (mt_min_slots[b_node->type])) ++ split++; ++ else ++ split--; ++ } ++ return split; ++} ++ ++/* ++ * mab_calc_split() - Calculate the split location and if there needs to be two ++ * splits. ++ * @bn: The maple_big_node with the data ++ * @mid_split: The second split, if required. 0 otherwise. ++ * ++ * Return: The first split location. The middle split is set in @mid_split. ++ */ ++static inline int mab_calc_split(struct ma_state *mas, ++ struct maple_big_node *bn, unsigned char *mid_split, unsigned long min) ++{ ++ unsigned char b_end = bn->b_end; ++ int split = b_end / 2; /* Assume equal split. */ ++ unsigned char slot_min, slot_count = mt_slots[bn->type]; ++ ++ /* ++ * To support gap tracking, all NULL entries are kept together and a node cannot ++ * end on a NULL entry, with the exception of the left-most leaf. The ++ * limitation means that the split of a node must be checked for this condition ++ * and be able to put more data in one direction or the other. ++ */ ++ if (unlikely((mas->mas_flags & MA_STATE_BULK))) { ++ *mid_split = 0; ++ split = b_end - mt_min_slots[bn->type]; ++ ++ if (!ma_is_leaf(bn->type)) ++ return split; ++ ++ mas->mas_flags |= MA_STATE_REBALANCE; ++ if (!bn->slot[split]) ++ split--; ++ return split; ++ } ++ ++ /* ++ * Although extremely rare, it is possible to enter what is known as the 3-way ++ * split scenario. The 3-way split comes about by means of a store of a range ++ * that overwrites the end and beginning of two full nodes. The result is a set ++ * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can ++ * also be located in different parent nodes which are also full. This can ++ * carry upwards all the way to the root in the worst case. ++ */ ++ if (unlikely(mab_middle_node(bn, split, slot_count))) { ++ split = b_end / 3; ++ *mid_split = split * 2; ++ } else { ++ slot_min = mt_min_slots[bn->type]; ++ ++ *mid_split = 0; ++ /* ++ * Avoid having a range less than the slot count unless it ++ * causes one node to be deficient. ++ * NOTE: mt_min_slots is 1 based, b_end and split are zero. ++ */ ++ while (((bn->pivot[split] - min) < slot_count - 1) && ++ (split < slot_count - 1) && (b_end - split > slot_min)) ++ split++; ++ } ++ ++ /* Avoid ending a node on a NULL entry */ ++ split = mab_no_null_split(bn, split, slot_count); ++ if (!(*mid_split)) ++ return split; ++ ++ *mid_split = mab_no_null_split(bn, *mid_split, slot_count); ++ ++ return split; ++} ++ ++/* ++ * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node ++ * and set @b_node->b_end to the next free slot. ++ * @mas: The maple state ++ * @mas_start: The starting slot to copy ++ * @mas_end: The end slot to copy (inclusively) ++ * @b_node: The maple_big_node to place the data ++ * @mab_start: The starting location in maple_big_node to store the data. ++ */ ++static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, ++ unsigned char mas_end, struct maple_big_node *b_node, ++ unsigned char mab_start) ++{ ++ enum maple_type mt; ++ struct maple_node *node; ++ void __rcu **slots; ++ unsigned long *pivots, *gaps; ++ int i = mas_start, j = mab_start; ++ unsigned char piv_end; ++ ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ pivots = ma_pivots(node, mt); ++ if (!i) { ++ b_node->pivot[j] = pivots[i++]; ++ if (unlikely(i > mas_end)) ++ goto complete; ++ j++; ++ } ++ ++ piv_end = min(mas_end, mt_pivots[mt]); ++ for (; i < piv_end; i++, j++) { ++ b_node->pivot[j] = pivots[i]; ++ if (unlikely(!b_node->pivot[j])) ++ break; ++ ++ if (unlikely(mas->max == b_node->pivot[j])) ++ goto complete; ++ } ++ ++ if (likely(i <= mas_end)) ++ b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); ++ ++complete: ++ b_node->b_end = ++j; ++ j -= mab_start; ++ slots = ma_slots(node, mt); ++ memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); ++ if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) { ++ gaps = ma_gaps(node, mt); ++ memcpy(b_node->gap + mab_start, gaps + mas_start, ++ sizeof(unsigned long) * j); ++ } ++} ++ ++/* ++ * mas_leaf_set_meta() - Set the metadata of a leaf if possible. ++ * @mas: The maple state ++ * @node: The maple node ++ * @pivots: pointer to the maple node pivots ++ * @mt: The maple type ++ * @end: The assumed end ++ * ++ * Note, end may be incremented within this function but not modified at the ++ * source. This is fine since the metadata is the last thing to be stored in a ++ * node during a write. ++ */ ++static inline void mas_leaf_set_meta(struct ma_state *mas, ++ struct maple_node *node, unsigned long *pivots, ++ enum maple_type mt, unsigned char end) ++{ ++ /* There is no room for metadata already */ ++ if (mt_pivots[mt] <= end) ++ return; ++ ++ if (pivots[end] && pivots[end] < mas->max) ++ end++; ++ ++ if (end < mt_slots[mt] - 1) ++ ma_set_meta(node, mt, 0, end); ++} ++ ++/* ++ * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. ++ * @b_node: the maple_big_node that has the data ++ * @mab_start: the start location in @b_node. ++ * @mab_end: The end location in @b_node (inclusively) ++ * @mas: The maple state with the maple encoded node. ++ */ ++static inline void mab_mas_cp(struct maple_big_node *b_node, ++ unsigned char mab_start, unsigned char mab_end, ++ struct ma_state *mas, bool new_max) ++{ ++ int i, j = 0; ++ enum maple_type mt = mte_node_type(mas->node); ++ struct maple_node *node = mte_to_node(mas->node); ++ void __rcu **slots = ma_slots(node, mt); ++ unsigned long *pivots = ma_pivots(node, mt); ++ unsigned long *gaps = NULL; ++ unsigned char end; ++ ++ if (mab_end - mab_start > mt_pivots[mt]) ++ mab_end--; ++ ++ if (!pivots[mt_pivots[mt] - 1]) ++ slots[mt_pivots[mt]] = NULL; ++ ++ i = mab_start; ++ do { ++ pivots[j++] = b_node->pivot[i++]; ++ } while (i <= mab_end && likely(b_node->pivot[i])); ++ ++ memcpy(slots, b_node->slot + mab_start, ++ sizeof(void *) * (i - mab_start)); ++ ++ if (new_max) ++ mas->max = b_node->pivot[i - 1]; ++ ++ end = j - 1; ++ if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { ++ unsigned long max_gap = 0; ++ unsigned char offset = 15; ++ ++ gaps = ma_gaps(node, mt); ++ do { ++ gaps[--j] = b_node->gap[--i]; ++ if (gaps[j] > max_gap) { ++ offset = j; ++ max_gap = gaps[j]; ++ } ++ } while (j); ++ ++ ma_set_meta(node, mt, offset, end); ++ } else { ++ mas_leaf_set_meta(mas, node, pivots, mt, end); ++ } ++} ++ ++/* ++ * mas_descend_adopt() - Descend through a sub-tree and adopt children. ++ * @mas: the maple state with the maple encoded node of the sub-tree. ++ * ++ * Descend through a sub-tree and adopt children who do not have the correct ++ * parents set. Follow the parents which have the correct parents as they are ++ * the new entries which need to be followed to find other incorrectly set ++ * parents. ++ */ ++static inline void mas_descend_adopt(struct ma_state *mas) ++{ ++ struct ma_state list[3], next[3]; ++ int i, n; ++ ++ /* ++ * At each level there may be up to 3 correct parent pointers which indicates ++ * the new nodes which need to be walked to find any new nodes at a lower level. ++ */ ++ ++ for (i = 0; i < 3; i++) { ++ list[i] = *mas; ++ list[i].offset = 0; ++ next[i].offset = 0; ++ } ++ next[0] = *mas; ++ ++ while (!mte_is_leaf(list[0].node)) { ++ n = 0; ++ for (i = 0; i < 3; i++) { ++ if (mas_is_none(&list[i])) ++ continue; ++ ++ if (i && list[i-1].node == list[i].node) ++ continue; ++ ++ while ((n < 3) && (mas_new_child(&list[i], &next[n]))) ++ n++; ++ ++ mas_adopt_children(&list[i], list[i].node); ++ } ++ ++ while (n < 3) ++ next[n++].node = MAS_NONE; ++ ++ /* descend by setting the list to the children */ ++ for (i = 0; i < 3; i++) ++ list[i] = next[i]; ++ } ++} ++ ++/* ++ * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. ++ * @mas: The maple state ++ * @end: The maple node end ++ * @mt: The maple node type ++ */ ++static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, ++ enum maple_type mt) ++{ ++ if (!(mas->mas_flags & MA_STATE_BULK)) ++ return; ++ ++ if (mte_is_root(mas->node)) ++ return; ++ ++ if (end > mt_min_slots[mt]) { ++ mas->mas_flags &= ~MA_STATE_REBALANCE; ++ return; ++ } ++} ++ ++/* ++ * mas_store_b_node() - Store an @entry into the b_node while also copying the ++ * data from a maple encoded node. ++ * @wr_mas: the maple write state ++ * @b_node: the maple_big_node to fill with data ++ * @offset_end: the offset to end copying ++ * ++ * Return: The actual end of the data stored in @b_node ++ */ ++static inline void mas_store_b_node(struct ma_wr_state *wr_mas, ++ struct maple_big_node *b_node, unsigned char offset_end) ++{ ++ unsigned char slot; ++ unsigned char b_end; ++ /* Possible underflow of piv will wrap back to 0 before use. */ ++ unsigned long piv; ++ struct ma_state *mas = wr_mas->mas; ++ ++ b_node->type = wr_mas->type; ++ b_end = 0; ++ slot = mas->offset; ++ if (slot) { ++ /* Copy start data up to insert. */ ++ mas_mab_cp(mas, 0, slot - 1, b_node, 0); ++ b_end = b_node->b_end; ++ piv = b_node->pivot[b_end - 1]; ++ } else ++ piv = mas->min - 1; ++ ++ if (piv + 1 < mas->index) { ++ /* Handle range starting after old range */ ++ b_node->slot[b_end] = wr_mas->content; ++ if (!wr_mas->content) ++ b_node->gap[b_end] = mas->index - 1 - piv; ++ b_node->pivot[b_end++] = mas->index - 1; ++ } ++ ++ /* Store the new entry. */ ++ mas->offset = b_end; ++ b_node->slot[b_end] = wr_mas->entry; ++ b_node->pivot[b_end] = mas->last; ++ ++ /* Appended. */ ++ if (mas->last >= mas->max) ++ goto b_end; ++ ++ /* Handle new range ending before old range ends */ ++ piv = mas_logical_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); ++ if (piv > mas->last) { ++ if (piv == ULONG_MAX) ++ mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type); ++ ++ if (offset_end != slot) ++ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, ++ offset_end); ++ ++ b_node->slot[++b_end] = wr_mas->content; ++ if (!wr_mas->content) ++ b_node->gap[b_end] = piv - mas->last + 1; ++ b_node->pivot[b_end] = piv; ++ } ++ ++ slot = offset_end + 1; ++ if (slot > wr_mas->node_end) ++ goto b_end; ++ ++ /* Copy end data to the end of the node. */ ++ mas_mab_cp(mas, slot, wr_mas->node_end + 1, b_node, ++b_end); ++ b_node->b_end--; ++ return; ++ ++b_end: ++ b_node->b_end = b_end; ++} ++ ++/* ++ * mas_prev_sibling() - Find the previous node with the same parent. ++ * @mas: the maple state ++ * ++ * Return: True if there is a previous sibling, false otherwise. ++ */ ++static inline bool mas_prev_sibling(struct ma_state *mas) ++{ ++ unsigned int p_slot = mte_parent_slot(mas->node); ++ ++ if (mte_is_root(mas->node)) ++ return false; ++ ++ if (!p_slot) ++ return false; ++ ++ mas_ascend(mas); ++ mas->offset = p_slot - 1; ++ mas_descend(mas); ++ return true; ++} ++ ++/* ++ * mas_next_sibling() - Find the next node with the same parent. ++ * @mas: the maple state ++ * ++ * Return: true if there is a next sibling, false otherwise. ++ */ ++static inline bool mas_next_sibling(struct ma_state *mas) ++{ ++ MA_STATE(parent, mas->tree, mas->index, mas->last); ++ ++ if (mte_is_root(mas->node)) ++ return false; ++ ++ parent = *mas; ++ mas_ascend(&parent); ++ parent.offset = mte_parent_slot(mas->node) + 1; ++ if (parent.offset > mas_data_end(&parent)) ++ return false; ++ ++ *mas = parent; ++ mas_descend(mas); ++ return true; ++} ++ ++/* ++ * mte_node_or_node() - Return the encoded node or MAS_NONE. ++ * @enode: The encoded maple node. ++ * ++ * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state. ++ * ++ * Return: @enode or MAS_NONE ++ */ ++static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) ++{ ++ if (enode) ++ return enode; ++ ++ return ma_enode_ptr(MAS_NONE); ++} ++ ++/* ++ * mas_wr_node_walk() - Find the correct offset for the index in the @mas. ++ * @wr_mas: The maple write state ++ * ++ * Uses mas_slot_locked() and does not need to worry about dead nodes. ++ */ ++static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ unsigned char count; ++ unsigned char offset; ++ unsigned long index, min, max; ++ ++ if (unlikely(ma_is_dense(wr_mas->type))) { ++ wr_mas->r_max = wr_mas->r_min = mas->index; ++ mas->offset = mas->index = mas->min; ++ return; ++ } ++ ++ wr_mas->node = mas_mn(wr_mas->mas); ++ wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); ++ count = wr_mas->node_end = ma_data_end(wr_mas->node, wr_mas->type, ++ wr_mas->pivots, mas->max); ++ offset = mas->offset; ++ min = mas_safe_min(mas, wr_mas->pivots, offset); ++ if (unlikely(offset == count)) ++ goto max; ++ ++ max = wr_mas->pivots[offset]; ++ index = mas->index; ++ if (unlikely(index <= max)) ++ goto done; ++ ++ if (unlikely(!max && offset)) ++ goto max; ++ ++ min = max + 1; ++ while (++offset < count) { ++ max = wr_mas->pivots[offset]; ++ if (index <= max) ++ goto done; ++ else if (unlikely(!max)) ++ break; ++ ++ min = max + 1; ++ } ++ ++max: ++ max = mas->max; ++done: ++ wr_mas->r_max = max; ++ wr_mas->r_min = min; ++ wr_mas->offset_end = mas->offset = offset; ++} ++ ++/* ++ * mas_topiary_range() - Add a range of slots to the topiary. ++ * @mas: The maple state ++ * @destroy: The topiary to add the slots (usually destroy) ++ * @start: The starting slot inclusively ++ * @end: The end slot inclusively ++ */ ++static inline void mas_topiary_range(struct ma_state *mas, ++ struct ma_topiary *destroy, unsigned char start, unsigned char end) ++{ ++ void __rcu **slots; ++ unsigned char offset; ++ ++ MT_BUG_ON(mas->tree, mte_is_leaf(mas->node)); ++ slots = ma_slots(mas_mn(mas), mte_node_type(mas->node)); ++ for (offset = start; offset <= end; offset++) { ++ struct maple_enode *enode = mas_slot_locked(mas, slots, offset); ++ ++ if (mte_dead_node(enode)) ++ continue; ++ ++ mat_add(destroy, enode); ++ } ++} ++ ++/* ++ * mast_topiary() - Add the portions of the tree to the removal list; either to ++ * be freed or discarded (destroy walk). ++ * @mast: The maple_subtree_state. ++ */ ++static inline void mast_topiary(struct maple_subtree_state *mast) ++{ ++ MA_WR_STATE(wr_mas, mast->orig_l, NULL); ++ unsigned char r_start, r_end; ++ unsigned char l_start, l_end; ++ void __rcu **l_slots, **r_slots; ++ ++ wr_mas.type = mte_node_type(mast->orig_l->node); ++ mast->orig_l->index = mast->orig_l->last; ++ mas_wr_node_walk(&wr_mas); ++ l_start = mast->orig_l->offset + 1; ++ l_end = mas_data_end(mast->orig_l); ++ r_start = 0; ++ r_end = mast->orig_r->offset; ++ ++ if (r_end) ++ r_end--; ++ ++ l_slots = ma_slots(mas_mn(mast->orig_l), ++ mte_node_type(mast->orig_l->node)); ++ ++ r_slots = ma_slots(mas_mn(mast->orig_r), ++ mte_node_type(mast->orig_r->node)); ++ ++ if ((l_start < l_end) && ++ mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_start))) { ++ l_start++; ++ } ++ ++ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_end))) { ++ if (r_end) ++ r_end--; ++ } ++ ++ if ((l_start > r_end) && (mast->orig_l->node == mast->orig_r->node)) ++ return; ++ ++ /* At the node where left and right sides meet, add the parts between */ ++ if (mast->orig_l->node == mast->orig_r->node) { ++ return mas_topiary_range(mast->orig_l, mast->destroy, ++ l_start, r_end); ++ } ++ ++ /* mast->orig_r is different and consumed. */ ++ if (mte_is_leaf(mast->orig_r->node)) ++ return; ++ ++ if (mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_end))) ++ l_end--; ++ ++ ++ if (l_start <= l_end) ++ mas_topiary_range(mast->orig_l, mast->destroy, l_start, l_end); ++ ++ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_start))) ++ r_start++; ++ ++ if (r_start <= r_end) ++ mas_topiary_range(mast->orig_r, mast->destroy, 0, r_end); ++} ++ ++/* ++ * mast_rebalance_next() - Rebalance against the next node ++ * @mast: The maple subtree state ++ * @old_r: The encoded maple node to the right (next node). ++ */ ++static inline void mast_rebalance_next(struct maple_subtree_state *mast) ++{ ++ unsigned char b_end = mast->bn->b_end; ++ ++ mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), ++ mast->bn, b_end); ++ mast->orig_r->last = mast->orig_r->max; ++} ++ ++/* ++ * mast_rebalance_prev() - Rebalance against the previous node ++ * @mast: The maple subtree state ++ * @old_l: The encoded maple node to the left (previous node) ++ */ ++static inline void mast_rebalance_prev(struct maple_subtree_state *mast) ++{ ++ unsigned char end = mas_data_end(mast->orig_l) + 1; ++ unsigned char b_end = mast->bn->b_end; ++ ++ mab_shift_right(mast->bn, end); ++ mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); ++ mast->l->min = mast->orig_l->min; ++ mast->orig_l->index = mast->orig_l->min; ++ mast->bn->b_end = end + b_end; ++ mast->l->offset += end; ++} ++ ++/* ++ * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring ++ * the node to the right. Checking the nodes to the right then the left at each ++ * level upwards until root is reached. Free and destroy as needed. ++ * Data is copied into the @mast->bn. ++ * @mast: The maple_subtree_state. ++ */ ++static inline ++bool mast_spanning_rebalance(struct maple_subtree_state *mast) ++{ ++ struct ma_state r_tmp = *mast->orig_r; ++ struct ma_state l_tmp = *mast->orig_l; ++ struct maple_enode *ancestor = NULL; ++ unsigned char start, end; ++ unsigned char depth = 0; ++ ++ r_tmp = *mast->orig_r; ++ l_tmp = *mast->orig_l; ++ do { ++ mas_ascend(mast->orig_r); ++ mas_ascend(mast->orig_l); ++ depth++; ++ if (!ancestor && ++ (mast->orig_r->node == mast->orig_l->node)) { ++ ancestor = mast->orig_r->node; ++ end = mast->orig_r->offset - 1; ++ start = mast->orig_l->offset + 1; ++ } ++ ++ if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { ++ if (!ancestor) { ++ ancestor = mast->orig_r->node; ++ start = 0; ++ } ++ ++ mast->orig_r->offset++; ++ do { ++ mas_descend(mast->orig_r); ++ mast->orig_r->offset = 0; ++ depth--; ++ } while (depth); ++ ++ mast_rebalance_next(mast); ++ do { ++ unsigned char l_off = 0; ++ struct maple_enode *child = r_tmp.node; ++ ++ mas_ascend(&r_tmp); ++ if (ancestor == r_tmp.node) ++ l_off = start; ++ ++ if (r_tmp.offset) ++ r_tmp.offset--; ++ ++ if (l_off < r_tmp.offset) ++ mas_topiary_range(&r_tmp, mast->destroy, ++ l_off, r_tmp.offset); ++ ++ if (l_tmp.node != child) ++ mat_add(mast->free, child); ++ ++ } while (r_tmp.node != ancestor); ++ ++ *mast->orig_l = l_tmp; ++ return true; ++ ++ } else if (mast->orig_l->offset != 0) { ++ if (!ancestor) { ++ ancestor = mast->orig_l->node; ++ end = mas_data_end(mast->orig_l); ++ } ++ ++ mast->orig_l->offset--; ++ do { ++ mas_descend(mast->orig_l); ++ mast->orig_l->offset = ++ mas_data_end(mast->orig_l); ++ depth--; ++ } while (depth); ++ ++ mast_rebalance_prev(mast); ++ do { ++ unsigned char r_off; ++ struct maple_enode *child = l_tmp.node; ++ ++ mas_ascend(&l_tmp); ++ if (ancestor == l_tmp.node) ++ r_off = end; ++ else ++ r_off = mas_data_end(&l_tmp); ++ ++ if (l_tmp.offset < r_off) ++ l_tmp.offset++; ++ ++ if (l_tmp.offset < r_off) ++ mas_topiary_range(&l_tmp, mast->destroy, ++ l_tmp.offset, r_off); ++ ++ if (r_tmp.node != child) ++ mat_add(mast->free, child); ++ ++ } while (l_tmp.node != ancestor); ++ ++ *mast->orig_r = r_tmp; ++ return true; ++ } ++ } while (!mte_is_root(mast->orig_r->node)); ++ ++ *mast->orig_r = r_tmp; ++ *mast->orig_l = l_tmp; ++ return false; ++} ++ ++/* ++ * mast_ascend_free() - Add current original maple state nodes to the free list ++ * and ascend. ++ * @mast: the maple subtree state. ++ * ++ * Ascend the original left and right sides and add the previous nodes to the ++ * free list. Set the slots to point to the correct location in the new nodes. ++ */ ++static inline void ++mast_ascend_free(struct maple_subtree_state *mast) ++{ ++ MA_WR_STATE(wr_mas, mast->orig_r, NULL); ++ struct maple_enode *left = mast->orig_l->node; ++ struct maple_enode *right = mast->orig_r->node; ++ ++ mas_ascend(mast->orig_l); ++ mas_ascend(mast->orig_r); ++ mat_add(mast->free, left); ++ ++ if (left != right) ++ mat_add(mast->free, right); ++ ++ mast->orig_r->offset = 0; ++ mast->orig_r->index = mast->r->max; ++ /* last should be larger than or equal to index */ ++ if (mast->orig_r->last < mast->orig_r->index) ++ mast->orig_r->last = mast->orig_r->index; ++ /* ++ * The node may not contain the value so set slot to ensure all ++ * of the nodes contents are freed or destroyed. ++ */ ++ wr_mas.type = mte_node_type(mast->orig_r->node); ++ mas_wr_node_walk(&wr_mas); ++ /* Set up the left side of things */ ++ mast->orig_l->offset = 0; ++ mast->orig_l->index = mast->l->min; ++ wr_mas.mas = mast->orig_l; ++ wr_mas.type = mte_node_type(mast->orig_l->node); ++ mas_wr_node_walk(&wr_mas); ++ ++ mast->bn->type = wr_mas.type; ++} ++ ++/* ++ * mas_new_ma_node() - Create and return a new maple node. Helper function. ++ * @mas: the maple state with the allocations. ++ * @b_node: the maple_big_node with the type encoding. ++ * ++ * Use the node type from the maple_big_node to allocate a new node from the ++ * ma_state. This function exists mainly for code readability. ++ * ++ * Return: A new maple encoded node ++ */ ++static inline struct maple_enode ++*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) ++{ ++ return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); ++} ++ ++/* ++ * mas_mab_to_node() - Set up right and middle nodes ++ * ++ * @mas: the maple state that contains the allocations. ++ * @b_node: the node which contains the data. ++ * @left: The pointer which will have the left node ++ * @right: The pointer which may have the right node ++ * @middle: the pointer which may have the middle node (rare) ++ * @mid_split: the split location for the middle node ++ * ++ * Return: the split of left. ++ */ ++static inline unsigned char mas_mab_to_node(struct ma_state *mas, ++ struct maple_big_node *b_node, struct maple_enode **left, ++ struct maple_enode **right, struct maple_enode **middle, ++ unsigned char *mid_split, unsigned long min) ++{ ++ unsigned char split = 0; ++ unsigned char slot_count = mt_slots[b_node->type]; ++ ++ *left = mas_new_ma_node(mas, b_node); ++ *right = NULL; ++ *middle = NULL; ++ *mid_split = 0; ++ ++ if (b_node->b_end < slot_count) { ++ split = b_node->b_end; ++ } else { ++ split = mab_calc_split(mas, b_node, mid_split, min); ++ *right = mas_new_ma_node(mas, b_node); ++ } ++ ++ if (*mid_split) ++ *middle = mas_new_ma_node(mas, b_node); ++ ++ return split; ++ ++} ++ ++/* ++ * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end ++ * pointer. ++ * @b_node - the big node to add the entry ++ * @mas - the maple state to get the pivot (mas->max) ++ * @entry - the entry to add, if NULL nothing happens. ++ */ ++static inline void mab_set_b_end(struct maple_big_node *b_node, ++ struct ma_state *mas, ++ void *entry) ++{ ++ if (!entry) ++ return; ++ ++ b_node->slot[b_node->b_end] = entry; ++ if (mt_is_alloc(mas->tree)) ++ b_node->gap[b_node->b_end] = mas_max_gap(mas); ++ b_node->pivot[b_node->b_end++] = mas->max; ++} ++ ++/* ++ * mas_set_split_parent() - combine_then_separate helper function. Sets the parent ++ * of @mas->node to either @left or @right, depending on @slot and @split ++ * ++ * @mas - the maple state with the node that needs a parent ++ * @left - possible parent 1 ++ * @right - possible parent 2 ++ * @slot - the slot the mas->node was placed ++ * @split - the split location between @left and @right ++ */ ++static inline void mas_set_split_parent(struct ma_state *mas, ++ struct maple_enode *left, ++ struct maple_enode *right, ++ unsigned char *slot, unsigned char split) ++{ ++ if (mas_is_none(mas)) ++ return; ++ ++ if ((*slot) <= split) ++ mte_set_parent(mas->node, left, *slot); ++ else if (right) ++ mte_set_parent(mas->node, right, (*slot) - split - 1); ++ ++ (*slot)++; ++} ++ ++/* ++ * mte_mid_split_check() - Check if the next node passes the mid-split ++ * @**l: Pointer to left encoded maple node. ++ * @**m: Pointer to middle encoded maple node. ++ * @**r: Pointer to right encoded maple node. ++ * @slot: The offset ++ * @*split: The split location. ++ * @mid_split: The middle split. ++ */ ++static inline void mte_mid_split_check(struct maple_enode **l, ++ struct maple_enode **r, ++ struct maple_enode *right, ++ unsigned char slot, ++ unsigned char *split, ++ unsigned char mid_split) ++{ ++ if (*r == right) ++ return; ++ ++ if (slot < mid_split) ++ return; ++ ++ *l = *r; ++ *r = right; ++ *split = mid_split; ++} ++ ++/* ++ * mast_set_split_parents() - Helper function to set three nodes parents. Slot ++ * is taken from @mast->l. ++ * @mast - the maple subtree state ++ * @left - the left node ++ * @right - the right node ++ * @split - the split location. ++ */ ++static inline void mast_set_split_parents(struct maple_subtree_state *mast, ++ struct maple_enode *left, ++ struct maple_enode *middle, ++ struct maple_enode *right, ++ unsigned char split, ++ unsigned char mid_split) ++{ ++ unsigned char slot; ++ struct maple_enode *l = left; ++ struct maple_enode *r = right; ++ ++ if (mas_is_none(mast->l)) ++ return; ++ ++ if (middle) ++ r = middle; ++ ++ slot = mast->l->offset; ++ ++ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); ++ mas_set_split_parent(mast->l, l, r, &slot, split); ++ ++ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); ++ mas_set_split_parent(mast->m, l, r, &slot, split); ++ ++ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); ++ mas_set_split_parent(mast->r, l, r, &slot, split); ++} ++ ++/* ++ * mas_wmb_replace() - Write memory barrier and replace ++ * @mas: The maple state ++ * @free: the maple topiary list of nodes to free ++ * @destroy: The maple topiary list of nodes to destroy (walk and free) ++ * ++ * Updates gap as necessary. ++ */ ++static inline void mas_wmb_replace(struct ma_state *mas, ++ struct ma_topiary *free, ++ struct ma_topiary *destroy) ++{ ++ /* All nodes must see old data as dead prior to replacing that data */ ++ smp_wmb(); /* Needed for RCU */ ++ ++ /* Insert the new data in the tree */ ++ mas_replace(mas, true); ++ ++ if (!mte_is_leaf(mas->node)) ++ mas_descend_adopt(mas); ++ ++ mas_mat_free(mas, free); ++ ++ if (destroy) ++ mas_mat_destroy(mas, destroy); ++ ++ if (mte_is_leaf(mas->node)) ++ return; ++ ++ mas_update_gap(mas); ++} ++ ++/* ++ * mast_new_root() - Set a new tree root during subtree creation ++ * @mast: The maple subtree state ++ * @mas: The maple state ++ */ ++static inline void mast_new_root(struct maple_subtree_state *mast, ++ struct ma_state *mas) ++{ ++ mas_mn(mast->l)->parent = ++ ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ if (!mte_dead_node(mast->orig_l->node) && ++ !mte_is_root(mast->orig_l->node)) { ++ do { ++ mast_ascend_free(mast); ++ mast_topiary(mast); ++ } while (!mte_is_root(mast->orig_l->node)); ++ } ++ if ((mast->orig_l->node != mas->node) && ++ (mast->l->depth > mas_mt_height(mas))) { ++ mat_add(mast->free, mas->node); ++ } ++} ++ ++/* ++ * mast_cp_to_nodes() - Copy data out to nodes. ++ * @mast: The maple subtree state ++ * @left: The left encoded maple node ++ * @middle: The middle encoded maple node ++ * @right: The right encoded maple node ++ * @split: The location to split between left and (middle ? middle : right) ++ * @mid_split: The location to split between middle and right. ++ */ ++static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, ++ struct maple_enode *left, struct maple_enode *middle, ++ struct maple_enode *right, unsigned char split, unsigned char mid_split) ++{ ++ bool new_lmax = true; ++ ++ mast->l->node = mte_node_or_none(left); ++ mast->m->node = mte_node_or_none(middle); ++ mast->r->node = mte_node_or_none(right); ++ ++ mast->l->min = mast->orig_l->min; ++ if (split == mast->bn->b_end) { ++ mast->l->max = mast->orig_r->max; ++ new_lmax = false; ++ } ++ ++ mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); ++ ++ if (middle) { ++ mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); ++ mast->m->min = mast->bn->pivot[split] + 1; ++ split = mid_split; ++ } ++ ++ mast->r->max = mast->orig_r->max; ++ if (right) { ++ mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); ++ mast->r->min = mast->bn->pivot[split] + 1; ++ } ++} ++ ++/* ++ * mast_combine_cp_left - Copy in the original left side of the tree into the ++ * combined data set in the maple subtree state big node. ++ * @mast: The maple subtree state ++ */ ++static inline void mast_combine_cp_left(struct maple_subtree_state *mast) ++{ ++ unsigned char l_slot = mast->orig_l->offset; ++ ++ if (!l_slot) ++ return; ++ ++ mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); ++} ++ ++/* ++ * mast_combine_cp_right: Copy in the original right side of the tree into the ++ * combined data set in the maple subtree state big node. ++ * @mast: The maple subtree state ++ */ ++static inline void mast_combine_cp_right(struct maple_subtree_state *mast) ++{ ++ if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) ++ return; ++ ++ mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, ++ mt_slot_count(mast->orig_r->node), mast->bn, ++ mast->bn->b_end); ++ mast->orig_r->last = mast->orig_r->max; ++} ++ ++/* ++ * mast_sufficient: Check if the maple subtree state has enough data in the big ++ * node to create at least one sufficient node ++ * @mast: the maple subtree state ++ */ ++static inline bool mast_sufficient(struct maple_subtree_state *mast) ++{ ++ if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * mast_overflow: Check if there is too much data in the subtree state for a ++ * single node. ++ * @mast: The maple subtree state ++ */ ++static inline bool mast_overflow(struct maple_subtree_state *mast) ++{ ++ if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node)) ++ return true; ++ ++ return false; ++} ++ ++static inline void *mtree_range_walk(struct ma_state *mas) ++{ ++ unsigned long *pivots; ++ unsigned char offset; ++ struct maple_node *node; ++ struct maple_enode *next, *last; ++ enum maple_type type; ++ void __rcu **slots; ++ unsigned char end; ++ unsigned long max, min; ++ unsigned long prev_max, prev_min; ++ ++ last = next = mas->node; ++ prev_min = min = mas->min; ++ max = mas->max; ++ do { ++ offset = 0; ++ last = next; ++ node = mte_to_node(next); ++ type = mte_node_type(next); ++ pivots = ma_pivots(node, type); ++ end = ma_data_end(node, type, pivots, max); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ ++ if (pivots[offset] >= mas->index) { ++ prev_max = max; ++ prev_min = min; ++ max = pivots[offset]; ++ goto next; ++ } ++ ++ do { ++ offset++; ++ } while ((offset < end) && (pivots[offset] < mas->index)); ++ ++ prev_min = min; ++ min = pivots[offset - 1] + 1; ++ prev_max = max; ++ if (likely(offset < end && pivots[offset])) ++ max = pivots[offset]; ++ ++next: ++ slots = ma_slots(node, type); ++ next = mt_slot(mas->tree, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ } while (!ma_is_leaf(type)); ++ ++ mas->offset = offset; ++ mas->index = min; ++ mas->last = max; ++ mas->min = prev_min; ++ mas->max = prev_max; ++ mas->node = last; ++ return (void *) next; ++ ++dead_node: ++ mas_reset(mas); ++ return NULL; ++} ++ ++/* ++ * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. ++ * @mas: The starting maple state ++ * @mast: The maple_subtree_state, keeps track of 4 maple states. ++ * @count: The estimated count of iterations needed. ++ * ++ * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root ++ * is hit. First @b_node is split into two entries which are inserted into the ++ * next iteration of the loop. @b_node is returned populated with the final ++ * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the ++ * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last ++ * to account of what has been copied into the new sub-tree. The update of ++ * orig_l_mas->last is used in mas_consume to find the slots that will need to ++ * be either freed or destroyed. orig_l_mas->depth keeps track of the height of ++ * the new sub-tree in case the sub-tree becomes the full tree. ++ * ++ * Return: the number of elements in b_node during the last loop. ++ */ ++static int mas_spanning_rebalance(struct ma_state *mas, ++ struct maple_subtree_state *mast, unsigned char count) ++{ ++ unsigned char split, mid_split; ++ unsigned char slot = 0; ++ struct maple_enode *left = NULL, *middle = NULL, *right = NULL; ++ ++ MA_STATE(l_mas, mas->tree, mas->index, mas->index); ++ MA_STATE(r_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(m_mas, mas->tree, mas->index, mas->index); ++ MA_TOPIARY(free, mas->tree); ++ MA_TOPIARY(destroy, mas->tree); ++ ++ /* ++ * The tree needs to be rebalanced and leaves need to be kept at the same level. ++ * Rebalancing is done by use of the ``struct maple_topiary``. ++ */ ++ mast->l = &l_mas; ++ mast->m = &m_mas; ++ mast->r = &r_mas; ++ mast->free = &free; ++ mast->destroy = &destroy; ++ l_mas.node = r_mas.node = m_mas.node = MAS_NONE; ++ if (!(mast->orig_l->min && mast->orig_r->max == ULONG_MAX) && ++ unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) ++ mast_spanning_rebalance(mast); ++ ++ mast->orig_l->depth = 0; ++ ++ /* ++ * Each level of the tree is examined and balanced, pushing data to the left or ++ * right, or rebalancing against left or right nodes is employed to avoid ++ * rippling up the tree to limit the amount of churn. Once a new sub-section of ++ * the tree is created, there may be a mix of new and old nodes. The old nodes ++ * will have the incorrect parent pointers and currently be in two trees: the ++ * original tree and the partially new tree. To remedy the parent pointers in ++ * the old tree, the new data is swapped into the active tree and a walk down ++ * the tree is performed and the parent pointers are updated. ++ * See mas_descend_adopt() for more information.. ++ */ ++ while (count--) { ++ mast->bn->b_end--; ++ mast->bn->type = mte_node_type(mast->orig_l->node); ++ split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle, ++ &mid_split, mast->orig_l->min); ++ mast_set_split_parents(mast, left, middle, right, split, ++ mid_split); ++ mast_cp_to_nodes(mast, left, middle, right, split, mid_split); ++ ++ /* ++ * Copy data from next level in the tree to mast->bn from next ++ * iteration ++ */ ++ memset(mast->bn, 0, sizeof(struct maple_big_node)); ++ mast->bn->type = mte_node_type(left); ++ mast->orig_l->depth++; ++ ++ /* Root already stored in l->node. */ ++ if (mas_is_root_limits(mast->l)) ++ goto new_root; ++ ++ mast_ascend_free(mast); ++ mast_combine_cp_left(mast); ++ l_mas.offset = mast->bn->b_end; ++ mab_set_b_end(mast->bn, &l_mas, left); ++ mab_set_b_end(mast->bn, &m_mas, middle); ++ mab_set_b_end(mast->bn, &r_mas, right); ++ ++ /* Copy anything necessary out of the right node. */ ++ mast_combine_cp_right(mast); ++ mast_topiary(mast); ++ mast->orig_l->last = mast->orig_l->max; ++ ++ if (mast_sufficient(mast)) ++ continue; ++ ++ if (mast_overflow(mast)) ++ continue; ++ ++ /* May be a new root stored in mast->bn */ ++ if (mas_is_root_limits(mast->orig_l)) ++ break; ++ ++ mast_spanning_rebalance(mast); ++ ++ /* rebalancing from other nodes may require another loop. */ ++ if (!count) ++ count++; ++ } ++ ++ l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), ++ mte_node_type(mast->orig_l->node)); ++ mast->orig_l->depth++; ++ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); ++ mte_set_parent(left, l_mas.node, slot); ++ if (middle) ++ mte_set_parent(middle, l_mas.node, ++slot); ++ ++ if (right) ++ mte_set_parent(right, l_mas.node, ++slot); ++ ++ if (mas_is_root_limits(mast->l)) { ++new_root: ++ mast_new_root(mast, mas); ++ } else { ++ mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; ++ } ++ ++ if (!mte_dead_node(mast->orig_l->node)) ++ mat_add(&free, mast->orig_l->node); ++ ++ mas->depth = mast->orig_l->depth; ++ *mast->orig_l = l_mas; ++ mte_set_node_dead(mas->node); ++ ++ /* Set up mas for insertion. */ ++ mast->orig_l->depth = mas->depth; ++ mast->orig_l->alloc = mas->alloc; ++ *mas = *mast->orig_l; ++ mas_wmb_replace(mas, &free, &destroy); ++ mtree_range_walk(mas); ++ return mast->bn->b_end; ++} ++ ++/* ++ * mas_rebalance() - Rebalance a given node. ++ * @mas: The maple state ++ * @b_node: The big maple node. ++ * ++ * Rebalance two nodes into a single node or two new nodes that are sufficient. ++ * Continue upwards until tree is sufficient. ++ * ++ * Return: the number of elements in b_node during the last loop. ++ */ ++static inline int mas_rebalance(struct ma_state *mas, ++ struct maple_big_node *b_node) ++{ ++ char empty_count = mas_mt_height(mas); ++ struct maple_subtree_state mast; ++ unsigned char shift, b_end = ++b_node->b_end; ++ ++ MA_STATE(l_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(r_mas, mas->tree, mas->index, mas->last); ++ ++ trace_ma_op(__func__, mas); ++ ++ /* ++ * Rebalancing occurs if a node is insufficient. Data is rebalanced ++ * against the node to the right if it exists, otherwise the node to the ++ * left of this node is rebalanced against this node. If rebalancing ++ * causes just one node to be produced instead of two, then the parent ++ * is also examined and rebalanced if it is insufficient. Every level ++ * tries to combine the data in the same way. If one node contains the ++ * entire range of the tree, then that node is used as a new root node. ++ */ ++ mas_node_count(mas, 1 + empty_count * 3); ++ if (mas_is_err(mas)) ++ return 0; ++ ++ mast.orig_l = &l_mas; ++ mast.orig_r = &r_mas; ++ mast.bn = b_node; ++ mast.bn->type = mte_node_type(mas->node); ++ ++ l_mas = r_mas = *mas; ++ ++ if (mas_next_sibling(&r_mas)) { ++ mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); ++ r_mas.last = r_mas.index = r_mas.max; ++ } else { ++ mas_prev_sibling(&l_mas); ++ shift = mas_data_end(&l_mas) + 1; ++ mab_shift_right(b_node, shift); ++ mas->offset += shift; ++ mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); ++ b_node->b_end = shift + b_end; ++ l_mas.index = l_mas.last = l_mas.min; ++ } ++ ++ return mas_spanning_rebalance(mas, &mast, empty_count); ++} ++ ++/* ++ * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple ++ * state. ++ * @mas: The maple state ++ * @end: The end of the left-most node. ++ * ++ * During a mass-insert event (such as forking), it may be necessary to ++ * rebalance the left-most node when it is not sufficient. ++ */ ++static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end) ++{ ++ enum maple_type mt = mte_node_type(mas->node); ++ struct maple_node reuse, *newnode, *parent, *new_left, *left, *node; ++ struct maple_enode *eparent; ++ unsigned char offset, tmp, split = mt_slots[mt] / 2; ++ void __rcu **l_slots, **slots; ++ unsigned long *l_pivs, *pivs, gap; ++ bool in_rcu = mt_in_rcu(mas->tree); ++ ++ MA_STATE(l_mas, mas->tree, mas->index, mas->last); ++ ++ l_mas = *mas; ++ mas_prev_sibling(&l_mas); ++ ++ /* set up node. */ ++ if (in_rcu) { ++ /* Allocate for both left and right as well as parent. */ ++ mas_node_count(mas, 3); ++ if (mas_is_err(mas)) ++ return; ++ ++ newnode = mas_pop_node(mas); ++ } else { ++ newnode = &reuse; ++ } ++ ++ node = mas_mn(mas); ++ newnode->parent = node->parent; ++ slots = ma_slots(newnode, mt); ++ pivs = ma_pivots(newnode, mt); ++ left = mas_mn(&l_mas); ++ l_slots = ma_slots(left, mt); ++ l_pivs = ma_pivots(left, mt); ++ if (!l_slots[split]) ++ split++; ++ tmp = mas_data_end(&l_mas) - split; ++ ++ memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp); ++ memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp); ++ pivs[tmp] = l_mas.max; ++ memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end); ++ memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end); ++ ++ l_mas.max = l_pivs[split]; ++ mas->min = l_mas.max + 1; ++ eparent = mt_mk_node(mte_parent(l_mas.node), ++ mas_parent_enum(&l_mas, l_mas.node)); ++ tmp += end; ++ if (!in_rcu) { ++ unsigned char max_p = mt_pivots[mt]; ++ unsigned char max_s = mt_slots[mt]; ++ ++ if (tmp < max_p) ++ memset(pivs + tmp, 0, ++ sizeof(unsigned long *) * (max_p - tmp)); ++ ++ if (tmp < mt_slots[mt]) ++ memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp)); ++ ++ memcpy(node, newnode, sizeof(struct maple_node)); ++ ma_set_meta(node, mt, 0, tmp - 1); ++ mte_set_pivot(eparent, mte_parent_slot(l_mas.node), ++ l_pivs[split]); ++ ++ /* Remove data from l_pivs. */ ++ tmp = split + 1; ++ memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); ++ memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); ++ ma_set_meta(left, mt, 0, split); ++ ++ goto done; ++ } ++ ++ /* RCU requires replacing both l_mas, mas, and parent. */ ++ mas->node = mt_mk_node(newnode, mt); ++ ma_set_meta(newnode, mt, 0, tmp); ++ ++ new_left = mas_pop_node(mas); ++ new_left->parent = left->parent; ++ mt = mte_node_type(l_mas.node); ++ slots = ma_slots(new_left, mt); ++ pivs = ma_pivots(new_left, mt); ++ memcpy(slots, l_slots, sizeof(void *) * split); ++ memcpy(pivs, l_pivs, sizeof(unsigned long) * split); ++ ma_set_meta(new_left, mt, 0, split); ++ l_mas.node = mt_mk_node(new_left, mt); ++ ++ /* replace parent. */ ++ offset = mte_parent_slot(mas->node); ++ mt = mas_parent_enum(&l_mas, l_mas.node); ++ parent = mas_pop_node(mas); ++ slots = ma_slots(parent, mt); ++ pivs = ma_pivots(parent, mt); ++ memcpy(parent, mte_to_node(eparent), sizeof(struct maple_node)); ++ rcu_assign_pointer(slots[offset], mas->node); ++ rcu_assign_pointer(slots[offset - 1], l_mas.node); ++ pivs[offset - 1] = l_mas.max; ++ eparent = mt_mk_node(parent, mt); ++done: ++ gap = mas_leaf_max_gap(mas); ++ mte_set_gap(eparent, mte_parent_slot(mas->node), gap); ++ gap = mas_leaf_max_gap(&l_mas); ++ mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap); ++ mas_ascend(mas); ++ ++ if (in_rcu) ++ mas_replace(mas, false); ++ ++ mas_update_gap(mas); ++} ++ ++/* ++ * mas_split_final_node() - Split the final node in a subtree operation. ++ * @mast: the maple subtree state ++ * @mas: The maple state ++ * @height: The height of the tree in case it's a new root. ++ */ ++static inline bool mas_split_final_node(struct maple_subtree_state *mast, ++ struct ma_state *mas, int height) ++{ ++ struct maple_enode *ancestor; ++ ++ if (mte_is_root(mas->node)) { ++ if (mt_is_alloc(mas->tree)) ++ mast->bn->type = maple_arange_64; ++ else ++ mast->bn->type = maple_range_64; ++ mas->depth = height; ++ } ++ /* ++ * Only a single node is used here, could be root. ++ * The Big_node data should just fit in a single node. ++ */ ++ ancestor = mas_new_ma_node(mas, mast->bn); ++ mte_set_parent(mast->l->node, ancestor, mast->l->offset); ++ mte_set_parent(mast->r->node, ancestor, mast->r->offset); ++ mte_to_node(ancestor)->parent = mas_mn(mas)->parent; ++ ++ mast->l->node = ancestor; ++ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); ++ mas->offset = mast->bn->b_end - 1; ++ return true; ++} ++ ++/* ++ * mast_fill_bnode() - Copy data into the big node in the subtree state ++ * @mast: The maple subtree state ++ * @mas: the maple state ++ * @skip: The number of entries to skip for new nodes insertion. ++ */ ++static inline void mast_fill_bnode(struct maple_subtree_state *mast, ++ struct ma_state *mas, ++ unsigned char skip) ++{ ++ bool cp = true; ++ struct maple_enode *old = mas->node; ++ unsigned char split; ++ ++ memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap)); ++ memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot)); ++ memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot)); ++ mast->bn->b_end = 0; ++ ++ if (mte_is_root(mas->node)) { ++ cp = false; ++ } else { ++ mas_ascend(mas); ++ mat_add(mast->free, old); ++ mas->offset = mte_parent_slot(mas->node); ++ } ++ ++ if (cp && mast->l->offset) ++ mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); ++ ++ split = mast->bn->b_end; ++ mab_set_b_end(mast->bn, mast->l, mast->l->node); ++ mast->r->offset = mast->bn->b_end; ++ mab_set_b_end(mast->bn, mast->r, mast->r->node); ++ if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) ++ cp = false; ++ ++ if (cp) ++ mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, ++ mast->bn, mast->bn->b_end); ++ ++ mast->bn->b_end--; ++ mast->bn->type = mte_node_type(mas->node); ++} ++ ++/* ++ * mast_split_data() - Split the data in the subtree state big node into regular ++ * nodes. ++ * @mast: The maple subtree state ++ * @mas: The maple state ++ * @split: The location to split the big node ++ */ ++static inline void mast_split_data(struct maple_subtree_state *mast, ++ struct ma_state *mas, unsigned char split) ++{ ++ unsigned char p_slot; ++ ++ mab_mas_cp(mast->bn, 0, split, mast->l, true); ++ mte_set_pivot(mast->r->node, 0, mast->r->max); ++ mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); ++ mast->l->offset = mte_parent_slot(mas->node); ++ mast->l->max = mast->bn->pivot[split]; ++ mast->r->min = mast->l->max + 1; ++ if (mte_is_leaf(mas->node)) ++ return; ++ ++ p_slot = mast->orig_l->offset; ++ mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, ++ &p_slot, split); ++ mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, ++ &p_slot, split); ++} ++ ++/* ++ * mas_push_data() - Instead of splitting a node, it is beneficial to push the ++ * data to the right or left node if there is room. ++ * @mas: The maple state ++ * @height: The current height of the maple state ++ * @mast: The maple subtree state ++ * @left: Push left or not. ++ * ++ * Keeping the height of the tree low means faster lookups. ++ * ++ * Return: True if pushed, false otherwise. ++ */ ++static inline bool mas_push_data(struct ma_state *mas, int height, ++ struct maple_subtree_state *mast, bool left) ++{ ++ unsigned char slot_total = mast->bn->b_end; ++ unsigned char end, space, split; ++ ++ MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); ++ tmp_mas = *mas; ++ tmp_mas.depth = mast->l->depth; ++ ++ if (left && !mas_prev_sibling(&tmp_mas)) ++ return false; ++ else if (!left && !mas_next_sibling(&tmp_mas)) ++ return false; ++ ++ end = mas_data_end(&tmp_mas); ++ slot_total += end; ++ space = 2 * mt_slot_count(mas->node) - 2; ++ /* -2 instead of -1 to ensure there isn't a triple split */ ++ if (ma_is_leaf(mast->bn->type)) ++ space--; ++ ++ if (mas->max == ULONG_MAX) ++ space--; ++ ++ if (slot_total >= space) ++ return false; ++ ++ /* Get the data; Fill mast->bn */ ++ mast->bn->b_end++; ++ if (left) { ++ mab_shift_right(mast->bn, end + 1); ++ mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); ++ mast->bn->b_end = slot_total + 1; ++ } else { ++ mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); ++ } ++ ++ /* Configure mast for splitting of mast->bn */ ++ split = mt_slots[mast->bn->type] - 2; ++ if (left) { ++ /* Switch mas to prev node */ ++ mat_add(mast->free, mas->node); ++ *mas = tmp_mas; ++ /* Start using mast->l for the left side. */ ++ tmp_mas.node = mast->l->node; ++ *mast->l = tmp_mas; ++ } else { ++ mat_add(mast->free, tmp_mas.node); ++ tmp_mas.node = mast->r->node; ++ *mast->r = tmp_mas; ++ split = slot_total - split; ++ } ++ split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); ++ /* Update parent slot for split calculation. */ ++ if (left) ++ mast->orig_l->offset += end + 1; ++ ++ mast_split_data(mast, mas, split); ++ mast_fill_bnode(mast, mas, 2); ++ mas_split_final_node(mast, mas, height + 1); ++ return true; ++} ++ ++/* ++ * mas_split() - Split data that is too big for one node into two. ++ * @mas: The maple state ++ * @b_node: The maple big node ++ * Return: 1 on success, 0 on failure. ++ */ ++static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) ++{ ++ ++ struct maple_subtree_state mast; ++ int height = 0; ++ unsigned char mid_split, split = 0; ++ ++ /* ++ * Splitting is handled differently from any other B-tree; the Maple ++ * Tree splits upwards. Splitting up means that the split operation ++ * occurs when the walk of the tree hits the leaves and not on the way ++ * down. The reason for splitting up is that it is impossible to know ++ * how much space will be needed until the leaf is (or leaves are) ++ * reached. Since overwriting data is allowed and a range could ++ * overwrite more than one range or result in changing one entry into 3 ++ * entries, it is impossible to know if a split is required until the ++ * data is examined. ++ * ++ * Splitting is a balancing act between keeping allocations to a minimum ++ * and avoiding a 'jitter' event where a tree is expanded to make room ++ * for an entry followed by a contraction when the entry is removed. To ++ * accomplish the balance, there are empty slots remaining in both left ++ * and right nodes after a split. ++ */ ++ MA_STATE(l_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(r_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); ++ MA_TOPIARY(mat, mas->tree); ++ ++ trace_ma_op(__func__, mas); ++ mas->depth = mas_mt_height(mas); ++ /* Allocation failures will happen early. */ ++ mas_node_count(mas, 1 + mas->depth * 2); ++ if (mas_is_err(mas)) ++ return 0; ++ ++ mast.l = &l_mas; ++ mast.r = &r_mas; ++ mast.orig_l = &prev_l_mas; ++ mast.orig_r = &prev_r_mas; ++ mast.free = &mat; ++ mast.bn = b_node; ++ ++ while (height++ <= mas->depth) { ++ if (mt_slots[b_node->type] > b_node->b_end) { ++ mas_split_final_node(&mast, mas, height); ++ break; ++ } ++ ++ l_mas = r_mas = *mas; ++ l_mas.node = mas_new_ma_node(mas, b_node); ++ r_mas.node = mas_new_ma_node(mas, b_node); ++ /* ++ * Another way that 'jitter' is avoided is to terminate a split up early if the ++ * left or right node has space to spare. This is referred to as "pushing left" ++ * or "pushing right" and is similar to the B* tree, except the nodes left or ++ * right can rarely be reused due to RCU, but the ripple upwards is halted which ++ * is a significant savings. ++ */ ++ /* Try to push left. */ ++ if (mas_push_data(mas, height, &mast, true)) ++ break; ++ ++ /* Try to push right. */ ++ if (mas_push_data(mas, height, &mast, false)) ++ break; ++ ++ split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min); ++ mast_split_data(&mast, mas, split); ++ /* ++ * Usually correct, mab_mas_cp in the above call overwrites ++ * r->max. ++ */ ++ mast.r->max = mas->max; ++ mast_fill_bnode(&mast, mas, 1); ++ prev_l_mas = *mast.l; ++ prev_r_mas = *mast.r; ++ } ++ ++ /* Set the original node as dead */ ++ mat_add(mast.free, mas->node); ++ mas->node = l_mas.node; ++ mas_wmb_replace(mas, mast.free, NULL); ++ mtree_range_walk(mas); ++ return 1; ++} ++ ++/* ++ * mas_reuse_node() - Reuse the node to store the data. ++ * @wr_mas: The maple write state ++ * @bn: The maple big node ++ * @end: The end of the data. ++ * ++ * Will always return false in RCU mode. ++ * ++ * Return: True if node was reused, false otherwise. ++ */ ++static inline bool mas_reuse_node(struct ma_wr_state *wr_mas, ++ struct maple_big_node *bn, unsigned char end) ++{ ++ /* Need to be rcu safe. */ ++ if (mt_in_rcu(wr_mas->mas->tree)) ++ return false; ++ ++ if (end > bn->b_end) { ++ int clear = mt_slots[wr_mas->type] - bn->b_end; ++ ++ memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--); ++ memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear); ++ } ++ mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false); ++ return true; ++} ++ ++/* ++ * mas_commit_b_node() - Commit the big node into the tree. ++ * @wr_mas: The maple write state ++ * @b_node: The maple big node ++ * @end: The end of the data. ++ */ ++static inline int mas_commit_b_node(struct ma_wr_state *wr_mas, ++ struct maple_big_node *b_node, unsigned char end) ++{ ++ struct maple_node *node; ++ unsigned char b_end = b_node->b_end; ++ enum maple_type b_type = b_node->type; ++ ++ if ((b_end < mt_min_slots[b_type]) && ++ (!mte_is_root(wr_mas->mas->node)) && ++ (mas_mt_height(wr_mas->mas) > 1)) ++ return mas_rebalance(wr_mas->mas, b_node); ++ ++ if (b_end >= mt_slots[b_type]) ++ return mas_split(wr_mas->mas, b_node); ++ ++ if (mas_reuse_node(wr_mas, b_node, end)) ++ goto reuse_node; ++ ++ mas_node_count(wr_mas->mas, 1); ++ if (mas_is_err(wr_mas->mas)) ++ return 0; ++ ++ node = mas_pop_node(wr_mas->mas); ++ node->parent = mas_mn(wr_mas->mas)->parent; ++ wr_mas->mas->node = mt_mk_node(node, b_type); ++ mab_mas_cp(b_node, 0, b_end, wr_mas->mas, true); ++ ++ mas_replace(wr_mas->mas, false); ++reuse_node: ++ mas_update_gap(wr_mas->mas); ++ return 1; ++} ++ ++/* ++ * mas_root_expand() - Expand a root to a node ++ * @mas: The maple state ++ * @entry: The entry to store into the tree ++ */ ++static inline int mas_root_expand(struct ma_state *mas, void *entry) ++{ ++ void *contents = mas_root_locked(mas); ++ enum maple_type type = maple_leaf_64; ++ struct maple_node *node; ++ void __rcu **slots; ++ unsigned long *pivots; ++ int slot = 0; ++ ++ mas_node_count(mas, 1); ++ if (unlikely(mas_is_err(mas))) ++ return 0; ++ ++ node = mas_pop_node(mas); ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ node->parent = ma_parent_ptr( ++ ((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ mas->node = mt_mk_node(node, type); ++ ++ if (mas->index) { ++ if (contents) { ++ rcu_assign_pointer(slots[slot], contents); ++ if (likely(mas->index > 1)) ++ slot++; ++ } ++ pivots[slot++] = mas->index - 1; ++ } ++ ++ rcu_assign_pointer(slots[slot], entry); ++ mas->offset = slot; ++ pivots[slot] = mas->last; ++ if (mas->last != ULONG_MAX) ++ slot++; ++ mas->depth = 1; ++ mas_set_height(mas); ++ ++ /* swap the new root into the tree */ ++ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); ++ ma_set_meta(node, maple_leaf_64, 0, slot); ++ return slot; ++} ++ ++static inline void mas_store_root(struct ma_state *mas, void *entry) ++{ ++ if (likely((mas->last != 0) || (mas->index != 0))) ++ mas_root_expand(mas, entry); ++ else if (((unsigned long) (entry) & 3) == 2) ++ mas_root_expand(mas, entry); ++ else { ++ rcu_assign_pointer(mas->tree->ma_root, entry); ++ mas->node = MAS_START; ++ } ++} ++ ++/* ++ * mas_is_span_wr() - Check if the write needs to be treated as a write that ++ * spans the node. ++ * @mas: The maple state ++ * @piv: The pivot value being written ++ * @type: The maple node type ++ * @entry: The data to write ++ * ++ * Spanning writes are writes that start in one node and end in another OR if ++ * the write of a %NULL will cause the node to end with a %NULL. ++ * ++ * Return: True if this is a spanning write, false otherwise. ++ */ ++static bool mas_is_span_wr(struct ma_wr_state *wr_mas) ++{ ++ unsigned long max; ++ unsigned long last = wr_mas->mas->last; ++ unsigned long piv = wr_mas->r_max; ++ enum maple_type type = wr_mas->type; ++ void *entry = wr_mas->entry; ++ ++ /* Contained in this pivot */ ++ if (piv > last) ++ return false; ++ ++ max = wr_mas->mas->max; ++ if (unlikely(ma_is_leaf(type))) { ++ /* Fits in the node, but may span slots. */ ++ if (last < max) ++ return false; ++ ++ /* Writes to the end of the node but not null. */ ++ if ((last == max) && entry) ++ return false; ++ ++ /* ++ * Writing ULONG_MAX is not a spanning write regardless of the ++ * value being written as long as the range fits in the node. ++ */ ++ if ((last == ULONG_MAX) && (last == max)) ++ return false; ++ } else if (piv == last) { ++ if (entry) ++ return false; ++ ++ /* Detect spanning store wr walk */ ++ if (last == ULONG_MAX) ++ return false; ++ } ++ ++ trace_ma_write(__func__, wr_mas->mas, piv, entry); ++ ++ return true; ++} ++ ++static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas) ++{ ++ wr_mas->mas->depth++; ++ wr_mas->type = mte_node_type(wr_mas->mas->node); ++ mas_wr_node_walk(wr_mas); ++ wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); ++} ++ ++static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas) ++{ ++ wr_mas->mas->max = wr_mas->r_max; ++ wr_mas->mas->min = wr_mas->r_min; ++ wr_mas->mas->node = wr_mas->content; ++ wr_mas->mas->offset = 0; ++} ++/* ++ * mas_wr_walk() - Walk the tree for a write. ++ * @wr_mas: The maple write state ++ * ++ * Uses mas_slot_locked() and does not need to worry about dead nodes. ++ * ++ * Return: True if it's contained in a node, false on spanning write. ++ */ ++static bool mas_wr_walk(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ ++ while (true) { ++ mas_wr_walk_descend(wr_mas); ++ if (unlikely(mas_is_span_wr(wr_mas))) ++ return false; ++ ++ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, ++ mas->offset); ++ if (ma_is_leaf(wr_mas->type)) ++ return true; ++ ++ mas_wr_walk_traverse(wr_mas); ++ } ++ ++ return true; ++} ++ ++static bool mas_wr_walk_index(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ ++ while (true) { ++ mas_wr_walk_descend(wr_mas); ++ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, ++ mas->offset); ++ if (ma_is_leaf(wr_mas->type)) ++ return true; ++ mas_wr_walk_traverse(wr_mas); ++ ++ } ++ return true; ++} ++/* ++ * mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs. ++ * @l_wr_mas: The left maple write state ++ * @r_wr_mas: The right maple write state ++ */ ++static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, ++ struct ma_wr_state *r_wr_mas) ++{ ++ struct ma_state *r_mas = r_wr_mas->mas; ++ struct ma_state *l_mas = l_wr_mas->mas; ++ unsigned char l_slot; ++ ++ l_slot = l_mas->offset; ++ if (!l_wr_mas->content) ++ l_mas->index = l_wr_mas->r_min; ++ ++ if ((l_mas->index == l_wr_mas->r_min) && ++ (l_slot && ++ !mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) { ++ if (l_slot > 1) ++ l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1; ++ else ++ l_mas->index = l_mas->min; ++ ++ l_mas->offset = l_slot - 1; ++ } ++ ++ if (!r_wr_mas->content) { ++ if (r_mas->last < r_wr_mas->r_max) ++ r_mas->last = r_wr_mas->r_max; ++ r_mas->offset++; ++ } else if ((r_mas->last == r_wr_mas->r_max) && ++ (r_mas->last < r_mas->max) && ++ !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { ++ r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, ++ r_wr_mas->type, r_mas->offset + 1); ++ r_mas->offset++; ++ } ++} ++ ++static inline void *mas_state_walk(struct ma_state *mas) ++{ ++ void *entry; ++ ++ entry = mas_start(mas); ++ if (mas_is_none(mas)) ++ return NULL; ++ ++ if (mas_is_ptr(mas)) ++ return entry; ++ ++ return mtree_range_walk(mas); ++} ++ ++/* ++ * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up ++ * to date. ++ * ++ * @mas: The maple state. ++ * ++ * Note: Leaves mas in undesirable state. ++ * Return: The entry for @mas->index or %NULL on dead node. ++ */ ++static inline void *mtree_lookup_walk(struct ma_state *mas) ++{ ++ unsigned long *pivots; ++ unsigned char offset; ++ struct maple_node *node; ++ struct maple_enode *next; ++ enum maple_type type; ++ void __rcu **slots; ++ unsigned char end; ++ unsigned long max; ++ ++ next = mas->node; ++ max = ULONG_MAX; ++ do { ++ offset = 0; ++ node = mte_to_node(next); ++ type = mte_node_type(next); ++ pivots = ma_pivots(node, type); ++ end = ma_data_end(node, type, pivots, max); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ ++ if (pivots[offset] >= mas->index) ++ goto next; ++ ++ do { ++ offset++; ++ } while ((offset < end) && (pivots[offset] < mas->index)); ++ ++ if (likely(offset > end)) ++ max = pivots[offset]; ++ ++next: ++ slots = ma_slots(node, type); ++ next = mt_slot(mas->tree, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ } while (!ma_is_leaf(type)); ++ ++ return (void *) next; ++ ++dead_node: ++ mas_reset(mas); ++ return NULL; ++} ++ ++/* ++ * mas_new_root() - Create a new root node that only contains the entry passed ++ * in. ++ * @mas: The maple state ++ * @entry: The entry to store. ++ * ++ * Only valid when the index == 0 and the last == ULONG_MAX ++ * ++ * Return 0 on error, 1 on success. ++ */ ++static inline int mas_new_root(struct ma_state *mas, void *entry) ++{ ++ struct maple_enode *root = mas_root_locked(mas); ++ enum maple_type type = maple_leaf_64; ++ struct maple_node *node; ++ void __rcu **slots; ++ unsigned long *pivots; ++ ++ if (!entry && !mas->index && mas->last == ULONG_MAX) { ++ mas->depth = 0; ++ mas_set_height(mas); ++ rcu_assign_pointer(mas->tree->ma_root, entry); ++ mas->node = MAS_START; ++ goto done; ++ } ++ ++ mas_node_count(mas, 1); ++ if (mas_is_err(mas)) ++ return 0; ++ ++ node = mas_pop_node(mas); ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ node->parent = ma_parent_ptr( ++ ((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ mas->node = mt_mk_node(node, type); ++ rcu_assign_pointer(slots[0], entry); ++ pivots[0] = mas->last; ++ mas->depth = 1; ++ mas_set_height(mas); ++ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); ++ ++done: ++ if (xa_is_node(root)) ++ mte_destroy_walk(root, mas->tree); ++ ++ return 1; ++} ++/* ++ * mas_wr_spanning_store() - Create a subtree with the store operation completed ++ * and new nodes where necessary, then place the sub-tree in the actual tree. ++ * Note that mas is expected to point to the node which caused the store to ++ * span. ++ * @wr_mas: The maple write state ++ * ++ * Return: 0 on error, positive on success. ++ */ ++static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) ++{ ++ struct maple_subtree_state mast; ++ struct maple_big_node b_node; ++ struct ma_state *mas; ++ unsigned char height; ++ ++ /* Left and Right side of spanning store */ ++ MA_STATE(l_mas, NULL, 0, 0); ++ MA_STATE(r_mas, NULL, 0, 0); ++ ++ MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); ++ MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); ++ ++ /* ++ * A store operation that spans multiple nodes is called a spanning ++ * store and is handled early in the store call stack by the function ++ * mas_is_span_wr(). When a spanning store is identified, the maple ++ * state is duplicated. The first maple state walks the left tree path ++ * to ``index``, the duplicate walks the right tree path to ``last``. ++ * The data in the two nodes are combined into a single node, two nodes, ++ * or possibly three nodes (see the 3-way split above). A ``NULL`` ++ * written to the last entry of a node is considered a spanning store as ++ * a rebalance is required for the operation to complete and an overflow ++ * of data may happen. ++ */ ++ mas = wr_mas->mas; ++ trace_ma_op(__func__, mas); ++ ++ if (unlikely(!mas->index && mas->last == ULONG_MAX)) ++ return mas_new_root(mas, wr_mas->entry); ++ /* ++ * Node rebalancing may occur due to this store, so there may be three new ++ * entries per level plus a new root. ++ */ ++ height = mas_mt_height(mas); ++ mas_node_count(mas, 1 + height * 3); ++ if (mas_is_err(mas)) ++ return 0; ++ ++ /* ++ * Set up right side. Need to get to the next offset after the spanning ++ * store to ensure it's not NULL and to combine both the next node and ++ * the node with the start together. ++ */ ++ r_mas = *mas; ++ /* Avoid overflow, walk to next slot in the tree. */ ++ if (r_mas.last + 1) ++ r_mas.last++; ++ ++ r_mas.index = r_mas.last; ++ mas_wr_walk_index(&r_wr_mas); ++ r_mas.last = r_mas.index = mas->last; ++ ++ /* Set up left side. */ ++ l_mas = *mas; ++ mas_wr_walk_index(&l_wr_mas); ++ ++ if (!wr_mas->entry) { ++ mas_extend_spanning_null(&l_wr_mas, &r_wr_mas); ++ mas->offset = l_mas.offset; ++ mas->index = l_mas.index; ++ mas->last = l_mas.last = r_mas.last; ++ } ++ ++ /* expanding NULLs may make this cover the entire range */ ++ if (!l_mas.index && r_mas.last == ULONG_MAX) { ++ mas_set_range(mas, 0, ULONG_MAX); ++ return mas_new_root(mas, wr_mas->entry); ++ } ++ ++ memset(&b_node, 0, sizeof(struct maple_big_node)); ++ /* Copy l_mas and store the value in b_node. */ ++ mas_store_b_node(&l_wr_mas, &b_node, l_wr_mas.node_end); ++ /* Copy r_mas into b_node. */ ++ if (r_mas.offset <= r_wr_mas.node_end) ++ mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end, ++ &b_node, b_node.b_end + 1); ++ else ++ b_node.b_end++; ++ ++ /* Stop spanning searches by searching for just index. */ ++ l_mas.index = l_mas.last = mas->index; ++ ++ mast.bn = &b_node; ++ mast.orig_l = &l_mas; ++ mast.orig_r = &r_mas; ++ /* Combine l_mas and r_mas and split them up evenly again. */ ++ return mas_spanning_rebalance(mas, &mast, height + 1); ++} ++ ++/* ++ * mas_wr_node_store() - Attempt to store the value in a node ++ * @wr_mas: The maple write state ++ * ++ * Attempts to reuse the node, but may allocate. ++ * ++ * Return: True if stored, false otherwise ++ */ ++static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ void __rcu **dst_slots; ++ unsigned long *dst_pivots; ++ unsigned char dst_offset; ++ unsigned char new_end = wr_mas->node_end; ++ unsigned char offset; ++ unsigned char node_slots = mt_slots[wr_mas->type]; ++ struct maple_node reuse, *newnode; ++ unsigned char copy_size, max_piv = mt_pivots[wr_mas->type]; ++ bool in_rcu = mt_in_rcu(mas->tree); ++ ++ offset = mas->offset; ++ if (mas->last == wr_mas->r_max) { ++ /* runs right to the end of the node */ ++ if (mas->last == mas->max) ++ new_end = offset; ++ /* don't copy this offset */ ++ wr_mas->offset_end++; ++ } else if (mas->last < wr_mas->r_max) { ++ /* new range ends in this range */ ++ if (unlikely(wr_mas->r_max == ULONG_MAX)) ++ mas_bulk_rebalance(mas, wr_mas->node_end, wr_mas->type); ++ ++ new_end++; ++ } else { ++ if (wr_mas->end_piv == mas->last) ++ wr_mas->offset_end++; ++ ++ new_end -= wr_mas->offset_end - offset - 1; ++ } ++ ++ /* new range starts within a range */ ++ if (wr_mas->r_min < mas->index) ++ new_end++; ++ ++ /* Not enough room */ ++ if (new_end >= node_slots) ++ return false; ++ ++ /* Not enough data. */ ++ if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) && ++ !(mas->mas_flags & MA_STATE_BULK)) ++ return false; ++ ++ /* set up node. */ ++ if (in_rcu) { ++ mas_node_count(mas, 1); ++ if (mas_is_err(mas)) ++ return false; ++ ++ newnode = mas_pop_node(mas); ++ } else { ++ memset(&reuse, 0, sizeof(struct maple_node)); ++ newnode = &reuse; ++ } ++ ++ newnode->parent = mas_mn(mas)->parent; ++ dst_pivots = ma_pivots(newnode, wr_mas->type); ++ dst_slots = ma_slots(newnode, wr_mas->type); ++ /* Copy from start to insert point */ ++ memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * (offset + 1)); ++ memcpy(dst_slots, wr_mas->slots, sizeof(void *) * (offset + 1)); ++ dst_offset = offset; ++ ++ /* Handle insert of new range starting after old range */ ++ if (wr_mas->r_min < mas->index) { ++ mas->offset++; ++ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->content); ++ dst_pivots[dst_offset++] = mas->index - 1; ++ } ++ ++ /* Store the new entry and range end. */ ++ if (dst_offset < max_piv) ++ dst_pivots[dst_offset] = mas->last; ++ mas->offset = dst_offset; ++ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->entry); ++ ++ /* ++ * this range wrote to the end of the node or it overwrote the rest of ++ * the data ++ */ ++ if (wr_mas->offset_end > wr_mas->node_end || mas->last >= mas->max) { ++ new_end = dst_offset; ++ goto done; ++ } ++ ++ dst_offset++; ++ /* Copy to the end of node if necessary. */ ++ copy_size = wr_mas->node_end - wr_mas->offset_end + 1; ++ memcpy(dst_slots + dst_offset, wr_mas->slots + wr_mas->offset_end, ++ sizeof(void *) * copy_size); ++ if (dst_offset < max_piv) { ++ if (copy_size > max_piv - dst_offset) ++ copy_size = max_piv - dst_offset; ++ ++ memcpy(dst_pivots + dst_offset, ++ wr_mas->pivots + wr_mas->offset_end, ++ sizeof(unsigned long) * copy_size); ++ } ++ ++ if ((wr_mas->node_end == node_slots - 1) && (new_end < node_slots - 1)) ++ dst_pivots[new_end] = mas->max; ++ ++done: ++ mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end); ++ if (in_rcu) { ++ mas->node = mt_mk_node(newnode, wr_mas->type); ++ mas_replace(mas, false); ++ } else { ++ memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); ++ } ++ trace_ma_write(__func__, mas, 0, wr_mas->entry); ++ mas_update_gap(mas); ++ return true; ++} ++ ++/* ++ * mas_wr_slot_store: Attempt to store a value in a slot. ++ * @wr_mas: the maple write state ++ * ++ * Return: True if stored, false otherwise ++ */ ++static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ unsigned long lmax; /* Logical max. */ ++ unsigned char offset = mas->offset; ++ ++ if ((wr_mas->r_max > mas->last) && ((wr_mas->r_min != mas->index) || ++ (offset != wr_mas->node_end))) ++ return false; ++ ++ if (offset == wr_mas->node_end - 1) ++ lmax = mas->max; ++ else ++ lmax = wr_mas->pivots[offset + 1]; ++ ++ /* going to overwrite too many slots. */ ++ if (lmax < mas->last) ++ return false; ++ ++ if (wr_mas->r_min == mas->index) { ++ /* overwriting two or more ranges with one. */ ++ if (lmax == mas->last) ++ return false; ++ ++ /* Overwriting all of offset and a portion of offset + 1. */ ++ rcu_assign_pointer(wr_mas->slots[offset], wr_mas->entry); ++ wr_mas->pivots[offset] = mas->last; ++ goto done; ++ } ++ ++ /* Doesn't end on the next range end. */ ++ if (lmax != mas->last) ++ return false; ++ ++ /* Overwriting a portion of offset and all of offset + 1 */ ++ if ((offset + 1 < mt_pivots[wr_mas->type]) && ++ (wr_mas->entry || wr_mas->pivots[offset + 1])) ++ wr_mas->pivots[offset + 1] = mas->last; ++ ++ rcu_assign_pointer(wr_mas->slots[offset + 1], wr_mas->entry); ++ wr_mas->pivots[offset] = mas->index - 1; ++ mas->offset++; /* Keep mas accurate. */ ++ ++done: ++ trace_ma_write(__func__, mas, 0, wr_mas->entry); ++ mas_update_gap(mas); ++ return true; ++} ++ ++static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) ++{ ++ while ((wr_mas->mas->last > wr_mas->end_piv) && ++ (wr_mas->offset_end < wr_mas->node_end)) ++ wr_mas->end_piv = wr_mas->pivots[++wr_mas->offset_end]; ++ ++ if (wr_mas->mas->last > wr_mas->end_piv) ++ wr_mas->end_piv = wr_mas->mas->max; ++} ++ ++static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ ++ if (mas->last < wr_mas->end_piv && !wr_mas->slots[wr_mas->offset_end]) ++ mas->last = wr_mas->end_piv; ++ ++ /* Check next slot(s) if we are overwriting the end */ ++ if ((mas->last == wr_mas->end_piv) && ++ (wr_mas->node_end != wr_mas->offset_end) && ++ !wr_mas->slots[wr_mas->offset_end + 1]) { ++ wr_mas->offset_end++; ++ if (wr_mas->offset_end == wr_mas->node_end) ++ mas->last = mas->max; ++ else ++ mas->last = wr_mas->pivots[wr_mas->offset_end]; ++ wr_mas->end_piv = mas->last; ++ } ++ ++ if (!wr_mas->content) { ++ /* If this one is null, the next and prev are not */ ++ mas->index = wr_mas->r_min; ++ } else { ++ /* Check prev slot if we are overwriting the start */ ++ if (mas->index == wr_mas->r_min && mas->offset && ++ !wr_mas->slots[mas->offset - 1]) { ++ mas->offset--; ++ wr_mas->r_min = mas->index = ++ mas_safe_min(mas, wr_mas->pivots, mas->offset); ++ wr_mas->r_max = wr_mas->pivots[mas->offset]; ++ } ++ } ++} ++ ++static inline bool mas_wr_append(struct ma_wr_state *wr_mas) ++{ ++ unsigned char end = wr_mas->node_end; ++ unsigned char new_end = end + 1; ++ struct ma_state *mas = wr_mas->mas; ++ unsigned char node_pivots = mt_pivots[wr_mas->type]; ++ ++ if ((mas->index != wr_mas->r_min) && (mas->last == wr_mas->r_max)) { ++ if (new_end < node_pivots) ++ wr_mas->pivots[new_end] = wr_mas->pivots[end]; ++ ++ if (new_end < node_pivots) ++ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); ++ ++ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->entry); ++ mas->offset = new_end; ++ wr_mas->pivots[end] = mas->index - 1; ++ ++ return true; ++ } ++ ++ if ((mas->index == wr_mas->r_min) && (mas->last < wr_mas->r_max)) { ++ if (new_end < node_pivots) ++ wr_mas->pivots[new_end] = wr_mas->pivots[end]; ++ ++ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->content); ++ if (new_end < node_pivots) ++ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); ++ ++ wr_mas->pivots[end] = mas->last; ++ rcu_assign_pointer(wr_mas->slots[end], wr_mas->entry); ++ return true; ++ } ++ ++ return false; ++} ++ ++/* ++ * mas_wr_bnode() - Slow path for a modification. ++ * @wr_mas: The write maple state ++ * ++ * This is where split, rebalance end up. ++ */ ++static void mas_wr_bnode(struct ma_wr_state *wr_mas) ++{ ++ struct maple_big_node b_node; ++ ++ trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry); ++ memset(&b_node, 0, sizeof(struct maple_big_node)); ++ mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); ++ mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end); ++} ++ ++static inline void mas_wr_modify(struct ma_wr_state *wr_mas) ++{ ++ unsigned char node_slots; ++ unsigned char node_size; ++ struct ma_state *mas = wr_mas->mas; ++ ++ /* Direct replacement */ ++ if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) { ++ rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry); ++ if (!!wr_mas->entry ^ !!wr_mas->content) ++ mas_update_gap(mas); ++ return; ++ } ++ ++ /* Attempt to append */ ++ node_slots = mt_slots[wr_mas->type]; ++ node_size = wr_mas->node_end - wr_mas->offset_end + mas->offset + 2; ++ if (mas->max == ULONG_MAX) ++ node_size++; ++ ++ /* slot and node store will not fit, go to the slow path */ ++ if (unlikely(node_size >= node_slots)) ++ goto slow_path; ++ ++ if (wr_mas->entry && (wr_mas->node_end < node_slots - 1) && ++ (mas->offset == wr_mas->node_end) && mas_wr_append(wr_mas)) { ++ if (!wr_mas->content || !wr_mas->entry) ++ mas_update_gap(mas); ++ return; ++ } ++ ++ if ((wr_mas->offset_end - mas->offset <= 1) && mas_wr_slot_store(wr_mas)) ++ return; ++ else if (mas_wr_node_store(wr_mas)) ++ return; ++ ++ if (mas_is_err(mas)) ++ return; ++ ++slow_path: ++ mas_wr_bnode(wr_mas); ++} ++ ++/* ++ * mas_wr_store_entry() - Internal call to store a value ++ * @mas: The maple state ++ * @entry: The entry to store. ++ * ++ * Return: The contents that was stored at the index. ++ */ ++static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ ++ wr_mas->content = mas_start(mas); ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_store_root(mas, wr_mas->entry); ++ return wr_mas->content; ++ } ++ ++ if (unlikely(!mas_wr_walk(wr_mas))) { ++ mas_wr_spanning_store(wr_mas); ++ return wr_mas->content; ++ } ++ ++ /* At this point, we are at the leaf node that needs to be altered. */ ++ wr_mas->end_piv = wr_mas->r_max; ++ mas_wr_end_piv(wr_mas); ++ ++ if (!wr_mas->entry) ++ mas_wr_extend_null(wr_mas); ++ ++ /* New root for a single pointer */ ++ if (unlikely(!mas->index && mas->last == ULONG_MAX)) { ++ mas_new_root(mas, wr_mas->entry); ++ return wr_mas->content; ++ } ++ ++ mas_wr_modify(wr_mas); ++ return wr_mas->content; ++} ++ ++/** ++ * mas_insert() - Internal call to insert a value ++ * @mas: The maple state ++ * @entry: The entry to store ++ * ++ * Return: %NULL or the contents that already exists at the requested index ++ * otherwise. The maple state needs to be checked for error conditions. ++ */ ++static inline void *mas_insert(struct ma_state *mas, void *entry) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ /* ++ * Inserting a new range inserts either 0, 1, or 2 pivots within the ++ * tree. If the insert fits exactly into an existing gap with a value ++ * of NULL, then the slot only needs to be written with the new value. ++ * If the range being inserted is adjacent to another range, then only a ++ * single pivot needs to be inserted (as well as writing the entry). If ++ * the new range is within a gap but does not touch any other ranges, ++ * then two pivots need to be inserted: the start - 1, and the end. As ++ * usual, the entry must be written. Most operations require a new node ++ * to be allocated and replace an existing node to ensure RCU safety, ++ * when in RCU mode. The exception to requiring a newly allocated node ++ * is when inserting at the end of a node (appending). When done ++ * carefully, appending can reuse the node in place. ++ */ ++ wr_mas.content = mas_start(mas); ++ if (wr_mas.content) ++ goto exists; ++ ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_store_root(mas, entry); ++ return NULL; ++ } ++ ++ /* spanning writes always overwrite something */ ++ if (!mas_wr_walk(&wr_mas)) ++ goto exists; ++ ++ /* At this point, we are at the leaf node that needs to be altered. */ ++ wr_mas.offset_end = mas->offset; ++ wr_mas.end_piv = wr_mas.r_max; ++ ++ if (wr_mas.content || (mas->last > wr_mas.r_max)) ++ goto exists; ++ ++ if (!entry) ++ return NULL; ++ ++ mas_wr_modify(&wr_mas); ++ return wr_mas.content; ++ ++exists: ++ mas_set_err(mas, -EEXIST); ++ return wr_mas.content; ++ ++} ++ ++/* ++ * mas_prev_node() - Find the prev non-null entry at the same level in the ++ * tree. The prev value will be mas->node[mas->offset] or MAS_NONE. ++ * @mas: The maple state ++ * @min: The lower limit to search ++ * ++ * The prev node value will be mas->node[mas->offset] or MAS_NONE. ++ * Return: 1 if the node is dead, 0 otherwise. ++ */ ++static inline int mas_prev_node(struct ma_state *mas, unsigned long min) ++{ ++ enum maple_type mt; ++ int offset, level; ++ void __rcu **slots; ++ struct maple_node *node; ++ struct maple_enode *enode; ++ unsigned long *pivots; ++ ++ if (mas_is_none(mas)) ++ return 0; ++ ++ level = 0; ++ do { ++ node = mas_mn(mas); ++ if (ma_is_root(node)) ++ goto no_entry; ++ ++ /* Walk up. */ ++ if (unlikely(mas_ascend(mas))) ++ return 1; ++ offset = mas->offset; ++ level++; ++ } while (!offset); ++ ++ offset--; ++ mt = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ mas->max = pivots[offset]; ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ if (mas->max < min) ++ goto no_entry_min; ++ ++ while (level > 1) { ++ level--; ++ enode = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = enode; ++ mt = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ offset = ma_data_end(node, mt, pivots, mas->max); ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++ ++ if (offset < mt_pivots[mt]) ++ mas->max = pivots[offset]; ++ ++ if (mas->max < min) ++ goto no_entry; ++ } ++ ++ mas->node = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->offset = mas_data_end(mas); ++ if (unlikely(mte_dead_node(mas->node))) ++ return 1; ++ ++ return 0; ++ ++no_entry_min: ++ mas->offset = offset; ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++no_entry: ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = MAS_NONE; ++ return 0; ++} ++ ++/* ++ * mas_next_node() - Get the next node at the same level in the tree. ++ * @mas: The maple state ++ * @max: The maximum pivot value to check. ++ * ++ * The next value will be mas->node[mas->offset] or MAS_NONE. ++ * Return: 1 on dead node, 0 otherwise. ++ */ ++static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, ++ unsigned long max) ++{ ++ unsigned long min, pivot; ++ unsigned long *pivots; ++ struct maple_enode *enode; ++ int level = 0; ++ unsigned char offset; ++ enum maple_type mt; ++ void __rcu **slots; ++ ++ if (mas->max >= max) ++ goto no_entry; ++ ++ level = 0; ++ do { ++ if (ma_is_root(node)) ++ goto no_entry; ++ ++ min = mas->max + 1; ++ if (min > max) ++ goto no_entry; ++ ++ if (unlikely(mas_ascend(mas))) ++ return 1; ++ ++ offset = mas->offset; ++ level++; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ pivots = ma_pivots(node, mt); ++ } while (unlikely(offset == ma_data_end(node, mt, pivots, mas->max))); ++ ++ slots = ma_slots(node, mt); ++ pivot = mas_safe_pivot(mas, pivots, ++offset, mt); ++ while (unlikely(level > 1)) { ++ /* Descend, if necessary */ ++ enode = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = enode; ++ level--; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ offset = 0; ++ pivot = pivots[0]; ++ } ++ ++ enode = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = enode; ++ mas->min = min; ++ mas->max = pivot; ++ return 0; ++ ++no_entry: ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = MAS_NONE; ++ return 0; ++} ++ ++/* ++ * mas_next_nentry() - Get the next node entry ++ * @mas: The maple state ++ * @max: The maximum value to check ++ * @*range_start: Pointer to store the start of the range. ++ * ++ * Sets @mas->offset to the offset of the next node entry, @mas->last to the ++ * pivot of the entry. ++ * ++ * Return: The next entry, %NULL otherwise ++ */ ++static inline void *mas_next_nentry(struct ma_state *mas, ++ struct maple_node *node, unsigned long max, enum maple_type type) ++{ ++ unsigned char count; ++ unsigned long pivot; ++ unsigned long *pivots; ++ void __rcu **slots; ++ void *entry; ++ ++ if (mas->last == mas->max) { ++ mas->index = mas->max; ++ return NULL; ++ } ++ ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ mas->index = mas_safe_min(mas, pivots, mas->offset); ++ if (ma_dead_node(node)) ++ return NULL; ++ ++ if (mas->index > max) ++ return NULL; ++ ++ count = ma_data_end(node, type, pivots, mas->max); ++ if (mas->offset > count) ++ return NULL; ++ ++ while (mas->offset < count) { ++ pivot = pivots[mas->offset]; ++ entry = mas_slot(mas, slots, mas->offset); ++ if (ma_dead_node(node)) ++ return NULL; ++ ++ if (entry) ++ goto found; ++ ++ if (pivot >= max) ++ return NULL; ++ ++ mas->index = pivot + 1; ++ mas->offset++; ++ } ++ ++ if (mas->index > mas->max) { ++ mas->index = mas->last; ++ return NULL; ++ } ++ ++ pivot = mas_safe_pivot(mas, pivots, mas->offset, type); ++ entry = mas_slot(mas, slots, mas->offset); ++ if (ma_dead_node(node)) ++ return NULL; ++ ++ if (!pivot) ++ return NULL; ++ ++ if (!entry) ++ return NULL; ++ ++found: ++ mas->last = pivot; ++ return entry; ++} ++ ++static inline void mas_rewalk(struct ma_state *mas, unsigned long index) ++{ ++ ++retry: ++ mas_set(mas, index); ++ mas_state_walk(mas); ++ if (mas_is_start(mas)) ++ goto retry; ++ ++ return; ++ ++} ++ ++/* ++ * mas_next_entry() - Internal function to get the next entry. ++ * @mas: The maple state ++ * @limit: The maximum range start. ++ * ++ * Set the @mas->node to the next entry and the range_start to ++ * the beginning value for the entry. Does not check beyond @limit. ++ * Sets @mas->index and @mas->last to the limit if it is hit. ++ * Restarts on dead nodes. ++ * ++ * Return: the next entry or %NULL. ++ */ ++static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) ++{ ++ void *entry = NULL; ++ struct maple_enode *prev_node; ++ struct maple_node *node; ++ unsigned char offset; ++ unsigned long last; ++ enum maple_type mt; ++ ++ last = mas->last; ++retry: ++ offset = mas->offset; ++ prev_node = mas->node; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ mas->offset++; ++ if (unlikely(mas->offset >= mt_slots[mt])) { ++ mas->offset = mt_slots[mt] - 1; ++ goto next_node; ++ } ++ ++ while (!mas_is_none(mas)) { ++ entry = mas_next_nentry(mas, node, limit, mt); ++ if (unlikely(ma_dead_node(node))) { ++ mas_rewalk(mas, last); ++ goto retry; ++ } ++ ++ if (likely(entry)) ++ return entry; ++ ++ if (unlikely((mas->index > limit))) ++ break; ++ ++next_node: ++ prev_node = mas->node; ++ offset = mas->offset; ++ if (unlikely(mas_next_node(mas, node, limit))) { ++ mas_rewalk(mas, last); ++ goto retry; ++ } ++ mas->offset = 0; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ } ++ ++ mas->index = mas->last = limit; ++ mas->offset = offset; ++ mas->node = prev_node; ++ return NULL; ++} ++ ++/* ++ * mas_prev_nentry() - Get the previous node entry. ++ * @mas: The maple state. ++ * @limit: The lower limit to check for a value. ++ * ++ * Return: the entry, %NULL otherwise. ++ */ ++static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit, ++ unsigned long index) ++{ ++ unsigned long pivot, min; ++ unsigned char offset; ++ struct maple_node *mn; ++ enum maple_type mt; ++ unsigned long *pivots; ++ void __rcu **slots; ++ void *entry; ++ ++retry: ++ if (!mas->offset) ++ return NULL; ++ ++ mn = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ offset = mas->offset - 1; ++ if (offset >= mt_slots[mt]) ++ offset = mt_slots[mt] - 1; ++ ++ slots = ma_slots(mn, mt); ++ pivots = ma_pivots(mn, mt); ++ if (offset == mt_pivots[mt]) ++ pivot = mas->max; ++ else ++ pivot = pivots[offset]; ++ ++ if (unlikely(ma_dead_node(mn))) { ++ mas_rewalk(mas, index); ++ goto retry; ++ } ++ ++ while (offset && ((!mas_slot(mas, slots, offset) && pivot >= limit) || ++ !pivot)) ++ pivot = pivots[--offset]; ++ ++ min = mas_safe_min(mas, pivots, offset); ++ entry = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(mn))) { ++ mas_rewalk(mas, index); ++ goto retry; ++ } ++ ++ if (likely(entry)) { ++ mas->offset = offset; ++ mas->last = pivot; ++ mas->index = min; ++ } ++ return entry; ++} ++ ++static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min) ++{ ++ void *entry; ++ ++retry: ++ while (likely(!mas_is_none(mas))) { ++ entry = mas_prev_nentry(mas, min, mas->index); ++ if (unlikely(mas->last < min)) ++ goto not_found; ++ ++ if (likely(entry)) ++ return entry; ++ ++ if (unlikely(mas_prev_node(mas, min))) { ++ mas_rewalk(mas, mas->index); ++ goto retry; ++ } ++ ++ mas->offset++; ++ } ++ ++ mas->offset--; ++not_found: ++ mas->index = mas->last = min; ++ return NULL; ++} ++ ++/* ++ * mas_rev_awalk() - Internal function. Reverse allocation walk. Find the ++ * highest gap address of a given size in a given node and descend. ++ * @mas: The maple state ++ * @size: The needed size. ++ * ++ * Return: True if found in a leaf, false otherwise. ++ * ++ */ ++static bool mas_rev_awalk(struct ma_state *mas, unsigned long size) ++{ ++ enum maple_type type = mte_node_type(mas->node); ++ struct maple_node *node = mas_mn(mas); ++ unsigned long *pivots, *gaps; ++ void __rcu **slots; ++ unsigned long gap = 0; ++ unsigned long max, min, index; ++ unsigned char offset; ++ ++ if (unlikely(mas_is_err(mas))) ++ return true; ++ ++ if (ma_is_dense(type)) { ++ /* dense nodes. */ ++ mas->offset = (unsigned char)(mas->index - mas->min); ++ return true; ++ } ++ ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ gaps = ma_gaps(node, type); ++ offset = mas->offset; ++ min = mas_safe_min(mas, pivots, offset); ++ /* Skip out of bounds. */ ++ while (mas->last < min) ++ min = mas_safe_min(mas, pivots, --offset); ++ ++ max = mas_safe_pivot(mas, pivots, offset, type); ++ index = mas->index; ++ while (index <= max) { ++ gap = 0; ++ if (gaps) ++ gap = gaps[offset]; ++ else if (!mas_slot(mas, slots, offset)) ++ gap = max - min + 1; ++ ++ if (gap) { ++ if ((size <= gap) && (size <= mas->last - min + 1)) ++ break; ++ ++ if (!gaps) { ++ /* Skip the next slot, it cannot be a gap. */ ++ if (offset < 2) ++ goto ascend; ++ ++ offset -= 2; ++ max = pivots[offset]; ++ min = mas_safe_min(mas, pivots, offset); ++ continue; ++ } ++ } ++ ++ if (!offset) ++ goto ascend; ++ ++ offset--; ++ max = min - 1; ++ min = mas_safe_min(mas, pivots, offset); ++ } ++ ++ if (unlikely(index > max)) { ++ mas_set_err(mas, -EBUSY); ++ return false; ++ } ++ ++ if (unlikely(ma_is_leaf(type))) { ++ mas->offset = offset; ++ mas->min = min; ++ mas->max = min + gap - 1; ++ return true; ++ } ++ ++ /* descend, only happens under lock. */ ++ mas->node = mas_slot(mas, slots, offset); ++ mas->min = min; ++ mas->max = max; ++ mas->offset = mas_data_end(mas); ++ return false; ++ ++ascend: ++ if (mte_is_root(mas->node)) ++ mas_set_err(mas, -EBUSY); ++ ++ return false; ++} ++ ++static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) ++{ ++ enum maple_type type = mte_node_type(mas->node); ++ unsigned long pivot, min, gap = 0; ++ unsigned char count, offset; ++ unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type); ++ void __rcu **slots = ma_slots(mas_mn(mas), type); ++ bool found = false; ++ ++ if (ma_is_dense(type)) { ++ mas->offset = (unsigned char)(mas->index - mas->min); ++ return true; ++ } ++ ++ gaps = ma_gaps(mte_to_node(mas->node), type); ++ offset = mas->offset; ++ count = mt_slots[type]; ++ min = mas_safe_min(mas, pivots, offset); ++ for (; offset < count; offset++) { ++ pivot = mas_safe_pivot(mas, pivots, offset, type); ++ if (offset && !pivot) ++ break; ++ ++ /* Not within lower bounds */ ++ if (mas->index > pivot) ++ goto next_slot; ++ ++ if (gaps) ++ gap = gaps[offset]; ++ else if (!mas_slot(mas, slots, offset)) ++ gap = min(pivot, mas->last) - max(mas->index, min) + 1; ++ else ++ goto next_slot; ++ ++ if (gap >= size) { ++ if (ma_is_leaf(type)) { ++ found = true; ++ goto done; ++ } ++ if (mas->index <= pivot) { ++ mas->node = mas_slot(mas, slots, offset); ++ mas->min = min; ++ mas->max = pivot; ++ offset = 0; ++ type = mte_node_type(mas->node); ++ count = mt_slots[type]; ++ break; ++ } ++ } ++next_slot: ++ min = pivot + 1; ++ if (mas->last <= pivot) { ++ mas_set_err(mas, -EBUSY); ++ return true; ++ } ++ } ++ ++ if (mte_is_root(mas->node)) ++ found = true; ++done: ++ mas->offset = offset; ++ return found; ++} ++ ++/** ++ * mas_walk() - Search for @mas->index in the tree. ++ * @mas: The maple state. ++ * ++ * mas->index and mas->last will be set to the range if there is a value. If ++ * mas->node is MAS_NONE, reset to MAS_START. ++ * ++ * Return: the entry at the location or %NULL. ++ */ ++void *mas_walk(struct ma_state *mas) ++{ ++ void *entry; ++ ++retry: ++ entry = mas_state_walk(mas); ++ if (mas_is_start(mas)) ++ goto retry; ++ ++ if (mas_is_ptr(mas)) { ++ if (!mas->index) { ++ mas->last = 0; ++ } else { ++ mas->index = 1; ++ mas->last = ULONG_MAX; ++ } ++ return entry; ++ } ++ ++ if (mas_is_none(mas)) { ++ mas->index = 0; ++ mas->last = ULONG_MAX; ++ } ++ ++ return entry; ++} ++ ++static inline bool mas_rewind_node(struct ma_state *mas) ++{ ++ unsigned char slot; ++ ++ do { ++ if (mte_is_root(mas->node)) { ++ slot = mas->offset; ++ if (!slot) ++ return false; ++ } else { ++ mas_ascend(mas); ++ slot = mas->offset; ++ } ++ } while (!slot); ++ ++ mas->offset = --slot; ++ return true; ++} ++ ++/* ++ * mas_skip_node() - Internal function. Skip over a node. ++ * @mas: The maple state. ++ * ++ * Return: true if there is another node, false otherwise. ++ */ ++static inline bool mas_skip_node(struct ma_state *mas) ++{ ++ unsigned char slot, slot_count; ++ unsigned long *pivots; ++ enum maple_type mt; ++ ++ mt = mte_node_type(mas->node); ++ slot_count = mt_slots[mt] - 1; ++ do { ++ if (mte_is_root(mas->node)) { ++ slot = mas->offset; ++ if (slot > slot_count) { ++ mas_set_err(mas, -EBUSY); ++ return false; ++ } ++ } else { ++ mas_ascend(mas); ++ slot = mas->offset; ++ mt = mte_node_type(mas->node); ++ slot_count = mt_slots[mt] - 1; ++ } ++ } while (slot > slot_count); ++ ++ mas->offset = ++slot; ++ pivots = ma_pivots(mas_mn(mas), mt); ++ if (slot > 0) ++ mas->min = pivots[slot - 1] + 1; ++ ++ if (slot <= slot_count) ++ mas->max = pivots[slot]; ++ ++ return true; ++} ++ ++/* ++ * mas_awalk() - Allocation walk. Search from low address to high, for a gap of ++ * @size ++ * @mas: The maple state ++ * @size: The size of the gap required ++ * ++ * Search between @mas->index and @mas->last for a gap of @size. ++ */ ++static inline void mas_awalk(struct ma_state *mas, unsigned long size) ++{ ++ struct maple_enode *last = NULL; ++ ++ /* ++ * There are 4 options: ++ * go to child (descend) ++ * go back to parent (ascend) ++ * no gap found. (return, slot == MAPLE_NODE_SLOTS) ++ * found the gap. (return, slot != MAPLE_NODE_SLOTS) ++ */ ++ while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) { ++ if (last == mas->node) ++ mas_skip_node(mas); ++ else ++ last = mas->node; ++ } ++} ++ ++/* ++ * mas_fill_gap() - Fill a located gap with @entry. ++ * @mas: The maple state ++ * @entry: The value to store ++ * @slot: The offset into the node to store the @entry ++ * @size: The size of the entry ++ * @index: The start location ++ */ ++static inline void mas_fill_gap(struct ma_state *mas, void *entry, ++ unsigned char slot, unsigned long size, unsigned long *index) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ unsigned char pslot = mte_parent_slot(mas->node); ++ struct maple_enode *mn = mas->node; ++ unsigned long *pivots; ++ enum maple_type ptype; ++ /* ++ * mas->index is the start address for the search ++ * which may no longer be needed. ++ * mas->last is the end address for the search ++ */ ++ ++ *index = mas->index; ++ mas->last = mas->index + size - 1; ++ ++ /* ++ * It is possible that using mas->max and mas->min to correctly ++ * calculate the index and last will cause an issue in the gap ++ * calculation, so fix the ma_state here ++ */ ++ mas_ascend(mas); ++ ptype = mte_node_type(mas->node); ++ pivots = ma_pivots(mas_mn(mas), ptype); ++ mas->max = mas_safe_pivot(mas, pivots, pslot, ptype); ++ mas->min = mas_safe_min(mas, pivots, pslot); ++ mas->node = mn; ++ mas->offset = slot; ++ mas_wr_store_entry(&wr_mas); ++} ++ ++/* ++ * mas_sparse_area() - Internal function. Return upper or lower limit when ++ * searching for a gap in an empty tree. ++ * @mas: The maple state ++ * @min: the minimum range ++ * @max: The maximum range ++ * @size: The size of the gap ++ * @fwd: Searching forward or back ++ */ ++static inline void mas_sparse_area(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size, bool fwd) ++{ ++ unsigned long start = 0; ++ ++ if (!unlikely(mas_is_none(mas))) ++ start++; ++ /* mas_is_ptr */ ++ ++ if (start < min) ++ start = min; ++ ++ if (fwd) { ++ mas->index = start; ++ mas->last = start + size - 1; ++ return; ++ } ++ ++ mas->index = max; ++} ++ ++/* ++ * mas_empty_area() - Get the lowest address within the range that is ++ * sufficient for the size requested. ++ * @mas: The maple state ++ * @min: The lowest value of the range ++ * @max: The highest value of the range ++ * @size: The size needed ++ */ ++int mas_empty_area(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size) ++{ ++ unsigned char offset; ++ unsigned long *pivots; ++ enum maple_type mt; ++ ++ if (mas_is_start(mas)) ++ mas_start(mas); ++ else if (mas->offset >= 2) ++ mas->offset -= 2; ++ else if (!mas_skip_node(mas)) ++ return -EBUSY; ++ ++ /* Empty set */ ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_sparse_area(mas, min, max, size, true); ++ return 0; ++ } ++ ++ /* The start of the window can only be within these values */ ++ mas->index = min; ++ mas->last = max; ++ mas_awalk(mas, size); ++ ++ if (unlikely(mas_is_err(mas))) ++ return xa_err(mas->node); ++ ++ offset = mas->offset; ++ if (unlikely(offset == MAPLE_NODE_SLOTS)) ++ return -EBUSY; ++ ++ mt = mte_node_type(mas->node); ++ pivots = ma_pivots(mas_mn(mas), mt); ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++ ++ if (offset < mt_pivots[mt]) ++ mas->max = pivots[offset]; ++ ++ if (mas->index < mas->min) ++ mas->index = mas->min; ++ ++ mas->last = mas->index + size - 1; ++ return 0; ++} ++ ++/* ++ * mas_empty_area_rev() - Get the highest address within the range that is ++ * sufficient for the size requested. ++ * @mas: The maple state ++ * @min: The lowest value of the range ++ * @max: The highest value of the range ++ * @size: The size needed ++ */ ++int mas_empty_area_rev(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size) ++{ ++ struct maple_enode *last = mas->node; ++ ++ if (mas_is_start(mas)) { ++ mas_start(mas); ++ mas->offset = mas_data_end(mas); ++ } else if (mas->offset >= 2) { ++ mas->offset -= 2; ++ } else if (!mas_rewind_node(mas)) { ++ return -EBUSY; ++ } ++ ++ /* Empty set. */ ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_sparse_area(mas, min, max, size, false); ++ return 0; ++ } ++ ++ /* The start of the window can only be within these values. */ ++ mas->index = min; ++ mas->last = max; ++ ++ while (!mas_rev_awalk(mas, size)) { ++ if (last == mas->node) { ++ if (!mas_rewind_node(mas)) ++ return -EBUSY; ++ } else { ++ last = mas->node; ++ } ++ } ++ ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); ++ ++ if (unlikely(mas->offset == MAPLE_NODE_SLOTS)) ++ return -EBUSY; ++ ++ /* ++ * mas_rev_awalk() has set mas->min and mas->max to the gap values. If ++ * the maximum is outside the window we are searching, then use the last ++ * location in the search. ++ * mas->max and mas->min is the range of the gap. ++ * mas->index and mas->last are currently set to the search range. ++ */ ++ ++ /* Trim the upper limit to the max. */ ++ if (mas->max <= mas->last) ++ mas->last = mas->max; ++ ++ mas->index = mas->last - size + 1; ++ return 0; ++} ++ ++static inline int mas_alloc(struct ma_state *mas, void *entry, ++ unsigned long size, unsigned long *index) ++{ ++ unsigned long min; ++ ++ mas_start(mas); ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_root_expand(mas, entry); ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); ++ ++ if (!mas->index) ++ return mte_pivot(mas->node, 0); ++ return mte_pivot(mas->node, 1); ++ } ++ ++ /* Must be walking a tree. */ ++ mas_awalk(mas, size); ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); ++ ++ if (mas->offset == MAPLE_NODE_SLOTS) ++ goto no_gap; ++ ++ /* ++ * At this point, mas->node points to the right node and we have an ++ * offset that has a sufficient gap. ++ */ ++ min = mas->min; ++ if (mas->offset) ++ min = mte_pivot(mas->node, mas->offset - 1) + 1; ++ ++ if (mas->index < min) ++ mas->index = min; ++ ++ mas_fill_gap(mas, entry, mas->offset, size, index); ++ return 0; ++ ++no_gap: ++ return -EBUSY; ++} ++ ++static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min, ++ unsigned long max, void *entry, ++ unsigned long size, unsigned long *index) ++{ ++ int ret = 0; ++ ++ ret = mas_empty_area_rev(mas, min, max, size); ++ if (ret) ++ return ret; ++ ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); ++ ++ if (mas->offset == MAPLE_NODE_SLOTS) ++ goto no_gap; ++ ++ mas_fill_gap(mas, entry, mas->offset, size, index); ++ return 0; ++ ++no_gap: ++ return -EBUSY; ++} ++ ++/* ++ * mas_dead_leaves() - Mark all leaves of a node as dead. ++ * @mas: The maple state ++ * @slots: Pointer to the slot array ++ * ++ * Must hold the write lock. ++ * ++ * Return: The number of leaves marked as dead. ++ */ ++static inline ++unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots) ++{ ++ struct maple_node *node; ++ enum maple_type type; ++ void *entry; ++ int offset; ++ ++ for (offset = 0; offset < mt_slot_count(mas->node); offset++) { ++ entry = mas_slot_locked(mas, slots, offset); ++ type = mte_node_type(entry); ++ node = mte_to_node(entry); ++ /* Use both node and type to catch LE & BE metadata */ ++ if (!node || !type) ++ break; ++ ++ mte_set_node_dead(entry); ++ smp_wmb(); /* Needed for RCU */ ++ node->type = type; ++ rcu_assign_pointer(slots[offset], node); ++ } ++ ++ return offset; ++} ++ ++static void __rcu **mas_dead_walk(struct ma_state *mas, unsigned char offset) ++{ ++ struct maple_node *node, *next; ++ void __rcu **slots = NULL; ++ ++ next = mas_mn(mas); ++ do { ++ mas->node = ma_enode_ptr(next); ++ node = mas_mn(mas); ++ slots = ma_slots(node, node->type); ++ next = mas_slot_locked(mas, slots, offset); ++ offset = 0; ++ } while (!ma_is_leaf(next->type)); ++ ++ return slots; ++} ++ ++static void mt_free_walk(struct rcu_head *head) ++{ ++ void __rcu **slots; ++ struct maple_node *node, *start; ++ struct maple_tree mt; ++ unsigned char offset; ++ enum maple_type type; ++ MA_STATE(mas, &mt, 0, 0); ++ ++ node = container_of(head, struct maple_node, rcu); ++ ++ if (ma_is_leaf(node->type)) ++ goto free_leaf; ++ ++ mt_init_flags(&mt, node->ma_flags); ++ mas_lock(&mas); ++ start = node; ++ mas.node = mt_mk_node(node, node->type); ++ slots = mas_dead_walk(&mas, 0); ++ node = mas_mn(&mas); ++ do { ++ mt_free_bulk(node->slot_len, slots); ++ offset = node->parent_slot + 1; ++ mas.node = node->piv_parent; ++ if (mas_mn(&mas) == node) ++ goto start_slots_free; ++ ++ type = mte_node_type(mas.node); ++ slots = ma_slots(mte_to_node(mas.node), type); ++ if ((offset < mt_slots[type]) && (slots[offset])) ++ slots = mas_dead_walk(&mas, offset); ++ ++ node = mas_mn(&mas); ++ } while ((node != start) || (node->slot_len < offset)); ++ ++ slots = ma_slots(node, node->type); ++ mt_free_bulk(node->slot_len, slots); ++ ++start_slots_free: ++ mas_unlock(&mas); ++free_leaf: ++ mt_free_rcu(&node->rcu); ++} ++ ++static inline void __rcu **mas_destroy_descend(struct ma_state *mas, ++ struct maple_enode *prev, unsigned char offset) ++{ ++ struct maple_node *node; ++ struct maple_enode *next = mas->node; ++ void __rcu **slots = NULL; ++ ++ do { ++ mas->node = next; ++ node = mas_mn(mas); ++ slots = ma_slots(node, mte_node_type(mas->node)); ++ next = mas_slot_locked(mas, slots, 0); ++ if ((mte_dead_node(next))) ++ next = mas_slot_locked(mas, slots, 1); ++ ++ mte_set_node_dead(mas->node); ++ node->type = mte_node_type(mas->node); ++ node->piv_parent = prev; ++ node->parent_slot = offset; ++ offset = 0; ++ prev = mas->node; ++ } while (!mte_is_leaf(next)); ++ ++ return slots; ++} ++ ++static void mt_destroy_walk(struct maple_enode *enode, unsigned char ma_flags, ++ bool free) ++{ ++ void __rcu **slots; ++ struct maple_node *node = mte_to_node(enode); ++ struct maple_enode *start; ++ struct maple_tree mt; ++ ++ MA_STATE(mas, &mt, 0, 0); ++ ++ if (mte_is_leaf(enode)) ++ goto free_leaf; ++ ++ mt_init_flags(&mt, ma_flags); ++ mas_lock(&mas); ++ ++ mas.node = start = enode; ++ slots = mas_destroy_descend(&mas, start, 0); ++ node = mas_mn(&mas); ++ do { ++ enum maple_type type; ++ unsigned char offset; ++ struct maple_enode *parent, *tmp; ++ ++ node->slot_len = mas_dead_leaves(&mas, slots); ++ if (free) ++ mt_free_bulk(node->slot_len, slots); ++ offset = node->parent_slot + 1; ++ mas.node = node->piv_parent; ++ if (mas_mn(&mas) == node) ++ goto start_slots_free; ++ ++ type = mte_node_type(mas.node); ++ slots = ma_slots(mte_to_node(mas.node), type); ++ if (offset >= mt_slots[type]) ++ goto next; ++ ++ tmp = mas_slot_locked(&mas, slots, offset); ++ if (mte_node_type(tmp) && mte_to_node(tmp)) { ++ parent = mas.node; ++ mas.node = tmp; ++ slots = mas_destroy_descend(&mas, parent, offset); ++ } ++next: ++ node = mas_mn(&mas); ++ } while (start != mas.node); ++ ++ node = mas_mn(&mas); ++ node->slot_len = mas_dead_leaves(&mas, slots); ++ if (free) ++ mt_free_bulk(node->slot_len, slots); ++ ++start_slots_free: ++ mas_unlock(&mas); ++ ++free_leaf: ++ if (free) ++ mt_free_rcu(&node->rcu); ++} ++ ++/* ++ * mte_destroy_walk() - Free a tree or sub-tree. ++ * @enode - the encoded maple node (maple_enode) to start ++ * @mn - the tree to free - needed for node types. ++ * ++ * Must hold the write lock. ++ */ ++static inline void mte_destroy_walk(struct maple_enode *enode, ++ struct maple_tree *mt) ++{ ++ struct maple_node *node = mte_to_node(enode); ++ ++ if (mt_in_rcu(mt)) { ++ mt_destroy_walk(enode, mt->ma_flags, false); ++ call_rcu(&node->rcu, mt_free_walk); ++ } else { ++ mt_destroy_walk(enode, mt->ma_flags, true); ++ } ++} ++ ++static void mas_wr_store_setup(struct ma_wr_state *wr_mas) ++{ ++ if (!mas_is_start(wr_mas->mas)) { ++ if (mas_is_none(wr_mas->mas)) { ++ mas_reset(wr_mas->mas); ++ } else { ++ wr_mas->r_max = wr_mas->mas->max; ++ wr_mas->type = mte_node_type(wr_mas->mas->node); ++ if (mas_is_span_wr(wr_mas)) ++ mas_reset(wr_mas->mas); ++ } ++ } ++ ++} ++ ++/* Interface */ ++ ++/** ++ * mas_store() - Store an @entry. ++ * @mas: The maple state. ++ * @entry: The entry to store. ++ * ++ * The @mas->index and @mas->last is used to set the range for the @entry. ++ * Note: The @mas should have pre-allocated entries to ensure there is memory to ++ * store the entry. Please see mas_expected_entries()/mas_destroy() for more details. ++ * ++ * Return: the first entry between mas->index and mas->last or %NULL. ++ */ ++void *mas_store(struct ma_state *mas, void *entry) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ trace_ma_write(__func__, mas, 0, entry); ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++ if (mas->index > mas->last) ++ pr_err("Error %lu > %lu %p\n", mas->index, mas->last, entry); ++ MT_BUG_ON(mas->tree, mas->index > mas->last); ++ if (mas->index > mas->last) { ++ mas_set_err(mas, -EINVAL); ++ return NULL; ++ } ++ ++#endif ++ ++ /* ++ * Storing is the same operation as insert with the added caveat that it ++ * can overwrite entries. Although this seems simple enough, one may ++ * want to examine what happens if a single store operation was to ++ * overwrite multiple entries within a self-balancing B-Tree. ++ */ ++ mas_wr_store_setup(&wr_mas); ++ mas_wr_store_entry(&wr_mas); ++ return wr_mas.content; ++} ++ ++/** ++ * mas_store_gfp() - Store a value into the tree. ++ * @mas: The maple state ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations if necessary. ++ * ++ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not ++ * be allocated. ++ */ ++int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ mas_wr_store_setup(&wr_mas); ++ trace_ma_write(__func__, mas, 0, entry); ++retry: ++ mas_wr_store_entry(&wr_mas); ++ if (unlikely(mas_nomem(mas, gfp))) ++ goto retry; ++ ++ if (unlikely(mas_is_err(mas))) ++ return xa_err(mas->node); ++ ++ return 0; ++} ++ ++/** ++ * mas_store_prealloc() - Store a value into the tree using memory ++ * preallocated in the maple state. ++ * @mas: The maple state ++ * @entry: The entry to store. ++ */ ++void mas_store_prealloc(struct ma_state *mas, void *entry) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ mas_wr_store_setup(&wr_mas); ++ trace_ma_write(__func__, mas, 0, entry); ++ mas_wr_store_entry(&wr_mas); ++ BUG_ON(mas_is_err(mas)); ++ mas_destroy(mas); ++} ++ ++/** ++ * mas_preallocate() - Preallocate enough nodes for a store operation ++ * @mas: The maple state ++ * @entry: The entry that will be stored ++ * @gfp: The GFP_FLAGS to use for allocations. ++ * ++ * Return: 0 on success, -ENOMEM if memory could not be allocated. ++ */ ++int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) ++{ ++ int ret; ++ ++ mas_node_count_gfp(mas, 1 + mas_mt_height(mas) * 3, gfp); ++ mas->mas_flags |= MA_STATE_PREALLOC; ++ if (likely(!mas_is_err(mas))) ++ return 0; ++ ++ mas_set_alloc_req(mas, 0); ++ ret = xa_err(mas->node); ++ mas_reset(mas); ++ mas_destroy(mas); ++ mas_reset(mas); ++ return ret; ++} ++ ++/* ++ * mas_destroy() - destroy a maple state. ++ * @mas: The maple state ++ * ++ * Upon completion, check the left-most node and rebalance against the node to ++ * the right if necessary. Frees any allocated nodes associated with this maple ++ * state. ++ */ ++void mas_destroy(struct ma_state *mas) ++{ ++ struct maple_alloc *node; ++ ++ /* ++ * When using mas_for_each() to insert an expected number of elements, ++ * it is possible that the number inserted is less than the expected ++ * number. To fix an invalid final node, a check is performed here to ++ * rebalance the previous node with the final node. ++ */ ++ if (mas->mas_flags & MA_STATE_REBALANCE) { ++ unsigned char end; ++ ++ if (mas_is_start(mas)) ++ mas_start(mas); ++ ++ mtree_range_walk(mas); ++ end = mas_data_end(mas) + 1; ++ if (end < mt_min_slot_count(mas->node) - 1) ++ mas_destroy_rebalance(mas, end); ++ ++ mas->mas_flags &= ~MA_STATE_REBALANCE; ++ } ++ mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC); ++ ++ while (mas->alloc && !((unsigned long)mas->alloc & 0x1)) { ++ node = mas->alloc; ++ mas->alloc = node->slot[0]; ++ if (node->node_count > 0) ++ mt_free_bulk(node->node_count, ++ (void __rcu **)&node->slot[1]); ++ kmem_cache_free(maple_node_cache, node); ++ } ++ mas->alloc = NULL; ++} ++ ++/* ++ * mas_expected_entries() - Set the expected number of entries that will be inserted. ++ * @mas: The maple state ++ * @nr_entries: The number of expected entries. ++ * ++ * This will attempt to pre-allocate enough nodes to store the expected number ++ * of entries. The allocations will occur using the bulk allocator interface ++ * for speed. Please call mas_destroy() on the @mas after inserting the entries ++ * to ensure any unused nodes are freed. ++ * ++ * Return: 0 on success, -ENOMEM if memory could not be allocated. ++ */ ++int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) ++{ ++ int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2; ++ struct maple_enode *enode = mas->node; ++ int nr_nodes; ++ int ret; ++ ++ /* ++ * Sometimes it is necessary to duplicate a tree to a new tree, such as ++ * forking a process and duplicating the VMAs from one tree to a new ++ * tree. When such a situation arises, it is known that the new tree is ++ * not going to be used until the entire tree is populated. For ++ * performance reasons, it is best to use a bulk load with RCU disabled. ++ * This allows for optimistic splitting that favours the left and reuse ++ * of nodes during the operation. ++ */ ++ ++ /* Optimize splitting for bulk insert in-order */ ++ mas->mas_flags |= MA_STATE_BULK; ++ ++ /* ++ * Avoid overflow, assume a gap between each entry and a trailing null. ++ * If this is wrong, it just means allocation can happen during ++ * insertion of entries. ++ */ ++ nr_nodes = max(nr_entries, nr_entries * 2 + 1); ++ if (!mt_is_alloc(mas->tree)) ++ nonleaf_cap = MAPLE_RANGE64_SLOTS - 2; ++ ++ /* Leaves; reduce slots to keep space for expansion */ ++ nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2); ++ /* Internal nodes */ ++ nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap); ++ /* Add working room for split (2 nodes) + new parents */ ++ mas_node_count(mas, nr_nodes + 3); ++ ++ /* Detect if allocations run out */ ++ mas->mas_flags |= MA_STATE_PREALLOC; ++ ++ if (!mas_is_err(mas)) ++ return 0; ++ ++ ret = xa_err(mas->node); ++ mas->node = enode; ++ mas_destroy(mas); ++ return ret; ++ ++} ++ ++/** ++ * mas_next() - Get the next entry. ++ * @mas: The maple state ++ * @max: The maximum index to check. ++ * ++ * Returns the next entry after @mas->index. ++ * Must hold rcu_read_lock or the write lock. ++ * Can return the zero entry. ++ * ++ * Return: The next entry or %NULL ++ */ ++void *mas_next(struct ma_state *mas, unsigned long max) ++{ ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; ++ ++ if (mas_is_start(mas)) ++ mas_walk(mas); /* Retries on dead nodes handled by mas_walk */ ++ ++ if (mas_is_ptr(mas)) { ++ if (!mas->index) { ++ mas->index = 1; ++ mas->last = ULONG_MAX; ++ } ++ return NULL; ++ } ++ ++ if (mas->last == ULONG_MAX) ++ return NULL; ++ ++ /* Retries on dead nodes handled by mas_next_entry */ ++ return mas_next_entry(mas, max); ++} ++EXPORT_SYMBOL_GPL(mas_next); ++ ++/** ++ * mt_next() - get the next value in the maple tree ++ * @mt: The maple tree ++ * @index: The start index ++ * @max: The maximum index to check ++ * ++ * Return: The entry at @index or higher, or %NULL if nothing is found. ++ */ ++void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max) ++{ ++ void *entry = NULL; ++ MA_STATE(mas, mt, index, index); ++ ++ rcu_read_lock(); ++ entry = mas_next(&mas, max); ++ rcu_read_unlock(); ++ return entry; ++} ++EXPORT_SYMBOL_GPL(mt_next); ++ ++/** ++ * mas_prev() - Get the previous entry ++ * @mas: The maple state ++ * @min: The minimum value to check. ++ * ++ * Must hold rcu_read_lock or the write lock. ++ * Will reset mas to MAS_START if the node is MAS_NONE. Will stop on not ++ * searchable nodes. ++ * ++ * Return: the previous value or %NULL. ++ */ ++void *mas_prev(struct ma_state *mas, unsigned long min) ++{ ++ if (!mas->index) { ++ /* Nothing comes before 0 */ ++ mas->last = 0; ++ return NULL; ++ } ++ ++ if (unlikely(mas_is_ptr(mas))) ++ return NULL; ++ ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; ++ ++ if (mas_is_start(mas)) { ++ mas_walk(mas); ++ if (!mas->index) ++ return NULL; ++ } ++ ++ if (mas_is_ptr(mas)) { ++ if (!mas->index) { ++ mas->last = 0; ++ return NULL; ++ } ++ ++ mas->index = mas->last = 0; ++ return mas_root_locked(mas); ++ } ++ return mas_prev_entry(mas, min); ++} ++EXPORT_SYMBOL_GPL(mas_prev); ++ ++/** ++ * mt_prev() - get the previous value in the maple tree ++ * @mt: The maple tree ++ * @index: The start index ++ * @min: The minimum index to check ++ * ++ * Return: The entry at @index or lower, or %NULL if nothing is found. ++ */ ++void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min) ++{ ++ void *entry = NULL; ++ MA_STATE(mas, mt, index, index); ++ ++ rcu_read_lock(); ++ entry = mas_prev(&mas, min); ++ rcu_read_unlock(); ++ return entry; ++} ++EXPORT_SYMBOL_GPL(mt_prev); ++ ++/** ++ * mas_pause() - Pause a mas_find/mas_for_each to drop the lock. ++ * @mas: The maple state to pause ++ * ++ * Some users need to pause a walk and drop the lock they're holding in ++ * order to yield to a higher priority thread or carry out an operation ++ * on an entry. Those users should call this function before they drop ++ * the lock. It resets the @mas to be suitable for the next iteration ++ * of the loop after the user has reacquired the lock. If most entries ++ * found during a walk require you to call mas_pause(), the mt_for_each() ++ * iterator may be more appropriate. ++ * ++ */ ++void mas_pause(struct ma_state *mas) ++{ ++ mas->node = MAS_PAUSE; ++} ++EXPORT_SYMBOL_GPL(mas_pause); ++ ++/** ++ * mas_find() - On the first call, find the entry at or after mas->index up to ++ * %max. Otherwise, find the entry after mas->index. ++ * @mas: The maple state ++ * @max: The maximum value to check. ++ * ++ * Must hold rcu_read_lock or the write lock. ++ * If an entry exists, last and index are updated accordingly. ++ * May set @mas->node to MAS_NONE. ++ * ++ * Return: The entry or %NULL. ++ */ ++void *mas_find(struct ma_state *mas, unsigned long max) ++{ ++ if (unlikely(mas_is_paused(mas))) { ++ if (unlikely(mas->last == ULONG_MAX)) { ++ mas->node = MAS_NONE; ++ return NULL; ++ } ++ mas->node = MAS_START; ++ mas->index = ++mas->last; ++ } ++ ++ if (unlikely(mas_is_start(mas))) { ++ /* First run or continue */ ++ void *entry; ++ ++ if (mas->index > max) ++ return NULL; ++ ++ entry = mas_walk(mas); ++ if (entry) ++ return entry; ++ } ++ ++ if (unlikely(!mas_searchable(mas))) ++ return NULL; ++ ++ /* Retries on dead nodes handled by mas_next_entry */ ++ return mas_next_entry(mas, max); ++} ++ ++/** ++ * mas_find_rev: On the first call, find the first non-null entry at or below ++ * mas->index down to %min. Otherwise find the first non-null entry below ++ * mas->index down to %min. ++ * @mas: The maple state ++ * @min: The minimum value to check. ++ * ++ * Must hold rcu_read_lock or the write lock. ++ * If an entry exists, last and index are updated accordingly. ++ * May set @mas->node to MAS_NONE. ++ * ++ * Return: The entry or %NULL. ++ */ ++void *mas_find_rev(struct ma_state *mas, unsigned long min) ++{ ++ if (unlikely(mas_is_paused(mas))) { ++ if (unlikely(mas->last == ULONG_MAX)) { ++ mas->node = MAS_NONE; ++ return NULL; ++ } ++ mas->node = MAS_START; ++ mas->last = --mas->index; ++ } ++ ++ if (unlikely(mas_is_start(mas))) { ++ /* First run or continue */ ++ void *entry; ++ ++ if (mas->index < min) ++ return NULL; ++ ++ entry = mas_walk(mas); ++ if (entry) ++ return entry; ++ } ++ ++ if (unlikely(!mas_searchable(mas))) ++ return NULL; ++ ++ if (mas->index < min) ++ return NULL; ++ ++ /* Retries on dead nodes handled by mas_next_entry */ ++ return mas_prev_entry(mas, min); ++} ++EXPORT_SYMBOL_GPL(mas_find); ++ ++/** ++ * mas_erase() - Find the range in which index resides and erase the entire ++ * range. ++ * @mas: The maple state ++ * ++ * Must hold the write lock. ++ * Searches for @mas->index, sets @mas->index and @mas->last to the range and ++ * erases that range. ++ * ++ * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. ++ */ ++void *mas_erase(struct ma_state *mas) ++{ ++ void *entry; ++ MA_WR_STATE(wr_mas, mas, NULL); ++ ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; ++ ++ /* Retry unnecessary when holding the write lock. */ ++ entry = mas_state_walk(mas); ++ if (!entry) ++ return NULL; ++ ++write_retry: ++ /* Must reset to ensure spanning writes of last slot are detected */ ++ mas_reset(mas); ++ mas_wr_store_setup(&wr_mas); ++ mas_wr_store_entry(&wr_mas); ++ if (mas_nomem(mas, GFP_KERNEL)) ++ goto write_retry; ++ ++ return entry; ++} ++EXPORT_SYMBOL_GPL(mas_erase); ++ ++/** ++ * mas_nomem() - Check if there was an error allocating and do the allocation ++ * if necessary If there are allocations, then free them. ++ * @mas: The maple state ++ * @gfp: The GFP_FLAGS to use for allocations ++ * Return: true on allocation, false otherwise. ++ */ ++bool mas_nomem(struct ma_state *mas, gfp_t gfp) ++ __must_hold(mas->tree->lock) ++{ ++ if (likely(mas->node != MA_ERROR(-ENOMEM))) { ++ mas_destroy(mas); ++ return false; ++ } ++ ++ if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { ++ mtree_unlock(mas->tree); ++ mas_alloc_nodes(mas, gfp); ++ mtree_lock(mas->tree); ++ } else { ++ mas_alloc_nodes(mas, gfp); ++ } ++ ++ if (!mas_allocated(mas)) ++ return false; ++ ++ mas->node = MAS_START; ++ return true; ++} ++ ++void __init maple_tree_init(void) ++{ ++ maple_node_cache = kmem_cache_create("maple_node", ++ sizeof(struct maple_node), sizeof(struct maple_node), ++ SLAB_PANIC, NULL); ++} ++ ++/** ++ * mtree_load() - Load a value stored in a maple tree ++ * @mt: The maple tree ++ * @index: The index to load ++ * ++ * Return: the entry or %NULL ++ */ ++void *mtree_load(struct maple_tree *mt, unsigned long index) ++{ ++ MA_STATE(mas, mt, index, index); ++ void *entry; ++ ++ trace_ma_read(__func__, &mas); ++ rcu_read_lock(); ++retry: ++ entry = mas_start(&mas); ++ if (unlikely(mas_is_none(&mas))) ++ goto unlock; ++ ++ if (unlikely(mas_is_ptr(&mas))) { ++ if (index) ++ entry = NULL; ++ ++ goto unlock; ++ } ++ ++ entry = mtree_lookup_walk(&mas); ++ if (!entry && unlikely(mas_is_start(&mas))) ++ goto retry; ++unlock: ++ rcu_read_unlock(); ++ if (xa_is_zero(entry)) ++ return NULL; ++ ++ return entry; ++} ++EXPORT_SYMBOL(mtree_load); ++ ++/** ++ * mtree_store_range() - Store an entry at a given range. ++ * @mt: The maple tree ++ * @index: The start of the range ++ * @last: The end of the range ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations ++ * ++ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not ++ * be allocated. ++ */ ++int mtree_store_range(struct maple_tree *mt, unsigned long index, ++ unsigned long last, void *entry, gfp_t gfp) ++{ ++ MA_STATE(mas, mt, index, last); ++ MA_WR_STATE(wr_mas, &mas, entry); ++ ++ trace_ma_write(__func__, &mas, 0, entry); ++ if (WARN_ON_ONCE(xa_is_advanced(entry))) ++ return -EINVAL; ++ ++ if (index > last) ++ return -EINVAL; ++ ++ mtree_lock(mt); ++retry: ++ mas_wr_store_entry(&wr_mas); ++ if (mas_nomem(&mas, gfp)) ++ goto retry; ++ ++ mtree_unlock(mt); ++ if (mas_is_err(&mas)) ++ return xa_err(mas.node); ++ ++ return 0; ++} ++EXPORT_SYMBOL(mtree_store_range); ++ ++/** ++ * mtree_store() - Store an entry at a given index. ++ * @mt: The maple tree ++ * @index: The index to store the value ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations ++ * ++ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not ++ * be allocated. ++ */ ++int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, ++ gfp_t gfp) ++{ ++ return mtree_store_range(mt, index, index, entry, gfp); ++} ++EXPORT_SYMBOL(mtree_store); ++ ++/** ++ * mtree_insert_range() - Insert an entry at a give range if there is no value. ++ * @mt: The maple tree ++ * @first: The start of the range ++ * @last: The end of the range ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations. ++ * ++ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid ++ * request, -ENOMEM if memory could not be allocated. ++ */ ++int mtree_insert_range(struct maple_tree *mt, unsigned long first, ++ unsigned long last, void *entry, gfp_t gfp) ++{ ++ MA_STATE(ms, mt, first, last); ++ ++ if (WARN_ON_ONCE(xa_is_advanced(entry))) ++ return -EINVAL; ++ ++ if (first > last) ++ return -EINVAL; ++ ++ mtree_lock(mt); ++retry: ++ mas_insert(&ms, entry); ++ if (mas_nomem(&ms, gfp)) ++ goto retry; ++ ++ mtree_unlock(mt); ++ if (mas_is_err(&ms)) ++ return xa_err(ms.node); ++ ++ return 0; ++} ++EXPORT_SYMBOL(mtree_insert_range); ++ ++/** ++ * mtree_insert() - Insert an entry at a give index if there is no value. ++ * @mt: The maple tree ++ * @index : The index to store the value ++ * @entry: The entry to store ++ * @gfp: The FGP_FLAGS to use for allocations. ++ * ++ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid ++ * request, -ENOMEM if memory could not be allocated. ++ */ ++int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, ++ gfp_t gfp) ++{ ++ return mtree_insert_range(mt, index, index, entry, gfp); ++} ++EXPORT_SYMBOL(mtree_insert); ++ ++int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp) ++{ ++ int ret = 0; ++ ++ MA_STATE(mas, mt, min, max - size); ++ if (!mt_is_alloc(mt)) ++ return -EINVAL; ++ ++ if (WARN_ON_ONCE(mt_is_reserved(entry))) ++ return -EINVAL; ++ ++ if (min > max) ++ return -EINVAL; ++ ++ if (max < size) ++ return -EINVAL; ++ ++ if (!size) ++ return -EINVAL; ++ ++ mtree_lock(mt); ++retry: ++ mas.offset = 0; ++ mas.index = min; ++ mas.last = max - size; ++ ret = mas_alloc(&mas, entry, size, startp); ++ if (mas_nomem(&mas, gfp)) ++ goto retry; ++ ++ mtree_unlock(mt); ++ return ret; ++} ++EXPORT_SYMBOL(mtree_alloc_range); ++ ++int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp) ++{ ++ int ret = 0; ++ ++ MA_STATE(mas, mt, min, max - size); ++ if (!mt_is_alloc(mt)) ++ return -EINVAL; ++ ++ if (WARN_ON_ONCE(mt_is_reserved(entry))) ++ return -EINVAL; ++ ++ if (min >= max) ++ return -EINVAL; ++ ++ if (max < size - 1) ++ return -EINVAL; ++ ++ if (!size) ++ return -EINVAL; ++ ++ mtree_lock(mt); ++retry: ++ ret = mas_rev_alloc(&mas, min, max, entry, size, startp); ++ if (mas_nomem(&mas, gfp)) ++ goto retry; ++ ++ mtree_unlock(mt); ++ return ret; ++} ++EXPORT_SYMBOL(mtree_alloc_rrange); ++ ++/** ++ * mtree_erase() - Find an index and erase the entire range. ++ * @mt: The maple tree ++ * @index: The index to erase ++ * ++ * Erasing is the same as a walk to an entry then a store of a NULL to that ++ * ENTIRE range. In fact, it is implemented as such using the advanced API. ++ * ++ * Return: The entry stored at the @index or %NULL ++ */ ++void *mtree_erase(struct maple_tree *mt, unsigned long index) ++{ ++ void *entry = NULL; ++ ++ MA_STATE(mas, mt, index, index); ++ trace_ma_op(__func__, &mas); ++ ++ mtree_lock(mt); ++ entry = mas_erase(&mas); ++ mtree_unlock(mt); ++ ++ return entry; ++} ++EXPORT_SYMBOL(mtree_erase); ++ ++/** ++ * __mt_destroy() - Walk and free all nodes of a locked maple tree. ++ * @mt: The maple tree ++ * ++ * Note: Does not handle locking. ++ */ ++void __mt_destroy(struct maple_tree *mt) ++{ ++ void *root = mt_root_locked(mt); ++ ++ rcu_assign_pointer(mt->ma_root, NULL); ++ if (xa_is_node(root)) ++ mte_destroy_walk(root, mt); ++ ++ mt->ma_flags = 0; ++} ++EXPORT_SYMBOL_GPL(__mt_destroy); ++ ++/** ++ * mtree_destroy() - Destroy a maple tree ++ * @mt: The maple tree ++ * ++ * Frees all resources used by the tree. Handles locking. ++ */ ++void mtree_destroy(struct maple_tree *mt) ++{ ++ mtree_lock(mt); ++ __mt_destroy(mt); ++ mtree_unlock(mt); ++} ++EXPORT_SYMBOL(mtree_destroy); ++ ++/** ++ * mt_find() - Search from the start up until an entry is found. ++ * @mt: The maple tree ++ * @index: Pointer which contains the start location of the search ++ * @max: The maximum value to check ++ * ++ * Handles locking. @index will be incremented to one beyond the range. ++ * ++ * Return: The entry at or after the @index or %NULL ++ */ ++void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) ++{ ++ MA_STATE(mas, mt, *index, *index); ++ void *entry; ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++ unsigned long copy = *index; ++#endif ++ ++ trace_ma_read(__func__, &mas); ++ ++ if ((*index) > max) ++ return NULL; ++ ++ rcu_read_lock(); ++retry: ++ entry = mas_state_walk(&mas); ++ if (mas_is_start(&mas)) ++ goto retry; ++ ++ if (unlikely(xa_is_zero(entry))) ++ entry = NULL; ++ ++ if (entry) ++ goto unlock; ++ ++ while (mas_searchable(&mas) && (mas.index < max)) { ++ entry = mas_next_entry(&mas, max); ++ if (likely(entry && !xa_is_zero(entry))) ++ break; ++ } ++ ++ if (unlikely(xa_is_zero(entry))) ++ entry = NULL; ++unlock: ++ rcu_read_unlock(); ++ if (likely(entry)) { ++ *index = mas.last + 1; ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++ if ((*index) && (*index) <= copy) ++ pr_err("index not increased! %lx <= %lx\n", ++ *index, copy); ++ MT_BUG_ON(mt, (*index) && ((*index) <= copy)); ++#endif ++ } ++ ++ return entry; ++} ++EXPORT_SYMBOL(mt_find); ++ ++/** ++ * mt_find_after() - Search from the start up until an entry is found. ++ * @mt: The maple tree ++ * @index: Pointer which contains the start location of the search ++ * @max: The maximum value to check ++ * ++ * Handles locking, detects wrapping on index == 0 ++ * ++ * Return: The entry at or after the @index or %NULL ++ */ ++void *mt_find_after(struct maple_tree *mt, unsigned long *index, ++ unsigned long max) ++{ ++ if (!(*index)) ++ return NULL; ++ ++ return mt_find(mt, index, max); ++} ++EXPORT_SYMBOL(mt_find_after); ++ ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++atomic_t maple_tree_tests_run; ++EXPORT_SYMBOL_GPL(maple_tree_tests_run); ++atomic_t maple_tree_tests_passed; ++EXPORT_SYMBOL_GPL(maple_tree_tests_passed); ++ ++#ifndef __KERNEL__ ++extern void kmem_cache_set_non_kernel(struct kmem_cache *, unsigned int); ++void mt_set_non_kernel(unsigned int val) ++{ ++ kmem_cache_set_non_kernel(maple_node_cache, val); ++} ++ ++extern unsigned long kmem_cache_get_alloc(struct kmem_cache *); ++unsigned long mt_get_alloc_size(void) ++{ ++ return kmem_cache_get_alloc(maple_node_cache); ++} ++ ++extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *); ++void mt_zero_nr_tallocated(void) ++{ ++ kmem_cache_zero_nr_tallocated(maple_node_cache); ++} ++ ++extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *); ++unsigned int mt_nr_tallocated(void) ++{ ++ return kmem_cache_nr_tallocated(maple_node_cache); ++} ++ ++extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *); ++unsigned int mt_nr_allocated(void) ++{ ++ return kmem_cache_nr_allocated(maple_node_cache); ++} ++ ++/* ++ * mas_dead_node() - Check if the maple state is pointing to a dead node. ++ * @mas: The maple state ++ * @index: The index to restore in @mas. ++ * ++ * Used in test code. ++ * Return: 1 if @mas has been reset to MAS_START, 0 otherwise. ++ */ ++static inline int mas_dead_node(struct ma_state *mas, unsigned long index) ++{ ++ if (unlikely(!mas_searchable(mas) || mas_is_start(mas))) ++ return 0; ++ ++ if (likely(!mte_dead_node(mas->node))) ++ return 0; ++ ++ mas_rewalk(mas, index); ++ return 1; ++} ++#endif /* not defined __KERNEL__ */ ++ ++/* ++ * mas_get_slot() - Get the entry in the maple state node stored at @offset. ++ * @mas: The maple state ++ * @offset: The offset into the slot array to fetch. ++ * ++ * Return: The entry stored at @offset. ++ */ ++static inline struct maple_enode *mas_get_slot(struct ma_state *mas, ++ unsigned char offset) ++{ ++ return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)), ++ offset); ++} ++ ++ ++/* ++ * mas_first_entry() - Go the first leaf and find the first entry. ++ * @mas: the maple state. ++ * @limit: the maximum index to check. ++ * @*r_start: Pointer to set to the range start. ++ * ++ * Sets mas->offset to the offset of the entry, r_start to the range minimum. ++ * ++ * Return: The first entry or MAS_NONE. ++ */ ++static inline void *mas_first_entry(struct ma_state *mas, struct maple_node *mn, ++ unsigned long limit, enum maple_type mt) ++ ++{ ++ unsigned long max; ++ unsigned long *pivots; ++ void __rcu **slots; ++ void *entry = NULL; ++ ++ mas->index = mas->min; ++ if (mas->index > limit) ++ goto none; ++ ++ max = mas->max; ++ mas->offset = 0; ++ while (likely(!ma_is_leaf(mt))) { ++ MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); ++ slots = ma_slots(mn, mt); ++ pivots = ma_pivots(mn, mt); ++ max = pivots[0]; ++ entry = mas_slot(mas, slots, 0); ++ if (unlikely(ma_dead_node(mn))) ++ return NULL; ++ mas->node = entry; ++ mn = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ } ++ MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); ++ ++ mas->max = max; ++ slots = ma_slots(mn, mt); ++ entry = mas_slot(mas, slots, 0); ++ if (unlikely(ma_dead_node(mn))) ++ return NULL; ++ ++ /* Slot 0 or 1 must be set */ ++ if (mas->index > limit) ++ goto none; ++ ++ if (likely(entry)) ++ return entry; ++ ++ pivots = ma_pivots(mn, mt); ++ mas->index = pivots[0] + 1; ++ mas->offset = 1; ++ entry = mas_slot(mas, slots, 1); ++ if (unlikely(ma_dead_node(mn))) ++ return NULL; ++ ++ if (mas->index > limit) ++ goto none; ++ ++ if (likely(entry)) ++ return entry; ++ ++none: ++ if (likely(!ma_dead_node(mn))) ++ mas->node = MAS_NONE; ++ return NULL; ++} ++ ++/* Depth first search, post-order */ ++static void mas_dfs_postorder(struct ma_state *mas, unsigned long max) ++{ ++ ++ struct maple_enode *p = MAS_NONE, *mn = mas->node; ++ unsigned long p_min, p_max; ++ ++ mas_next_node(mas, mas_mn(mas), max); ++ if (!mas_is_none(mas)) ++ return; ++ ++ if (mte_is_root(mn)) ++ return; ++ ++ mas->node = mn; ++ mas_ascend(mas); ++ while (mas->node != MAS_NONE) { ++ p = mas->node; ++ p_min = mas->min; ++ p_max = mas->max; ++ mas_prev_node(mas, 0); ++ } ++ ++ if (p == MAS_NONE) ++ return; ++ ++ mas->node = p; ++ mas->max = p_max; ++ mas->min = p_min; ++} ++ ++/* Tree validations */ ++static void mt_dump_node(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth); ++static void mt_dump_range(unsigned long min, unsigned long max, ++ unsigned int depth) ++{ ++ static const char spaces[] = " "; ++ ++ if (min == max) ++ pr_info("%.*s%lu: ", depth * 2, spaces, min); ++ else ++ pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max); ++} ++ ++static void mt_dump_entry(void *entry, unsigned long min, unsigned long max, ++ unsigned int depth) ++{ ++ mt_dump_range(min, max, depth); ++ ++ if (xa_is_value(entry)) ++ pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry), ++ xa_to_value(entry), entry); ++ else if (xa_is_zero(entry)) ++ pr_cont("zero (%ld)\n", xa_to_internal(entry)); ++ else if (mt_is_reserved(entry)) ++ pr_cont("UNKNOWN ENTRY (%p)\n", entry); ++ else ++ pr_cont("%p\n", entry); ++} ++ ++static void mt_dump_range64(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth) ++{ ++ struct maple_range_64 *node = &mte_to_node(entry)->mr64; ++ bool leaf = mte_is_leaf(entry); ++ unsigned long first = min; ++ int i; ++ ++ pr_cont(" contents: "); ++ for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) ++ pr_cont("%p %lu ", node->slot[i], node->pivot[i]); ++ pr_cont("%p\n", node->slot[i]); ++ for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) { ++ unsigned long last = max; ++ ++ if (i < (MAPLE_RANGE64_SLOTS - 1)) ++ last = node->pivot[i]; ++ else if (!node->slot[i] && max != mt_max[mte_node_type(entry)]) ++ break; ++ if (last == 0 && i > 0) ++ break; ++ if (leaf) ++ mt_dump_entry(mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ else if (node->slot[i]) ++ mt_dump_node(mt, mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ ++ if (last == max) ++ break; ++ if (last > max) { ++ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", ++ node, last, max, i); ++ break; ++ } ++ first = last + 1; ++ } ++} ++ ++static void mt_dump_arange64(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth) ++{ ++ struct maple_arange_64 *node = &mte_to_node(entry)->ma64; ++ bool leaf = mte_is_leaf(entry); ++ unsigned long first = min; ++ int i; ++ ++ pr_cont(" contents: "); ++ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) ++ pr_cont("%lu ", node->gap[i]); ++ pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap); ++ for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) ++ pr_cont("%p %lu ", node->slot[i], node->pivot[i]); ++ pr_cont("%p\n", node->slot[i]); ++ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) { ++ unsigned long last = max; ++ ++ if (i < (MAPLE_ARANGE64_SLOTS - 1)) ++ last = node->pivot[i]; ++ else if (!node->slot[i]) ++ break; ++ if (last == 0 && i > 0) ++ break; ++ if (leaf) ++ mt_dump_entry(mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ else if (node->slot[i]) ++ mt_dump_node(mt, mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ ++ if (last == max) ++ break; ++ if (last > max) { ++ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", ++ node, last, max, i); ++ break; ++ } ++ first = last + 1; ++ } ++} ++ ++static void mt_dump_node(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth) ++{ ++ struct maple_node *node = mte_to_node(entry); ++ unsigned int type = mte_node_type(entry); ++ unsigned int i; ++ ++ mt_dump_range(min, max, depth); ++ ++ pr_cont("node %p depth %d type %d parent %p", node, depth, type, ++ node ? node->parent : NULL); ++ switch (type) { ++ case maple_dense: ++ pr_cont("\n"); ++ for (i = 0; i < MAPLE_NODE_SLOTS; i++) { ++ if (min + i > max) ++ pr_cont("OUT OF RANGE: "); ++ mt_dump_entry(mt_slot(mt, node->slot, i), ++ min + i, min + i, depth); ++ } ++ break; ++ case maple_leaf_64: ++ case maple_range_64: ++ mt_dump_range64(mt, entry, min, max, depth); ++ break; ++ case maple_arange_64: ++ mt_dump_arange64(mt, entry, min, max, depth); ++ break; ++ ++ default: ++ pr_cont(" UNKNOWN TYPE\n"); ++ } ++} ++ ++void mt_dump(const struct maple_tree *mt) ++{ ++ void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt)); ++ ++ pr_info("maple_tree(%p) flags %X, height %u root %p\n", ++ mt, mt->ma_flags, mt_height(mt), entry); ++ if (!xa_is_node(entry)) ++ mt_dump_entry(entry, 0, 0, 0); ++ else if (entry) ++ mt_dump_node(mt, entry, 0, mt_max[mte_node_type(entry)], 0); ++} ++ ++/* ++ * Calculate the maximum gap in a node and check if that's what is reported in ++ * the parent (unless root). ++ */ ++static void mas_validate_gaps(struct ma_state *mas) ++{ ++ struct maple_enode *mte = mas->node; ++ struct maple_node *p_mn; ++ unsigned long gap = 0, max_gap = 0; ++ unsigned long p_end, p_start = mas->min; ++ unsigned char p_slot; ++ unsigned long *gaps = NULL; ++ unsigned long *pivots = ma_pivots(mte_to_node(mte), mte_node_type(mte)); ++ int i; ++ ++ if (ma_is_dense(mte_node_type(mte))) { ++ for (i = 0; i < mt_slot_count(mte); i++) { ++ if (mas_get_slot(mas, i)) { ++ if (gap > max_gap) ++ max_gap = gap; ++ gap = 0; ++ continue; ++ } ++ gap++; ++ } ++ goto counted; ++ } ++ ++ gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte)); ++ for (i = 0; i < mt_slot_count(mte); i++) { ++ p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte)); ++ ++ if (!gaps) { ++ if (mas_get_slot(mas, i)) { ++ gap = 0; ++ goto not_empty; ++ } ++ ++ gap += p_end - p_start + 1; ++ } else { ++ void *entry = mas_get_slot(mas, i); ++ ++ gap = gaps[i]; ++ if (!entry) { ++ if (gap != p_end - p_start + 1) { ++ pr_err("%p[%u] -> %p %lu != %lu - %lu + 1\n", ++ mas_mn(mas), i, ++ mas_get_slot(mas, i), gap, ++ p_end, p_start); ++ mt_dump(mas->tree); ++ ++ MT_BUG_ON(mas->tree, ++ gap != p_end - p_start + 1); ++ } ++ } else { ++ if (gap > p_end - p_start + 1) { ++ pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", ++ mas_mn(mas), i, gap, p_end, p_start, ++ p_end - p_start + 1); ++ MT_BUG_ON(mas->tree, ++ gap > p_end - p_start + 1); ++ } ++ } ++ } ++ ++ if (gap > max_gap) ++ max_gap = gap; ++not_empty: ++ p_start = p_end + 1; ++ if (p_end >= mas->max) ++ break; ++ } ++ ++counted: ++ if (mte_is_root(mte)) ++ return; ++ ++ p_slot = mte_parent_slot(mas->node); ++ p_mn = mte_parent(mte); ++ MT_BUG_ON(mas->tree, max_gap > mas->max); ++ if (ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap) { ++ pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap); ++ mt_dump(mas->tree); ++ } ++ ++ MT_BUG_ON(mas->tree, ++ ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap); ++} ++ ++static void mas_validate_parent_slot(struct ma_state *mas) ++{ ++ struct maple_node *parent; ++ struct maple_enode *node; ++ enum maple_type p_type = mas_parent_enum(mas, mas->node); ++ unsigned char p_slot = mte_parent_slot(mas->node); ++ void __rcu **slots; ++ int i; ++ ++ if (mte_is_root(mas->node)) ++ return; ++ ++ parent = mte_parent(mas->node); ++ slots = ma_slots(parent, p_type); ++ MT_BUG_ON(mas->tree, mas_mn(mas) == parent); ++ ++ /* Check prev/next parent slot for duplicate node entry */ ++ ++ for (i = 0; i < mt_slots[p_type]; i++) { ++ node = mas_slot(mas, slots, i); ++ if (i == p_slot) { ++ if (node != mas->node) ++ pr_err("parent %p[%u] does not have %p\n", ++ parent, i, mas_mn(mas)); ++ MT_BUG_ON(mas->tree, node != mas->node); ++ } else if (node == mas->node) { ++ pr_err("Invalid child %p at parent %p[%u] p_slot %u\n", ++ mas_mn(mas), parent, i, p_slot); ++ MT_BUG_ON(mas->tree, node == mas->node); ++ } ++ } ++} ++ ++static void mas_validate_child_slot(struct ma_state *mas) ++{ ++ enum maple_type type = mte_node_type(mas->node); ++ void __rcu **slots = ma_slots(mte_to_node(mas->node), type); ++ unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type); ++ struct maple_enode *child; ++ unsigned char i; ++ ++ if (mte_is_leaf(mas->node)) ++ return; ++ ++ for (i = 0; i < mt_slots[type]; i++) { ++ child = mas_slot(mas, slots, i); ++ if (!pivots[i] || pivots[i] == mas->max) ++ break; ++ ++ if (!child) ++ break; ++ ++ if (mte_parent_slot(child) != i) { ++ pr_err("Slot error at %p[%u]: child %p has pslot %u\n", ++ mas_mn(mas), i, mte_to_node(child), ++ mte_parent_slot(child)); ++ MT_BUG_ON(mas->tree, 1); ++ } ++ ++ if (mte_parent(child) != mte_to_node(mas->node)) { ++ pr_err("child %p has parent %p not %p\n", ++ mte_to_node(child), mte_parent(child), ++ mte_to_node(mas->node)); ++ MT_BUG_ON(mas->tree, 1); ++ } ++ } ++} ++ ++/* ++ * Validate all pivots are within mas->min and mas->max. ++ */ ++static void mas_validate_limits(struct ma_state *mas) ++{ ++ int i; ++ unsigned long prev_piv = 0; ++ enum maple_type type = mte_node_type(mas->node); ++ void __rcu **slots = ma_slots(mte_to_node(mas->node), type); ++ unsigned long *pivots = ma_pivots(mas_mn(mas), type); ++ ++ /* all limits are fine here. */ ++ if (mte_is_root(mas->node)) ++ return; ++ ++ for (i = 0; i < mt_slots[type]; i++) { ++ unsigned long piv; ++ ++ piv = mas_safe_pivot(mas, pivots, i, type); ++ ++ if (!piv && (i != 0)) ++ break; ++ ++ if (!mte_is_leaf(mas->node)) { ++ void *entry = mas_slot(mas, slots, i); ++ ++ if (!entry) ++ pr_err("%p[%u] cannot be null\n", ++ mas_mn(mas), i); ++ ++ MT_BUG_ON(mas->tree, !entry); ++ } ++ ++ if (prev_piv > piv) { ++ pr_err("%p[%u] piv %lu < prev_piv %lu\n", ++ mas_mn(mas), i, piv, prev_piv); ++ MT_BUG_ON(mas->tree, piv < prev_piv); ++ } ++ ++ if (piv < mas->min) { ++ pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i, ++ piv, mas->min); ++ MT_BUG_ON(mas->tree, piv < mas->min); ++ } ++ if (piv > mas->max) { ++ pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i, ++ piv, mas->max); ++ MT_BUG_ON(mas->tree, piv > mas->max); ++ } ++ prev_piv = piv; ++ if (piv == mas->max) ++ break; ++ } ++ for (i += 1; i < mt_slots[type]; i++) { ++ void *entry = mas_slot(mas, slots, i); ++ ++ if (entry && (i != mt_slots[type] - 1)) { ++ pr_err("%p[%u] should not have entry %p\n", mas_mn(mas), ++ i, entry); ++ MT_BUG_ON(mas->tree, entry != NULL); ++ } ++ ++ if (i < mt_pivots[type]) { ++ unsigned long piv = pivots[i]; ++ ++ if (!piv) ++ continue; ++ ++ pr_err("%p[%u] should not have piv %lu\n", ++ mas_mn(mas), i, piv); ++ MT_BUG_ON(mas->tree, i < mt_pivots[type] - 1); ++ } ++ } ++} ++ ++static void mt_validate_nulls(struct maple_tree *mt) ++{ ++ void *entry, *last = (void *)1; ++ unsigned char offset = 0; ++ void __rcu **slots; ++ MA_STATE(mas, mt, 0, 0); ++ ++ mas_start(&mas); ++ if (mas_is_none(&mas) || (mas.node == MAS_ROOT)) ++ return; ++ ++ while (!mte_is_leaf(mas.node)) ++ mas_descend(&mas); ++ ++ slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); ++ do { ++ entry = mas_slot(&mas, slots, offset); ++ if (!last && !entry) { ++ pr_err("Sequential nulls end at %p[%u]\n", ++ mas_mn(&mas), offset); ++ } ++ MT_BUG_ON(mt, !last && !entry); ++ last = entry; ++ if (offset == mas_data_end(&mas)) { ++ mas_next_node(&mas, mas_mn(&mas), ULONG_MAX); ++ if (mas_is_none(&mas)) ++ return; ++ offset = 0; ++ slots = ma_slots(mte_to_node(mas.node), ++ mte_node_type(mas.node)); ++ } else { ++ offset++; ++ } ++ ++ } while (!mas_is_none(&mas)); ++} ++ ++/* ++ * validate a maple tree by checking: ++ * 1. The limits (pivots are within mas->min to mas->max) ++ * 2. The gap is correctly set in the parents ++ */ ++void mt_validate(struct maple_tree *mt) ++{ ++ unsigned char end; ++ ++ MA_STATE(mas, mt, 0, 0); ++ rcu_read_lock(); ++ mas_start(&mas); ++ if (!mas_searchable(&mas)) ++ goto done; ++ ++ mas_first_entry(&mas, mas_mn(&mas), ULONG_MAX, mte_node_type(mas.node)); ++ while (!mas_is_none(&mas)) { ++ MT_BUG_ON(mas.tree, mte_dead_node(mas.node)); ++ if (!mte_is_root(mas.node)) { ++ end = mas_data_end(&mas); ++ if ((end < mt_min_slot_count(mas.node)) && ++ (mas.max != ULONG_MAX)) { ++ pr_err("Invalid size %u of %p\n", end, ++ mas_mn(&mas)); ++ MT_BUG_ON(mas.tree, 1); ++ } ++ ++ } ++ mas_validate_parent_slot(&mas); ++ mas_validate_child_slot(&mas); ++ mas_validate_limits(&mas); ++ if (mt_is_alloc(mt)) ++ mas_validate_gaps(&mas); ++ mas_dfs_postorder(&mas, ULONG_MAX); ++ } ++ mt_validate_nulls(mt); ++done: ++ rcu_read_unlock(); ++ ++} ++ ++#endif /* CONFIG_DEBUG_MAPLE_TREE */ +diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c +new file mode 100644 +index 000000000000..4f69e009a015 +--- /dev/null ++++ b/lib/test_maple_tree.c +@@ -0,0 +1,38307 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++/* ++ * test_maple_tree.c: Test the maple tree API ++ * Copyright (c) 2018 Liam R. Howlett ++ * Author: Liam R. Howlett ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#define MTREE_ALLOC_MAX 0x2000000000000Ul ++#define CONFIG_DEBUG_MAPLE_TREE ++#define CONFIG_MAPLE_SEARCH ++/* #define BENCH_SLOT_STORE */ ++/* #define BENCH_NODE_STORE */ ++/* #define BENCH_AWALK */ ++/* #define BENCH_WALK */ ++/* #define BENCH_MT_FOR_EACH */ ++/* #define BENCH_FORK */ ++static ++int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp) ++{ ++ return mtree_insert(mt, index, xa_mk_value(index & LONG_MAX), gfp); ++} ++ ++static void mtree_erase_index(struct maple_tree *mt, unsigned long index) ++{ ++ MT_BUG_ON(mt, mtree_erase(mt, index) != xa_mk_value(index & LONG_MAX)); ++ MT_BUG_ON(mt, mtree_load(mt, index) != NULL); ++} ++ ++static int mtree_test_insert(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ return mtree_insert(mt, index, ptr, GFP_KERNEL); ++} ++ ++static int mtree_test_store_range(struct maple_tree *mt, unsigned long start, ++ unsigned long end, void *ptr) ++{ ++ return mtree_store_range(mt, start, end, ptr, GFP_KERNEL); ++} ++ ++static int mtree_test_store(struct maple_tree *mt, unsigned long start, ++ void *ptr) ++{ ++ return mtree_test_store_range(mt, start, start, ptr); ++} ++ ++static int mtree_test_insert_range(struct maple_tree *mt, unsigned long start, ++ unsigned long end, void *ptr) ++{ ++ return mtree_insert_range(mt, start, end, ptr, GFP_KERNEL); ++} ++ ++static void *mtree_test_load(struct maple_tree *mt, unsigned long index) ++{ ++ return mtree_load(mt, index); ++} ++ ++static void *mtree_test_erase(struct maple_tree *mt, unsigned long index) ++{ ++ return mtree_erase(mt, index); ++} ++ ++static noinline void check_mtree_alloc_range(struct maple_tree *mt, ++ unsigned long start, unsigned long end, unsigned long size, ++ unsigned long expected, int eret, void *ptr) ++{ ++ ++ unsigned long result = expected + 1; ++ int ret; ++ ++ ret = mtree_alloc_range(mt, &result, ptr, size, start, end, ++ GFP_KERNEL); ++ MT_BUG_ON(mt, ret != eret); ++ if (ret) ++ return; ++ ++ MT_BUG_ON(mt, result != expected); ++} ++ ++static noinline void check_mtree_alloc_rrange(struct maple_tree *mt, ++ unsigned long start, unsigned long end, unsigned long size, ++ unsigned long expected, int eret, void *ptr) ++{ ++ ++ unsigned long result = expected + 1; ++ int ret; ++ ++ ret = mtree_alloc_rrange(mt, &result, ptr, size, start, end - 1, ++ GFP_KERNEL); ++ MT_BUG_ON(mt, ret != eret); ++ if (ret) ++ return; ++ ++ MT_BUG_ON(mt, result != expected); ++} ++ ++static noinline void check_load(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ void *ret = mtree_test_load(mt, index); ++ ++ if (ret != ptr) ++ pr_err("Load %lu returned %p expect %p\n", index, ret, ptr); ++ MT_BUG_ON(mt, ret != ptr); ++} ++ ++static noinline void check_store_range(struct maple_tree *mt, ++ unsigned long start, unsigned long end, void *ptr, int expected) ++{ ++ int ret = -EINVAL; ++ unsigned long i; ++ ++ ret = mtree_test_store_range(mt, start, end, ptr); ++ MT_BUG_ON(mt, ret != expected); ++ ++ if (ret) ++ return; ++ ++ for (i = start; i <= end; i++) ++ check_load(mt, i, ptr); ++} ++ ++static noinline void check_insert_range(struct maple_tree *mt, ++ unsigned long start, unsigned long end, void *ptr, int expected) ++{ ++ int ret = -EINVAL; ++ unsigned long i; ++ ++ ret = mtree_test_insert_range(mt, start, end, ptr); ++ MT_BUG_ON(mt, ret != expected); ++ ++ if (ret) ++ return; ++ ++ for (i = start; i <= end; i++) ++ check_load(mt, i, ptr); ++} ++ ++static noinline void check_insert(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ int ret = -EINVAL; ++ ++ ret = mtree_test_insert(mt, index, ptr); ++ MT_BUG_ON(mt, ret != 0); ++} ++ ++static noinline void check_erase(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr); ++} ++ ++static noinline void check_dup_insert(struct maple_tree *mt, ++ unsigned long index, void *ptr) ++{ ++ int ret = -EINVAL; ++ ++ ret = mtree_test_insert(mt, index, ptr); ++ MT_BUG_ON(mt, ret != -EEXIST); ++} ++ ++ ++static noinline ++void check_index_load(struct maple_tree *mt, unsigned long index) ++{ ++ return check_load(mt, index, xa_mk_value(index & LONG_MAX)); ++} ++ ++static noinline void check_nomem(struct maple_tree *mt) ++{ ++ MA_STATE(ms, mt, 1, 1); ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ /* Ensure no bypassing of allocation failures */ ++ mt_set_non_kernel(0); ++ ++ /* Storing something at 1 requires memory allocation */ ++ MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM); ++ /* Storing something at 0 does not */ ++ MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0); ++ ++ /* ++ * Simulate two threads racing; the first one fails to allocate ++ * memory to insert an entry at 1, then the second one succeeds ++ * in allocating memory to insert an entry at 2. The first one ++ * then needs to free the node it allocated. LeakSanitizer will ++ * notice this, as will the 'nr_allocated' debugging aid in the ++ * userspace test suite. ++ */ ++ mtree_lock(mt); ++ mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */ ++ MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM)); ++ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ ++ MT_BUG_ON(mt, ms.node != MAS_START); ++ mtree_unlock(mt); ++ MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0); ++ mtree_lock(mt); ++ mas_store(&ms, &ms); /* insert 1 -> &ms */ ++ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++} ++ ++static inline int not_empty(struct maple_node *node) ++{ ++ int i; ++ ++ if (node->parent) ++ return 1; ++ ++ for (i = 0; i < ARRAY_SIZE(node->slot); i++) ++ if (node->slot[i]) ++ return 1; ++ ++ return 0; ++} ++ ++static noinline void check_new_node(struct maple_tree *mt) ++{ ++ ++ struct maple_node *mn, *mn2, *mn3; ++ struct maple_alloc *smn; ++ struct maple_node *nodes[100]; ++ int i, j, total; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ /* Try allocating 3 nodes */ ++ mtree_lock(mt); ++ /* request 3 nodes to be allocated. */ ++ mas_node_count(&mas, 3); ++ /* Allocation request of 3. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); ++ /* Allocate failed. */ ++ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, mas.alloc == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); ++ mas_push_node(&mas, mn); ++ mas_nomem(&mas, GFP_KERNEL); /* free */ ++ mtree_unlock(mt); ++ ++ ++ /* Try allocating 1 node, then 2 more */ ++ mtree_lock(mt); ++ /* Set allocation request to 1. */ ++ mas_set_alloc_req(&mas, 1); ++ /* Check Allocation request of 1. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); ++ mas_set_err(&mas, -ENOMEM); ++ /* Validate allocation request. */ ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ /* Eat the requested node. */ ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, mn->slot[0] != NULL); ++ MT_BUG_ON(mt, mn->slot[1] != NULL); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ ma_free_rcu(mn); ++ mas.node = MAS_START; ++ mas_nomem(&mas, GFP_KERNEL); ++ /* Allocate 3 nodes, will fail. */ ++ mas_node_count(&mas, 3); ++ /* Drop the lock and allocate 3 nodes. */ ++ mas_nomem(&mas, GFP_KERNEL); ++ /* Ensure 3 are allocated. */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ /* Allocation request of 0. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); ++ ++ MT_BUG_ON(mt, mas.alloc == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); ++ /* Ensure we counted 3. */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ /* Free. */ ++ mas_nomem(&mas, GFP_KERNEL); ++ ++ /* Set allocation request to 1. */ ++ mas_set_alloc_req(&mas, 1); ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); ++ mas_set_err(&mas, -ENOMEM); ++ /* Validate allocation request. */ ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 1); ++ /* Check the node is only one node. */ ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, mn->slot[0] != NULL); ++ MT_BUG_ON(mt, mn->slot[1] != NULL); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 1); ++ MT_BUG_ON(mt, mas.alloc->node_count); ++ ++ mas_set_alloc_req(&mas, 2); /* request 2 more. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ MT_BUG_ON(mt, mas.alloc == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); ++ for (i = 2; i >= 0; i--) { ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); ++ MT_BUG_ON(mt, !mn); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ ++ total = 64; ++ mas_set_alloc_req(&mas, total); /* request 2 more. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != total); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ for (i = total; i > 0; i--) { ++ unsigned int e = 0; /* expected node_count */ ++ ++ if (i >= 35) ++ e = i - 35; ++ else if (i >= 5) ++ e = i - 5; ++ else if (i >= 2) ++ e = i - 2; ++ MT_BUG_ON(mt, mas.alloc->node_count != e); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); ++ MT_BUG_ON(mt, !mn); ++ ma_free_rcu(mn); ++ } ++ ++ total = 100; ++ for (i = 1; i < total; i++) { ++ mas_set_alloc_req(&mas, i); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ for (j = i; j > 0; j--) { ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); ++ MT_BUG_ON(mt, !mn); ++ MT_BUG_ON(mt, not_empty(mn)); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != j); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ mas_set_alloc_req(&mas, i); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ for (j = 0; j <= i/2; j++) { ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); ++ nodes[j] = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); ++ } ++ ++ while (j) { ++ j--; ++ mas_push_node(&mas, nodes[j]); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); ++ for (j = 0; j <= i/2; j++) { ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); ++ } ++ MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); ++ ++ } ++ ++ /* Set allocation request. */ ++ total = 500; ++ mas_node_count(&mas, total); ++ /* Drop the lock and allocate the nodes. */ ++ mas_nomem(&mas, GFP_KERNEL); ++ MT_BUG_ON(mt, !mas.alloc); ++ i = 1; ++ smn = mas.alloc; ++ while (i < total) { ++ for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) { ++ i++; ++ MT_BUG_ON(mt, !smn->slot[j]); ++ if (i == total) ++ break; ++ } ++ smn = smn->slot[0]; /* next. */ ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != total); ++ mas_nomem(&mas, GFP_KERNEL); /* Free. */ ++ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ for (i = 1; i < 128; i++) { ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ ++ for (j = i; j > 0; j--) { /*Free the requests */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ } ++ ++ for (i = 1; i < MAPLE_NODE_MASK + 1; i++) { ++ MA_STATE(mas2, mt, 0, 0); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ ++ for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ mas_push_node(&mas2, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas2) != j); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ MT_BUG_ON(mt, mas_allocated(&mas2) != i); ++ ++ for (j = i; j > 0; j--) { /*Free the requests */ ++ MT_BUG_ON(mt, mas_allocated(&mas2) != j); ++ mn = mas_pop_node(&mas2); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas2) != 0); ++ } ++ ++ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ ++ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); ++ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2); ++ ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); ++ ++ /* Check the limit of pop/push/pop */ ++ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); ++ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_alloc_req(&mas)); ++ MT_BUG_ON(mt, mas.alloc->node_count); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas.alloc->node_count); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) { ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ ++ for (i = 3; i < MAPLE_NODE_MASK * 3; i++) { ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ mas_push_node(&mas, mn); /* put it back */ ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ mn2 = mas_pop_node(&mas); /* get the next node. */ ++ mas_push_node(&mas, mn); /* put them back */ ++ mas_push_node(&mas, mn2); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ mn2 = mas_pop_node(&mas); /* get the next node. */ ++ mn3 = mas_pop_node(&mas); /* get the next node. */ ++ mas_push_node(&mas, mn); /* put them back */ ++ mas_push_node(&mas, mn2); ++ mas_push_node(&mas, mn3); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mas_destroy(&mas); ++ } ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, 5); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 5); ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, 10); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mas.node = MAS_START; ++ MT_BUG_ON(mt, mas_allocated(&mas) != 10); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mas.node = MAS_START; ++ MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); ++ mas_destroy(&mas); ++ ++ mtree_unlock(mt); ++} ++ ++static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, ++ bool verbose) ++{ ++ unsigned long i = max, j; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ ++ mt_zero_nr_tallocated(); ++ while (i) { ++ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); ++ for (j = i; j <= max; j++) ++ check_index_load(mt, j); ++ ++ check_load(mt, i - 1, NULL); ++ mt_set_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mt_clear_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ i--; ++ } ++ check_load(mt, max + 1, NULL); ++ ++ if (verbose) { ++ rcu_barrier(); ++ mt_dump(mt); ++ pr_info(" %s test of 0-%lu %luK in %d active (%d total)\n", ++ __func__, max, mt_get_alloc_size()/1024, mt_nr_allocated(), ++ mt_nr_tallocated()); ++ } ++} ++ ++static noinline void check_seq(struct maple_tree *mt, unsigned long max, ++ bool verbose) ++{ ++ unsigned long i, j; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ ++ mt_zero_nr_tallocated(); ++ for (i = 0; i <= max; i++) { ++ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); ++ for (j = 0; j <= i; j++) ++ check_index_load(mt, j); ++ ++ if (i) ++ MT_BUG_ON(mt, !mt_height(mt)); ++ check_load(mt, i + 1, NULL); ++ } ++ if (verbose) { ++ rcu_barrier(); ++ mt_dump(mt); ++ pr_info(" seq test of 0-%lu %luK in %d active (%d total)\n", ++ max, mt_get_alloc_size()/1024, mt_nr_allocated(), ++ mt_nr_tallocated()); ++ } ++} ++ ++static noinline void check_lb_not_empty(struct maple_tree *mt) ++{ ++ unsigned long i, j; ++ unsigned long huge = 4000UL * 1000 * 1000; ++ ++ ++ i = huge; ++ while (i > 4096) { ++ check_insert(mt, i, (void *) i); ++ for (j = huge; j >= i; j /= 2) { ++ check_load(mt, j-1, NULL); ++ check_load(mt, j, (void *) j); ++ check_load(mt, j+1, NULL); ++ } ++ i /= 2; ++ } ++ mtree_destroy(mt); ++} ++ ++static noinline void check_lower_bound_split(struct maple_tree *mt) ++{ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_lb_not_empty(mt); ++} ++ ++static noinline void check_upper_bound_split(struct maple_tree *mt) ++{ ++ unsigned long i, j; ++ unsigned long huge = 4000UL * 1000 * 1000; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ ++ i = 4096; ++ while (i < huge) { ++ check_insert(mt, i, (void *) i); ++ for (j = i; j >= huge; j *= 2) { ++ check_load(mt, j-1, NULL); ++ check_load(mt, j, (void *) j); ++ check_load(mt, j+1, NULL); ++ } ++ i *= 2; ++ } ++ mtree_destroy(mt); ++} ++ ++static noinline void check_mid_split(struct maple_tree *mt) ++{ ++ unsigned long huge = 8000UL * 1000 * 1000; ++ ++ check_insert(mt, huge, (void *) huge); ++ check_insert(mt, 0, xa_mk_value(0)); ++ check_lb_not_empty(mt); ++} ++ ++static noinline void check_rev_find(struct maple_tree *mt) ++{ ++ int i, nr_entries = 200; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mas_set(&mas, 1000); ++ val = mas_find_rev(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(100)); ++ val = mas_find_rev(&mas, 1000); ++ MT_BUG_ON(mt, val != NULL); ++ ++ mas_set(&mas, 999); ++ val = mas_find_rev(&mas, 997); ++ MT_BUG_ON(mt, val != NULL); ++ ++ mas_set(&mas, 1000); ++ val = mas_find_rev(&mas, 900); ++ MT_BUG_ON(mt, val != xa_mk_value(100)); ++ val = mas_find_rev(&mas, 900); ++ MT_BUG_ON(mt, val != xa_mk_value(99)); ++ ++ mas_set(&mas, 20); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(2)); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(1)); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(0)); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != NULL); ++} ++ ++static noinline void check_find(struct maple_tree *mt) ++{ ++ unsigned long val = 0; ++ unsigned long count = 20; ++ unsigned long max; ++ unsigned long last = 0, index = 0; ++ void *entry, *entry2; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ /* Insert 0. */ ++ MT_BUG_ON(mt, mtree_insert_index(mt, val++, GFP_KERNEL)); ++ ++ for (int i = 0; i <= count; i++) { ++ if (val != 64) ++ MT_BUG_ON(mt, mtree_insert_index(mt, val, GFP_KERNEL)); ++ else ++ MT_BUG_ON(mt, mtree_insert(mt, val, ++ XA_ZERO_ENTRY, GFP_KERNEL)); ++ ++ val <<= 2; ++ } ++ ++ val = 0; ++ mas_set(&mas, val); ++ mas_lock(&mas); ++ while ((entry = mas_find(&mas, 268435456)) != NULL) { ++ if (val != 64) ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ else ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ } ++ mas_unlock(&mas); ++ ++ val = 0; ++ mas_set(&mas, val); ++ mas_lock(&mas); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (val != 64) ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ else ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ } ++ mas_unlock(&mas); ++ ++ /* Test mas_pause */ ++ val = 0; ++ mas_set(&mas, val); ++ mas_lock(&mas); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (val != 64) ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ else ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ ++ mas_pause(&mas); ++ mas_unlock(&mas); ++ mas_lock(&mas); ++ } ++ mas_unlock(&mas); ++ ++ val = 0; ++ max = 300; /* A value big enough to include XA_ZERO_ENTRY at 64. */ ++ mt_for_each(mt, entry, index, max) { ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ val <<= 2; ++ if (val == 64) /* Skip zero entry. */ ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ } ++ ++ val = 0; ++ max = 0; ++ index = 0; ++ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); ++ mt_for_each(mt, entry, index, ULONG_MAX) { ++ if (val == 4398046511104) ++ MT_BUG_ON(mt, entry != ++ xa_mk_value(ULONG_MAX & LONG_MAX)); ++ else ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ val <<= 2; ++ if (val == 64) /* Skip zero entry. */ ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ max++; ++ MT_BUG_ON(mt, max > 25); ++ } ++ mtree_erase_index(mt, ULONG_MAX); ++ ++ mas_reset(&mas); ++ index = 17; ++ entry = mt_find(mt, &index, 512); ++ MT_BUG_ON(mt, xa_mk_value(256) != entry); ++ ++ mas_reset(&mas); ++ index = 17; ++ entry = mt_find(mt, &index, 20); ++ MT_BUG_ON(mt, entry != NULL); ++ ++ ++ /* Range check.. */ ++ /* Insert ULONG_MAX */ ++ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); ++ ++ val = 0; ++ mas_set(&mas, 0); ++ mas_lock(&mas); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (val == 64) ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ else if (val == 4398046511104) ++ MT_BUG_ON(mt, entry != xa_mk_value(ULONG_MAX & LONG_MAX)); ++ else ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ val <<= 2; ++ ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ mas_pause(&mas); ++ mas_unlock(&mas); ++ mas_lock(&mas); ++ } ++ mas_unlock(&mas); ++ ++ mas_set(&mas, 1048576); ++ mas_lock(&mas); ++ entry = mas_find(&mas, 1048576); ++ mas_unlock(&mas); ++ MT_BUG_ON(mas.tree, entry == NULL); ++ ++ /* ++ * Find last value. ++ * 1. get the expected value, leveraging the existence of an end entry ++ * 2. delete end entry ++ * 3. find the last value but searching for ULONG_MAX and then using ++ * prev ++ */ ++ /* First, get the expected result. */ ++ mas_lock(&mas); ++ mas_reset(&mas); ++ mas.index = ULONG_MAX; /* start at max.. */ ++ entry = mas_find(&mas, ULONG_MAX); ++ entry = mas_prev(&mas, 0); ++ index = mas.index; ++ last = mas.last; ++ ++ /* Erase the last entry. */ ++ mas_reset(&mas); ++ mas.index = ULONG_MAX; ++ mas.last = ULONG_MAX; ++ mas_erase(&mas); ++ ++ /* Get the previous value from MAS_START */ ++ mas_reset(&mas); ++ entry2 = mas_prev(&mas, 0); ++ ++ /* Check results. */ ++ MT_BUG_ON(mt, entry != entry2); ++ MT_BUG_ON(mt, index != mas.index); ++ MT_BUG_ON(mt, last != mas.last); ++ ++ ++ mas.node = MAS_NONE; ++ mas.index = ULONG_MAX; ++ mas.last = ULONG_MAX; ++ entry2 = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, entry != entry2); ++ ++ mas_set(&mas, 0); ++ MT_BUG_ON(mt, mas_prev(&mas, 0) != NULL); ++ ++ mas_unlock(&mas); ++ mtree_destroy(mt); ++} ++ ++static noinline void check_find_2(struct maple_tree *mt) ++{ ++ unsigned long i, j; ++ void *entry; ++ ++ MA_STATE(mas, mt, 0, 0); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) ++ MT_BUG_ON(mt, true); ++ rcu_read_unlock(); ++ ++ for (i = 0; i < 256; i++) { ++ mtree_insert_index(mt, i, GFP_KERNEL); ++ j = 0; ++ mas_set(&mas, 0); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ MT_BUG_ON(mt, entry != xa_mk_value(j)); ++ j++; ++ } ++ rcu_read_unlock(); ++ MT_BUG_ON(mt, j != i + 1); ++ } ++ ++ for (i = 0; i < 256; i++) { ++ mtree_erase_index(mt, i); ++ j = i + 1; ++ mas_set(&mas, 0); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (xa_is_zero(entry)) ++ continue; ++ ++ MT_BUG_ON(mt, entry != xa_mk_value(j)); ++ j++; ++ } ++ rcu_read_unlock(); ++ MT_BUG_ON(mt, j != 256); ++ } ++ ++ /*MT_BUG_ON(mt, !mtree_empty(mt)); */ ++} ++ ++#define erase_ptr(i) entry[i%2] ++#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2]) ++#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2]) ++#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2]) ++ ++static noinline void check_erase_testset(struct maple_tree *mt) ++{ ++ unsigned long set[] = { 5015, 5014, 5017, 25, 1000, ++ 1001, 1002, 1003, 1005, 0, ++ 6003, 6002, 6008, 6012, 6015, ++ 7003, 7002, 7008, 7012, 7015, ++ 8003, 8002, 8008, 8012, 8015, ++ 9003, 9002, 9008, 9012, 9015, ++ 10003, 10002, 10008, 10012, 10015, ++ 11003, 11002, 11008, 11012, 11015, ++ 12003, 12002, 12008, 12012, 12015, ++ 13003, 13002, 13008, 13012, 13015, ++ 14003, 14002, 14008, 14012, 14015, ++ 15003, 15002, 15008, 15012, 15015, ++ }; ++ ++ ++ void *ptr = &set; ++ void *entry[2] = { ptr, mt }; ++ void *root_node; ++ ++ ++ rcu_register_thread(); ++ mt_set_in_rcu(mt); ++ for (int i = 0; i < 4; i++) ++ erase_check_insert(mt, i); ++ for (int i = 0; i < 4; i++) ++ erase_check_load(mt, i); ++ ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 1); ++ erase_check_load(mt, 0); ++ check_load(mt, set[1], NULL); ++ for (int i = 2; i < 4; i++) ++ erase_check_load(mt, i); ++ ++ ++ erase_check_erase(mt, 2); ++ erase_check_load(mt, 0); ++ check_load(mt, set[1], NULL); ++ check_load(mt, set[2], NULL); ++ ++ erase_check_insert(mt, 1); ++ erase_check_insert(mt, 2); ++ ++ for (int i = 0; i < 4; i++) ++ erase_check_load(mt, i); ++ ++ /* Check erase and load without an allocation. */ ++ erase_check_load(mt, 3); ++ erase_check_erase(mt, 1); ++ erase_check_load(mt, 0); ++ check_load(mt, set[1], NULL); ++ for (int i = 2; i < 4; i++) ++ erase_check_load(mt, i); ++ ++ /* ++ * Set the newly erased node. This will produce a different allocated ++ * node to avoid busy slots. ++ */ ++ root_node = mt->ma_root; ++ erase_check_insert(mt, 1); ++ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ erase_check_load(mt, 2); ++ check_load(mt, 5018, NULL); ++ erase_check_load(mt, 3); ++ ++ erase_check_erase(mt, 2); /* erase 5017 to check append */ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ check_load(mt, set[2], NULL); ++ check_load(mt, 5018, NULL); ++ ++ erase_check_load(mt, 3); ++ ++ root_node = mt->ma_root; ++ erase_check_insert(mt, 2); ++ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ erase_check_load(mt, 2); ++ check_load(mt, 5018, NULL); ++ erase_check_load(mt, 3); ++ ++ mt_set_non_kernel(1); ++ erase_check_erase(mt, 2); /* erase 5017 to check append */ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ check_load(mt, set[2], NULL); ++ erase_check_erase(mt, 0); /* erase 5015 to check append */ ++ check_load(mt, set[0], NULL); ++ check_load(mt, 5016, NULL); ++ erase_check_insert(mt, 4); /* 1000 < Should not split. */ ++ check_load(mt, set[0], NULL); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ check_load(mt, set[2], NULL); ++ check_load(mt, 5018, NULL); ++ erase_check_load(mt, 4); ++ check_load(mt, 999, NULL); ++ check_load(mt, 1001, NULL); ++ erase_check_load(mt, 4); ++ if (mt_in_rcu(mt)) ++ MT_BUG_ON(mt, root_node == mt->ma_root); ++ else ++ MT_BUG_ON(mt, root_node != mt->ma_root); ++ ++ /* Should not have split. */ ++ MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root)); ++ ++ ++ /* Coalesce testing */ ++ erase_check_insert(mt, 0); ++ erase_check_insert(mt, 2); ++ ++ for (int i = 5; i < 25; i++) { ++ erase_check_insert(mt, i); ++ for (int j = i; j >= 0; j--) ++ erase_check_load(mt, j); ++ } ++ ++ erase_check_erase(mt, 14); /*6015 */ ++ for (int i = 0; i < 25; i++) { ++ if (i == 14) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ erase_check_erase(mt, 16); /*7002 */ ++ for (int i = 0; i < 25; i++) { ++ if (i == 16 || i == 14) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ ++ mt_set_non_kernel(1); ++ erase_check_erase(mt, 13); /*6012 */ ++ for (int i = 0; i < 25; i++) { ++ if (i == 16 || i == 14 || i == 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ erase_check_erase(mt, 15); /*7003 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 16 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 17 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ erase_check_erase(mt, 18); /*7012 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 18 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 19); /*7015 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 19 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ erase_check_erase(mt, 20); /*8003 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 20 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ erase_check_erase(mt, 21); /*8002 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 21 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 22); /*8008 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 22 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ for (int i = 23; i < 25; i++) ++ erase_check_erase(mt, i); ++ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 25 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ ++ /* Shrinking tree test. */ ++ ++ for (int i = 13; i < ARRAY_SIZE(set); i++) ++ erase_check_insert(mt, i); ++ ++ mt_set_non_kernel(99); ++ for (int i = 18; i < ARRAY_SIZE(set); i++) { ++ erase_check_erase(mt, i); ++ for (int j = 0; j < ARRAY_SIZE(set); j++) { ++ if (j < 18 || j > i) ++ erase_check_load(mt, j); ++ else ++ check_load(mt, set[j], NULL); ++ } ++ } ++ mt_set_non_kernel(35); ++ for (int i = 0; i < 18; i++) { ++ erase_check_erase(mt, i); ++ for (int j = 0; j < ARRAY_SIZE(set); j++) { ++ if (j < 18 && j > i) ++ erase_check_load(mt, j); ++ else ++ check_load(mt, set[j], NULL); ++ } ++ } ++ erase_check_insert(mt, 8); ++ erase_check_insert(mt, 9); ++ erase_check_erase(mt, 8); ++ rcu_unregister_thread(); ++} ++ ++#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \ ++ a[(i)], a[(i + 1)], ptr) ++#define STORE 1 ++#define SNULL 2 ++#define ERASE 3 ++#define ec_type_str(x) \ ++ (((x) == STORE) ? \ ++ "STORE" : \ ++ (((x) == SNULL) ? \ ++ "SNULL" : "ERASE") \ ++ ) ++#define check_erase2_debug 0 ++void *mas_next(struct ma_state *mas, unsigned long max); ++ ++/* Calculate the overwritten entries. */ ++int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end, ++ void *s_entry, unsigned long s_min, ++ void *e_entry, unsigned long e_max, ++ unsigned long *set, int i, bool null_entry) ++{ ++ int count = 0, span = 0; ++ unsigned long retry = 0; ++ void *entry; ++ struct ma_state tmp; ++ ++ ++ /* count slots */ ++ memcpy(&tmp, mas_start, sizeof(tmp)); ++ entry = mas_next(&tmp, mas_end->last); ++ while (entry) { ++ BUG_ON(retry > 50); /* stop infinite retry on testing. */ ++ if (xa_is_zero(s_entry)) { ++ retry++; ++ continue; ++ } ++ count++; ++ span++; ++ entry = mas_next(&tmp, mas_end->last); ++ } ++ ++ if (null_entry) { ++ /* Check splitting end. */ ++ if (e_entry && (e_max > mas_end->last)) ++ count--; ++ ++ /* check overwrite of entire start */ ++ if (s_entry && (s_min == mas_start->index)) ++ count++; ++ } else { /* !null_entry (store) */ ++ bool esplit = e_max > mas_end->last; ++ bool ssplit = s_min != mas_start->index; ++ ++ if (s_entry && e_entry) { ++ if (esplit && ssplit) ++ count--; ++ else if (ssplit) ++ count--; ++ else if (esplit) { ++ if (span) ++ count--; ++ } ++ } else if (s_entry && !e_entry) { ++ if (ssplit) ++ count--; ++ } else if (!s_entry && e_entry) { ++ if (esplit) ++ count--; ++ count--; ++ } else { ++ count--; ++ } ++ } ++ return count; ++} ++ ++/* ++ * mas_node_walk() - Walk a maple node to offset of the index. ++ * @mas: The maple state ++ * @type: The maple node type ++ * @*range_min: Pointer to store the minimum range of the offset ++ * @*range_max: Pointer to store the maximum range of the offset ++ * ++ * The offset will be stored in the maple state. ++ * ++ */ ++static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, ++ enum maple_type type, unsigned long *range_min, ++ unsigned long *range_max) ++ ++{ ++ unsigned long *pivots; ++ unsigned char count; ++ unsigned long prev, max; ++ unsigned char offset; ++ unsigned long index; ++ ++ if (unlikely(ma_is_dense(type))) { ++ (*range_max) = (*range_min) = mas->index; ++ if (unlikely(ma_dead_node(node))) ++ return; ++ ++ mas->offset = mas->index = mas->min; ++ return; ++ } ++ ++ pivots = ma_pivots(node, type); ++ max = pivots[0]; ++ if (unlikely(ma_dead_node(node))) ++ return; ++ ++ offset = 0; ++ prev = mas->min; ++ index = mas->index; ++ if (unlikely(index <= max)) ++ goto offset_zero; ++ ++ count = mt_pivots[type]; ++ while (++offset < count) { ++ prev = max; ++ max = pivots[offset]; ++ if (unlikely(ma_dead_node(node))) ++ return; ++ ++ if (index <= max) ++ goto offset_found; ++ else if (unlikely(!max)) ++ goto mas_max; ++ } ++ ++ prev = max; ++mas_max: ++ max = mas->max; ++offset_found: ++ prev++; ++offset_zero: ++ mas->offset = offset; ++ if (ma_is_leaf(type)) { ++ *range_max = max; ++ *range_min = prev; ++ } else { ++ mas->max = max; ++ mas->min = prev; ++ } ++} ++ ++/* ++ * mas_descend_walk(): Locates a value and sets the mas->node and slot ++ * accordingly. range_min and range_max are set to the range which the entry is ++ * valid. ++ * @mas: The maple state ++ * @*range_min: A pointer to store the minimum of the range ++ * @*range_max: A pointer to store the maximum of the range ++ * ++ * Check mas->node is still valid on return of any value. ++ * ++ * Return: true if pointing to a valid node and offset. False otherwise. ++ */ ++static inline bool mas_descend_walk(struct ma_state *mas, ++ unsigned long *range_min, unsigned long *range_max) ++{ ++ struct maple_enode *next; ++ struct maple_node *node; ++ enum maple_type type; ++ ++ next = mas->node; ++ while (true) { ++ node = mte_to_node(next); ++ type = mte_node_type(next); ++ mas_node_walk(mas, node, type, range_min, range_max); ++ next = mas_slot(mas, ma_slots(node, type), mas->offset); ++ if (unlikely(ma_dead_node(node))) ++ return false; ++ ++ if (unlikely(ma_is_leaf(type))) ++ return true; ++ ++ /* Descend. */ ++ mas->node = next; ++ } ++ return false; ++} ++ ++/* ++ * mas_tree_walk() - Walk to @mas->index and set the range values. ++ * @mas: The maple state. ++ * @*range_min: The minimum range to be set. ++ * @*range_max: The maximum range to be set. ++ * ++ * Ranges are only valid if there is a valid entry at @mas->index. ++ * ++ * Return: True if a value exists, false otherwise. ++ */ ++static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, ++ unsigned long *range_max) ++{ ++ bool ret; ++ ++retry: ++ ret = false; ++ mas_start(mas); ++ if (mas_is_none(mas)) ++ goto not_found; ++ ++ if (mas_is_ptr(mas)) { ++ *range_min = *range_max = 0; ++ if (!mas->index) ++ return true; ++ ++ goto not_found; ++ } ++ ++ ret = mas_descend_walk(mas, range_min, range_max); ++ if (unlikely(mte_dead_node(mas->node))) { ++ mas->node = MAS_START; ++ goto retry; ++ } ++ ++ return ret; ++ ++not_found: ++ mas->offset = MAPLE_NODE_SLOTS; ++ return false; ++} ++ ++static inline void *mas_range_load(struct ma_state *mas, ++ unsigned long *range_min, unsigned long *range_max) ++ ++{ ++ void *entry = NULL; ++ unsigned long index = mas->index; ++ ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; ++retry: ++ if (mas_tree_walk(mas, range_min, range_max)) ++ if (unlikely(mas->node == MAS_ROOT)) ++ return mas_root(mas); ++ ++ if (likely(mas->offset != MAPLE_NODE_SLOTS)) ++ entry = mas_get_slot(mas, mas->offset); ++ ++ if (mas_dead_node(mas, index)) ++ goto retry; ++ ++ return entry; ++} ++static noinline void check_erase2_testset(struct maple_tree *mt, ++ unsigned long *set, unsigned long size) ++{ ++ int entry_count = 0; ++ int check = 0; ++ void *foo; ++ unsigned long addr = 0; ++ void *s_entry = NULL, *e_entry = NULL; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ for (int i = 0; i < size; i += 3) { ++ unsigned long s_min, s_max; ++ unsigned long e_min, e_max; ++ void *value = NULL; ++ ++ MA_STATE(mas_start, mt, set[i+1], set[i+1]); ++ MA_STATE(mas_end, mt, set[i+2], set[i+2]); ++ mt_set_non_kernel(127); ++#if check_erase2_debug ++ pr_err("%s: %d %s %lu - %lu\n", __func__, i, ++ ec_type_str(set[i]), ++ set[i+1], set[i+2]); ++#endif ++ s_entry = mas_range_load(&mas_start, &s_min, &s_max); ++ e_entry = mas_range_load(&mas_end, &e_min, &e_max); ++ ++ switch (set[i]) { ++ case SNULL: ++ if ((s_min == set[i+1]) && (s_max == set[i+2])) { ++ if (s_entry) ++ entry_count--; ++ } else if ((s_min != set[i+1]) && (s_max != set[i+2])) { ++ entry_count++; ++ } else if ((mas_start.node != mas_end.node) || ++ (mas_start.offset != mas_end.offset)) { ++ entry_count -= ++ mas_ce2_over_count(&mas_start, &mas_end, ++ s_entry, s_min, ++ e_entry, e_max, set, i, ++ true); ++ } ++ ++ ++ erase_check_store_range(mt, set, i + 1, value); ++ break; ++ case STORE: ++ value = xa_mk_value(set[i + 1]); ++ if (mas_start.offset > mt_slot_count(mas_start.node)) { ++ entry_count++; /* appending an entry. */ ++ } else if ((s_min == e_min) && (s_max == e_max)) { ++ if (!entry_count) ++ entry_count++; ++ ++ else if (s_entry) { ++ if (e_max > mas_end.last) ++ entry_count++; ++ ++ if (s_min < mas_start.index) ++ entry_count++; ++ ++ } else { ++ entry_count++; ++ } ++ } else { ++ entry_count -= ++ mas_ce2_over_count(&mas_start, &mas_end, ++ s_entry, s_min, ++ e_entry, e_max, set, i, ++ false); ++ } ++ ++ erase_check_store_range(mt, set, i + 1, value); ++ break; ++ case ERASE: ++ if (!s_entry) ++ break; ++ check_erase(mt, set[i+1], xa_mk_value(set[i+1])); ++ entry_count--; ++ break; ++ } ++ mt_validate(mt); ++ if (entry_count) ++ MT_BUG_ON(mt, !mt_height(mt)); ++#if check_erase2_debug > 1 ++ mt_dump(mt); ++#endif ++#if check_erase2_debug ++ pr_err("Done\n"); ++#endif ++ ++ check = 0; ++ addr = 0; ++ mt_for_each(mt, foo, addr, ULONG_MAX) { ++ check++; ++#if check_erase2_debug > 2 ++ pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check); ++#endif ++ if (check > entry_count) ++ break; ++ } ++ ++#if check_erase2_debug > 2 ++ pr_err("mt_for_each %d and count %d\n", check, entry_count); ++#endif ++ ++ MT_BUG_ON(mt, check != entry_count); ++ ++ check = 0; ++ addr = 0; ++ mas_reset(&mas); ++ mas.index = 0; ++ rcu_read_lock(); ++ mas_for_each(&mas, foo, ULONG_MAX) { ++ if (xa_is_zero(foo)) { ++ if (addr == mas.index) { ++ mt_dump(mas.tree); ++ pr_err("retry failed %lu - %lu\n", ++ mas.index, mas.last); ++ MT_BUG_ON(mt, 1); ++ } ++ addr = mas.index; ++ continue; ++ } ++#if check_erase2_debug > 2 ++ pr_err("mas: %lu -> %p\n", mas.index, foo); ++#endif ++ check++; ++ if (check > entry_count) ++ break; ++ } ++ rcu_read_unlock(); ++#if check_erase2_debug > 2 ++ pr_err("mas_for_each %d and count %d\n", check, entry_count); ++ mt_validate(mt); ++#endif ++ ++ MT_BUG_ON(mt, check != entry_count); ++ ++ MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); ++ } ++} ++ ++ ++/* These tests were pulled from kvm tests. */ ++static noinline void check_erase2_sets(struct maple_tree *mt) ++{ ++ void *entry; ++ unsigned long start = 0; ++ unsigned long set[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140721266458624, 140737488351231, ++ERASE, 140721266458624, 140737488351231, ++STORE, 140721266458624, 140721266462719, ++STORE, 94735788949504, 94735789121535, ++ERASE, 94735788949504, 94735789121535, ++STORE, 94735788949504, 94735788965887, ++STORE, 94735788965888, 94735789121535, ++ERASE, 94735788965888, 94735789121535, ++STORE, 94735788965888, 94735789068287, ++STORE, 94735789068288, 94735789109247, ++STORE, 94735789109248, 94735789121535, ++STORE, 140253902692352, 140253902864383, ++ERASE, 140253902692352, 140253902864383, ++STORE, 140253902692352, 140253902696447, ++STORE, 140253902696448, 140253902864383, ++ }; ++ unsigned long set2[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735933583360, 140737488351231, ++ERASE, 140735933583360, 140737488351231, ++STORE, 140735933583360, 140735933587455, ++STORE, 94811003260928, 94811003432959, ++ERASE, 94811003260928, 94811003432959, ++STORE, 94811003260928, 94811003277311, ++STORE, 94811003277312, 94811003432959, ++ERASE, 94811003277312, 94811003432959, ++STORE, 94811003277312, 94811003379711, ++STORE, 94811003379712, 94811003420671, ++STORE, 94811003420672, 94811003432959, ++STORE, 140277094653952, 140277094825983, ++ERASE, 140277094653952, 140277094825983, ++STORE, 140277094653952, 140277094658047, ++STORE, 140277094658048, 140277094825983, ++ERASE, 140277094658048, 140277094825983, ++STORE, 140277094658048, 140277094780927, ++STORE, 140277094780928, 140277094813695, ++STORE, 140277094813696, 140277094821887, ++STORE, 140277094821888, 140277094825983, ++STORE, 140735933906944, 140735933911039, ++ }; ++ unsigned long set3[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735790264320, 140737488351231, ++ERASE, 140735790264320, 140737488351231, ++STORE, 140735790264320, 140735790268415, ++STORE, 94016597282816, 94016597454847, ++ERASE, 94016597282816, 94016597454847, ++STORE, 94016597282816, 94016597299199, ++STORE, 94016597299200, 94016597454847, ++ERASE, 94016597299200, 94016597454847, ++STORE, 94016597299200, 94016597401599, ++STORE, 94016597401600, 94016597442559, ++STORE, 94016597442560, 94016597454847, ++STORE, 140496959283200, 140496959455231, ++ERASE, 140496959283200, 140496959455231, ++STORE, 140496959283200, 140496959287295, ++STORE, 140496959287296, 140496959455231, ++ERASE, 140496959287296, 140496959455231, ++STORE, 140496959287296, 140496959410175, ++STORE, 140496959410176, 140496959442943, ++STORE, 140496959442944, 140496959451135, ++STORE, 140496959451136, 140496959455231, ++STORE, 140735791718400, 140735791722495, ++STORE, 140735791706112, 140735791718399, ++STORE, 47135835713536, 47135835721727, ++STORE, 47135835721728, 47135835729919, ++STORE, 47135835729920, 47135835893759, ++ERASE, 47135835729920, 47135835893759, ++STORE, 47135835729920, 47135835742207, ++STORE, 47135835742208, 47135835893759, ++STORE, 47135835840512, 47135835893759, ++STORE, 47135835742208, 47135835840511, ++ERASE, 47135835742208, 47135835840511, ++STORE, 47135835742208, 47135835840511, ++STORE, 47135835885568, 47135835893759, ++STORE, 47135835840512, 47135835885567, ++ERASE, 47135835840512, 47135835885567, ++STORE, 47135835840512, 47135835893759, ++ERASE, 47135835840512, 47135835893759, ++STORE, 47135835840512, 47135835885567, ++STORE, 47135835885568, 47135835893759, ++ }; ++ ++ unsigned long set4[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140728251703296, 140737488351231, ++ERASE, 140728251703296, 140737488351231, ++STORE, 140728251703296, 140728251707391, ++STORE, 94668429205504, 94668429377535, ++ERASE, 94668429205504, 94668429377535, ++STORE, 94668429205504, 94668429221887, ++STORE, 94668429221888, 94668429377535, ++ERASE, 94668429221888, 94668429377535, ++STORE, 94668429221888, 94668429324287, ++STORE, 94668429324288, 94668429365247, ++STORE, 94668429365248, 94668429377535, ++STORE, 47646523273216, 47646523445247, ++ERASE, 47646523273216, 47646523445247, ++STORE, 47646523273216, 47646523277311, ++STORE, 47646523277312, 47646523445247, ++ERASE, 47646523277312, 47646523445247, ++STORE, 47646523277312, 47646523400191, ++ }; ++ ++ unsigned long set5[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140726874062848, 140737488351231, ++ERASE, 140726874062848, 140737488351231, ++STORE, 140726874062848, 140726874066943, ++STORE, 94248892870656, 94248893042687, ++ERASE, 94248892870656, 94248893042687, ++STORE, 94248892870656, 94248892887039, ++STORE, 94248892887040, 94248893042687, ++ERASE, 94248892887040, 94248893042687, ++STORE, 94248892887040, 94248892989439, ++STORE, 94248892989440, 94248893030399, ++STORE, 94248893030400, 94248893042687, ++STORE, 47884786266112, 47884786438143, ++ERASE, 47884786266112, 47884786438143, ++STORE, 47884786266112, 47884786270207, ++STORE, 47884786270208, 47884786438143, ++ERASE, 47884786270208, 47884786438143, ++STORE, 47884786270208, 47884786393087, ++STORE, 47884786393088, 47884786425855, ++STORE, 47884786425856, 47884786434047, ++STORE, 47884786434048, 47884786438143, ++STORE, 140726874513408, 140726874517503, ++STORE, 140726874501120, 140726874513407, ++STORE, 47884786438144, 47884786446335, ++STORE, 47884786446336, 47884786454527, ++STORE, 47884786454528, 47884786618367, ++ERASE, 47884786454528, 47884786618367, ++STORE, 47884786454528, 47884786466815, ++STORE, 47884786466816, 47884786618367, ++STORE, 47884786565120, 47884786618367, ++STORE, 47884786466816, 47884786565119, ++ERASE, 47884786466816, 47884786565119, ++STORE, 47884786466816, 47884786565119, ++STORE, 47884786610176, 47884786618367, ++STORE, 47884786565120, 47884786610175, ++ERASE, 47884786565120, 47884786610175, ++STORE, 47884786565120, 47884786618367, ++ERASE, 47884786565120, 47884786618367, ++STORE, 47884786565120, 47884786610175, ++STORE, 47884786610176, 47884786618367, ++ERASE, 47884786610176, 47884786618367, ++STORE, 47884786610176, 47884786618367, ++STORE, 47884786618368, 47884789669887, ++STORE, 47884787163136, 47884789669887, ++STORE, 47884786618368, 47884787163135, ++ERASE, 47884787163136, 47884789669887, ++STORE, 47884787163136, 47884789448703, ++STORE, 47884789448704, 47884789669887, ++STORE, 47884788858880, 47884789448703, ++STORE, 47884787163136, 47884788858879, ++ERASE, 47884787163136, 47884788858879, ++STORE, 47884787163136, 47884788858879, ++STORE, 47884789444608, 47884789448703, ++STORE, 47884788858880, 47884789444607, ++ERASE, 47884788858880, 47884789444607, ++STORE, 47884788858880, 47884789444607, ++STORE, 47884789653504, 47884789669887, ++STORE, 47884789448704, 47884789653503, ++ERASE, 47884789448704, 47884789653503, ++STORE, 47884789448704, 47884789653503, ++ERASE, 47884789653504, 47884789669887, ++STORE, 47884789653504, 47884789669887, ++STORE, 47884789669888, 47884791508991, ++STORE, 47884789809152, 47884791508991, ++STORE, 47884789669888, 47884789809151, ++ERASE, 47884789809152, 47884791508991, ++STORE, 47884789809152, 47884791468031, ++STORE, 47884791468032, 47884791508991, ++STORE, 47884791152640, 47884791468031, ++STORE, 47884789809152, 47884791152639, ++ERASE, 47884789809152, 47884791152639, ++STORE, 47884789809152, 47884791152639, ++STORE, 47884791463936, 47884791468031, ++STORE, 47884791152640, 47884791463935, ++ERASE, 47884791152640, 47884791463935, ++STORE, 47884791152640, 47884791463935, ++STORE, 47884791492608, 47884791508991, ++STORE, 47884791468032, 47884791492607, ++ERASE, 47884791468032, 47884791492607, ++STORE, 47884791468032, 47884791492607, ++ERASE, 47884791492608, 47884791508991, ++STORE, 47884791492608, 47884791508991, ++STORE, 47884791508992, 47884791644159, ++ERASE, 47884791508992, 47884791644159, ++STORE, 47884791508992, 47884791533567, ++STORE, 47884791533568, 47884791644159, ++STORE, 47884791595008, 47884791644159, ++STORE, 47884791533568, 47884791595007, ++ERASE, 47884791533568, 47884791595007, ++STORE, 47884791533568, 47884791595007, ++STORE, 47884791619584, 47884791644159, ++STORE, 47884791595008, 47884791619583, ++ERASE, 47884791595008, 47884791619583, ++STORE, 47884791595008, 47884791644159, ++ERASE, 47884791595008, 47884791644159, ++STORE, 47884791595008, 47884791619583, ++STORE, 47884791619584, 47884791644159, ++STORE, 47884791627776, 47884791644159, ++STORE, 47884791619584, 47884791627775, ++ERASE, 47884791619584, 47884791627775, ++STORE, 47884791619584, 47884791627775, ++ERASE, 47884791627776, 47884791644159, ++STORE, 47884791627776, 47884791644159, ++STORE, 47884791644160, 47884791664639, ++ERASE, 47884791644160, 47884791664639, ++STORE, 47884791644160, 47884791648255, ++STORE, 47884791648256, 47884791664639, ++STORE, 47884791652352, 47884791664639, ++STORE, 47884791648256, 47884791652351, ++ERASE, 47884791648256, 47884791652351, ++STORE, 47884791648256, 47884791652351, ++STORE, 47884791656448, 47884791664639, ++STORE, 47884791652352, 47884791656447, ++ERASE, 47884791652352, 47884791656447, ++STORE, 47884791652352, 47884791664639, ++ERASE, 47884791652352, 47884791664639, ++STORE, 47884791652352, 47884791656447, ++STORE, 47884791656448, 47884791664639, ++ERASE, 47884791656448, 47884791664639, ++STORE, 47884791656448, 47884791664639, ++STORE, 47884791664640, 47884791672831, ++ERASE, 47884791468032, 47884791492607, ++STORE, 47884791468032, 47884791484415, ++STORE, 47884791484416, 47884791492607, ++ERASE, 47884791656448, 47884791664639, ++STORE, 47884791656448, 47884791660543, ++STORE, 47884791660544, 47884791664639, ++ERASE, 47884791619584, 47884791627775, ++STORE, 47884791619584, 47884791623679, ++STORE, 47884791623680, 47884791627775, ++ }; ++ ++ unsigned long set6[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722999021568, 140737488351231, ++ERASE, 140722999021568, 140737488351231, ++STORE, 140722999021568, 140722999025663, ++STORE, 94901500268544, 94901500440575, ++ERASE, 94901500268544, 94901500440575, ++STORE, 94901500268544, 94901500284927, ++STORE, 94901500284928, 94901500440575, ++ERASE, 94901500284928, 94901500440575, ++STORE, 94901500284928, 94901500387327, ++STORE, 94901500387328, 94901500428287, ++STORE, 94901500428288, 94901500440575, ++STORE, 47430426660864, 47430426832895, ++ERASE, 47430426660864, 47430426832895, ++STORE, 47430426660864, 47430426664959, ++STORE, 47430426664960, 47430426832895, ++ERASE, 47430426664960, 47430426832895, ++STORE, 47430426664960, 47430426787839, ++STORE, 47430426787840, 47430426820607, ++STORE, 47430426820608, 47430426828799, ++STORE, 47430426828800, 47430426832895, ++STORE, 140722999115776, 140722999119871, ++STORE, 140722999103488, 140722999115775, ++STORE, 47430426832896, 47430426841087, ++STORE, 47430426841088, 47430426849279, ++STORE, 47430426849280, 47430427013119, ++ERASE, 47430426849280, 47430427013119, ++STORE, 47430426849280, 47430426861567, ++STORE, 47430426861568, 47430427013119, ++STORE, 47430426959872, 47430427013119, ++STORE, 47430426861568, 47430426959871, ++ERASE, 47430426861568, 47430426959871, ++STORE, 47430426861568, 47430426959871, ++STORE, 47430427004928, 47430427013119, ++STORE, 47430426959872, 47430427004927, ++ERASE, 47430426959872, 47430427004927, ++STORE, 47430426959872, 47430427013119, ++ERASE, 47430426959872, 47430427013119, ++STORE, 47430426959872, 47430427004927, ++STORE, 47430427004928, 47430427013119, ++ERASE, 47430427004928, 47430427013119, ++STORE, 47430427004928, 47430427013119, ++STORE, 47430427013120, 47430430064639, ++STORE, 47430427557888, 47430430064639, ++STORE, 47430427013120, 47430427557887, ++ERASE, 47430427557888, 47430430064639, ++STORE, 47430427557888, 47430429843455, ++STORE, 47430429843456, 47430430064639, ++STORE, 47430429253632, 47430429843455, ++STORE, 47430427557888, 47430429253631, ++ERASE, 47430427557888, 47430429253631, ++STORE, 47430427557888, 47430429253631, ++STORE, 47430429839360, 47430429843455, ++STORE, 47430429253632, 47430429839359, ++ERASE, 47430429253632, 47430429839359, ++STORE, 47430429253632, 47430429839359, ++STORE, 47430430048256, 47430430064639, ++STORE, 47430429843456, 47430430048255, ++ERASE, 47430429843456, 47430430048255, ++STORE, 47430429843456, 47430430048255, ++ERASE, 47430430048256, 47430430064639, ++STORE, 47430430048256, 47430430064639, ++STORE, 47430430064640, 47430431903743, ++STORE, 47430430203904, 47430431903743, ++STORE, 47430430064640, 47430430203903, ++ERASE, 47430430203904, 47430431903743, ++STORE, 47430430203904, 47430431862783, ++STORE, 47430431862784, 47430431903743, ++STORE, 47430431547392, 47430431862783, ++STORE, 47430430203904, 47430431547391, ++ERASE, 47430430203904, 47430431547391, ++STORE, 47430430203904, 47430431547391, ++STORE, 47430431858688, 47430431862783, ++STORE, 47430431547392, 47430431858687, ++ERASE, 47430431547392, 47430431858687, ++STORE, 47430431547392, 47430431858687, ++STORE, 47430431887360, 47430431903743, ++STORE, 47430431862784, 47430431887359, ++ERASE, 47430431862784, 47430431887359, ++STORE, 47430431862784, 47430431887359, ++ERASE, 47430431887360, 47430431903743, ++STORE, 47430431887360, 47430431903743, ++STORE, 47430431903744, 47430432038911, ++ERASE, 47430431903744, 47430432038911, ++STORE, 47430431903744, 47430431928319, ++STORE, 47430431928320, 47430432038911, ++STORE, 47430431989760, 47430432038911, ++STORE, 47430431928320, 47430431989759, ++ERASE, 47430431928320, 47430431989759, ++STORE, 47430431928320, 47430431989759, ++STORE, 47430432014336, 47430432038911, ++STORE, 47430431989760, 47430432014335, ++ERASE, 47430431989760, 47430432014335, ++STORE, 47430431989760, 47430432038911, ++ERASE, 47430431989760, 47430432038911, ++STORE, 47430431989760, 47430432014335, ++STORE, 47430432014336, 47430432038911, ++STORE, 47430432022528, 47430432038911, ++STORE, 47430432014336, 47430432022527, ++ERASE, 47430432014336, 47430432022527, ++STORE, 47430432014336, 47430432022527, ++ERASE, 47430432022528, 47430432038911, ++STORE, 47430432022528, 47430432038911, ++STORE, 47430432038912, 47430432059391, ++ERASE, 47430432038912, 47430432059391, ++STORE, 47430432038912, 47430432043007, ++STORE, 47430432043008, 47430432059391, ++STORE, 47430432047104, 47430432059391, ++STORE, 47430432043008, 47430432047103, ++ERASE, 47430432043008, 47430432047103, ++STORE, 47430432043008, 47430432047103, ++STORE, 47430432051200, 47430432059391, ++STORE, 47430432047104, 47430432051199, ++ERASE, 47430432047104, 47430432051199, ++STORE, 47430432047104, 47430432059391, ++ERASE, 47430432047104, 47430432059391, ++STORE, 47430432047104, 47430432051199, ++STORE, 47430432051200, 47430432059391, ++ERASE, 47430432051200, 47430432059391, ++STORE, 47430432051200, 47430432059391, ++STORE, 47430432059392, 47430432067583, ++ERASE, 47430431862784, 47430431887359, ++STORE, 47430431862784, 47430431879167, ++STORE, 47430431879168, 47430431887359, ++ERASE, 47430432051200, 47430432059391, ++STORE, 47430432051200, 47430432055295, ++STORE, 47430432055296, 47430432059391, ++ERASE, 47430432014336, 47430432022527, ++STORE, 47430432014336, 47430432018431, ++STORE, 47430432018432, 47430432022527, ++ }; ++ unsigned long set7[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140729808330752, 140737488351231, ++ERASE, 140729808330752, 140737488351231, ++STORE, 140729808330752, 140729808334847, ++STORE, 94629632020480, 94629632192511, ++ERASE, 94629632020480, 94629632192511, ++STORE, 94629632020480, 94629632036863, ++STORE, 94629632036864, 94629632192511, ++ERASE, 94629632036864, 94629632192511, ++STORE, 94629632036864, 94629632139263, ++STORE, 94629632139264, 94629632180223, ++STORE, 94629632180224, 94629632192511, ++STORE, 47439981776896, 47439981948927, ++ERASE, 47439981776896, 47439981948927, ++STORE, 47439981776896, 47439981780991, ++STORE, 47439981780992, 47439981948927, ++ERASE, 47439981780992, 47439981948927, ++STORE, 47439981780992, 47439981903871, ++STORE, 47439981903872, 47439981936639, ++STORE, 47439981936640, 47439981944831, ++STORE, 47439981944832, 47439981948927, ++STORE, 140729808474112, 140729808478207, ++STORE, 140729808461824, 140729808474111, ++STORE, 47439981948928, 47439981957119, ++STORE, 47439981957120, 47439981965311, ++STORE, 47439981965312, 47439982129151, ++ERASE, 47439981965312, 47439982129151, ++STORE, 47439981965312, 47439981977599, ++STORE, 47439981977600, 47439982129151, ++STORE, 47439982075904, 47439982129151, ++STORE, 47439981977600, 47439982075903, ++ERASE, 47439981977600, 47439982075903, ++STORE, 47439981977600, 47439982075903, ++STORE, 47439982120960, 47439982129151, ++STORE, 47439982075904, 47439982120959, ++ERASE, 47439982075904, 47439982120959, ++STORE, 47439982075904, 47439982129151, ++ERASE, 47439982075904, 47439982129151, ++STORE, 47439982075904, 47439982120959, ++STORE, 47439982120960, 47439982129151, ++ERASE, 47439982120960, 47439982129151, ++STORE, 47439982120960, 47439982129151, ++STORE, 47439982129152, 47439985180671, ++STORE, 47439982673920, 47439985180671, ++STORE, 47439982129152, 47439982673919, ++ERASE, 47439982673920, 47439985180671, ++STORE, 47439982673920, 47439984959487, ++STORE, 47439984959488, 47439985180671, ++STORE, 47439984369664, 47439984959487, ++STORE, 47439982673920, 47439984369663, ++ERASE, 47439982673920, 47439984369663, ++STORE, 47439982673920, 47439984369663, ++STORE, 47439984955392, 47439984959487, ++STORE, 47439984369664, 47439984955391, ++ERASE, 47439984369664, 47439984955391, ++STORE, 47439984369664, 47439984955391, ++STORE, 47439985164288, 47439985180671, ++STORE, 47439984959488, 47439985164287, ++ERASE, 47439984959488, 47439985164287, ++STORE, 47439984959488, 47439985164287, ++ERASE, 47439985164288, 47439985180671, ++STORE, 47439985164288, 47439985180671, ++STORE, 47439985180672, 47439987019775, ++STORE, 47439985319936, 47439987019775, ++STORE, 47439985180672, 47439985319935, ++ERASE, 47439985319936, 47439987019775, ++STORE, 47439985319936, 47439986978815, ++STORE, 47439986978816, 47439987019775, ++STORE, 47439986663424, 47439986978815, ++STORE, 47439985319936, 47439986663423, ++ERASE, 47439985319936, 47439986663423, ++STORE, 47439985319936, 47439986663423, ++STORE, 47439986974720, 47439986978815, ++STORE, 47439986663424, 47439986974719, ++ERASE, 47439986663424, 47439986974719, ++STORE, 47439986663424, 47439986974719, ++STORE, 47439987003392, 47439987019775, ++STORE, 47439986978816, 47439987003391, ++ERASE, 47439986978816, 47439987003391, ++STORE, 47439986978816, 47439987003391, ++ERASE, 47439987003392, 47439987019775, ++STORE, 47439987003392, 47439987019775, ++STORE, 47439987019776, 47439987154943, ++ERASE, 47439987019776, 47439987154943, ++STORE, 47439987019776, 47439987044351, ++STORE, 47439987044352, 47439987154943, ++STORE, 47439987105792, 47439987154943, ++STORE, 47439987044352, 47439987105791, ++ERASE, 47439987044352, 47439987105791, ++STORE, 47439987044352, 47439987105791, ++STORE, 47439987130368, 47439987154943, ++STORE, 47439987105792, 47439987130367, ++ERASE, 47439987105792, 47439987130367, ++STORE, 47439987105792, 47439987154943, ++ERASE, 47439987105792, 47439987154943, ++STORE, 47439987105792, 47439987130367, ++STORE, 47439987130368, 47439987154943, ++STORE, 47439987138560, 47439987154943, ++STORE, 47439987130368, 47439987138559, ++ERASE, 47439987130368, 47439987138559, ++STORE, 47439987130368, 47439987138559, ++ERASE, 47439987138560, 47439987154943, ++STORE, 47439987138560, 47439987154943, ++STORE, 47439987154944, 47439987175423, ++ERASE, 47439987154944, 47439987175423, ++STORE, 47439987154944, 47439987159039, ++STORE, 47439987159040, 47439987175423, ++STORE, 47439987163136, 47439987175423, ++STORE, 47439987159040, 47439987163135, ++ERASE, 47439987159040, 47439987163135, ++STORE, 47439987159040, 47439987163135, ++STORE, 47439987167232, 47439987175423, ++STORE, 47439987163136, 47439987167231, ++ERASE, 47439987163136, 47439987167231, ++STORE, 47439987163136, 47439987175423, ++ERASE, 47439987163136, 47439987175423, ++STORE, 47439987163136, 47439987167231, ++STORE, 47439987167232, 47439987175423, ++ERASE, 47439987167232, 47439987175423, ++STORE, 47439987167232, 47439987175423, ++STORE, 47439987175424, 47439987183615, ++ERASE, 47439986978816, 47439987003391, ++STORE, 47439986978816, 47439986995199, ++STORE, 47439986995200, 47439987003391, ++ERASE, 47439987167232, 47439987175423, ++STORE, 47439987167232, 47439987171327, ++STORE, 47439987171328, 47439987175423, ++ERASE, 47439987130368, 47439987138559, ++STORE, 47439987130368, 47439987134463, ++STORE, 47439987134464, 47439987138559, ++ }; ++ unsigned long set8[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722482974720, 140737488351231, ++ERASE, 140722482974720, 140737488351231, ++STORE, 140722482974720, 140722482978815, ++STORE, 94121505034240, 94121505206271, ++ERASE, 94121505034240, 94121505206271, ++STORE, 94121505034240, 94121505050623, ++STORE, 94121505050624, 94121505206271, ++ERASE, 94121505050624, 94121505206271, ++STORE, 94121505050624, 94121505153023, ++STORE, 94121505153024, 94121505193983, ++STORE, 94121505193984, 94121505206271, ++STORE, 47708483284992, 47708483457023, ++ERASE, 47708483284992, 47708483457023, ++STORE, 47708483284992, 47708483289087, ++STORE, 47708483289088, 47708483457023, ++ERASE, 47708483289088, 47708483457023, ++STORE, 47708483289088, 47708483411967, ++STORE, 47708483411968, 47708483444735, ++STORE, 47708483444736, 47708483452927, ++STORE, 47708483452928, 47708483457023, ++STORE, 140722483142656, 140722483146751, ++STORE, 140722483130368, 140722483142655, ++STORE, 47708483457024, 47708483465215, ++STORE, 47708483465216, 47708483473407, ++STORE, 47708483473408, 47708483637247, ++ERASE, 47708483473408, 47708483637247, ++STORE, 47708483473408, 47708483485695, ++STORE, 47708483485696, 47708483637247, ++STORE, 47708483584000, 47708483637247, ++STORE, 47708483485696, 47708483583999, ++ERASE, 47708483485696, 47708483583999, ++STORE, 47708483485696, 47708483583999, ++STORE, 47708483629056, 47708483637247, ++STORE, 47708483584000, 47708483629055, ++ERASE, 47708483584000, 47708483629055, ++STORE, 47708483584000, 47708483637247, ++ERASE, 47708483584000, 47708483637247, ++STORE, 47708483584000, 47708483629055, ++STORE, 47708483629056, 47708483637247, ++ERASE, 47708483629056, 47708483637247, ++STORE, 47708483629056, 47708483637247, ++STORE, 47708483637248, 47708486688767, ++STORE, 47708484182016, 47708486688767, ++STORE, 47708483637248, 47708484182015, ++ERASE, 47708484182016, 47708486688767, ++STORE, 47708484182016, 47708486467583, ++STORE, 47708486467584, 47708486688767, ++STORE, 47708485877760, 47708486467583, ++STORE, 47708484182016, 47708485877759, ++ERASE, 47708484182016, 47708485877759, ++STORE, 47708484182016, 47708485877759, ++STORE, 47708486463488, 47708486467583, ++STORE, 47708485877760, 47708486463487, ++ERASE, 47708485877760, 47708486463487, ++STORE, 47708485877760, 47708486463487, ++STORE, 47708486672384, 47708486688767, ++STORE, 47708486467584, 47708486672383, ++ERASE, 47708486467584, 47708486672383, ++STORE, 47708486467584, 47708486672383, ++ERASE, 47708486672384, 47708486688767, ++STORE, 47708486672384, 47708486688767, ++STORE, 47708486688768, 47708488527871, ++STORE, 47708486828032, 47708488527871, ++STORE, 47708486688768, 47708486828031, ++ERASE, 47708486828032, 47708488527871, ++STORE, 47708486828032, 47708488486911, ++STORE, 47708488486912, 47708488527871, ++STORE, 47708488171520, 47708488486911, ++STORE, 47708486828032, 47708488171519, ++ERASE, 47708486828032, 47708488171519, ++STORE, 47708486828032, 47708488171519, ++STORE, 47708488482816, 47708488486911, ++STORE, 47708488171520, 47708488482815, ++ERASE, 47708488171520, 47708488482815, ++STORE, 47708488171520, 47708488482815, ++STORE, 47708488511488, 47708488527871, ++STORE, 47708488486912, 47708488511487, ++ERASE, 47708488486912, 47708488511487, ++STORE, 47708488486912, 47708488511487, ++ERASE, 47708488511488, 47708488527871, ++STORE, 47708488511488, 47708488527871, ++STORE, 47708488527872, 47708488663039, ++ERASE, 47708488527872, 47708488663039, ++STORE, 47708488527872, 47708488552447, ++STORE, 47708488552448, 47708488663039, ++STORE, 47708488613888, 47708488663039, ++STORE, 47708488552448, 47708488613887, ++ERASE, 47708488552448, 47708488613887, ++STORE, 47708488552448, 47708488613887, ++STORE, 47708488638464, 47708488663039, ++STORE, 47708488613888, 47708488638463, ++ERASE, 47708488613888, 47708488638463, ++STORE, 47708488613888, 47708488663039, ++ERASE, 47708488613888, 47708488663039, ++STORE, 47708488613888, 47708488638463, ++STORE, 47708488638464, 47708488663039, ++STORE, 47708488646656, 47708488663039, ++STORE, 47708488638464, 47708488646655, ++ERASE, 47708488638464, 47708488646655, ++STORE, 47708488638464, 47708488646655, ++ERASE, 47708488646656, 47708488663039, ++STORE, 47708488646656, 47708488663039, ++STORE, 47708488663040, 47708488683519, ++ERASE, 47708488663040, 47708488683519, ++STORE, 47708488663040, 47708488667135, ++STORE, 47708488667136, 47708488683519, ++STORE, 47708488671232, 47708488683519, ++STORE, 47708488667136, 47708488671231, ++ERASE, 47708488667136, 47708488671231, ++STORE, 47708488667136, 47708488671231, ++STORE, 47708488675328, 47708488683519, ++STORE, 47708488671232, 47708488675327, ++ERASE, 47708488671232, 47708488675327, ++STORE, 47708488671232, 47708488683519, ++ERASE, 47708488671232, 47708488683519, ++STORE, 47708488671232, 47708488675327, ++STORE, 47708488675328, 47708488683519, ++ERASE, 47708488675328, 47708488683519, ++STORE, 47708488675328, 47708488683519, ++STORE, 47708488683520, 47708488691711, ++ERASE, 47708488486912, 47708488511487, ++STORE, 47708488486912, 47708488503295, ++STORE, 47708488503296, 47708488511487, ++ERASE, 47708488675328, 47708488683519, ++STORE, 47708488675328, 47708488679423, ++STORE, 47708488679424, 47708488683519, ++ERASE, 47708488638464, 47708488646655, ++STORE, 47708488638464, 47708488642559, ++STORE, 47708488642560, 47708488646655, ++ }; ++ ++ unsigned long set9[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140736427839488, 140737488351231, ++ERASE, 140736427839488, 140736427839488, ++STORE, 140736427839488, 140736427843583, ++STORE, 94071213395968, 94071213567999, ++ERASE, 94071213395968, 94071213395968, ++STORE, 94071213395968, 94071213412351, ++STORE, 94071213412352, 94071213567999, ++ERASE, 94071213412352, 94071213412352, ++STORE, 94071213412352, 94071213514751, ++STORE, 94071213514752, 94071213555711, ++STORE, 94071213555712, 94071213567999, ++STORE, 139968410644480, 139968410816511, ++ERASE, 139968410644480, 139968410644480, ++STORE, 139968410644480, 139968410648575, ++STORE, 139968410648576, 139968410816511, ++ERASE, 139968410648576, 139968410648576, ++STORE, 139968410648576, 139968410771455, ++STORE, 139968410771456, 139968410804223, ++STORE, 139968410804224, 139968410812415, ++STORE, 139968410812416, 139968410816511, ++STORE, 140736429277184, 140736429281279, ++STORE, 140736429264896, 140736429277183, ++STORE, 47664384352256, 47664384360447, ++STORE, 47664384360448, 47664384368639, ++STORE, 47664384368640, 47664384532479, ++ERASE, 47664384368640, 47664384368640, ++STORE, 47664384368640, 47664384380927, ++STORE, 47664384380928, 47664384532479, ++STORE, 47664384479232, 47664384532479, ++STORE, 47664384380928, 47664384479231, ++ERASE, 47664384380928, 47664384380928, ++STORE, 47664384380928, 47664384479231, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384479232, 47664384524287, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384532479, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384524287, ++STORE, 47664384524288, 47664384532479, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384532480, 47664387583999, ++STORE, 47664385077248, 47664387583999, ++STORE, 47664384532480, 47664385077247, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664387362815, ++STORE, 47664387362816, 47664387583999, ++STORE, 47664386772992, 47664387362815, ++STORE, 47664385077248, 47664386772991, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664386772991, ++STORE, 47664387358720, 47664387362815, ++STORE, 47664386772992, 47664387358719, ++ERASE, 47664386772992, 47664386772992, ++STORE, 47664386772992, 47664387358719, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387567616, 47664387567616, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387584000, 47664389423103, ++STORE, 47664387723264, 47664389423103, ++STORE, 47664387584000, 47664387723263, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389382143, ++STORE, 47664389382144, 47664389423103, ++STORE, 47664389066752, 47664389382143, ++STORE, 47664387723264, 47664389066751, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389066751, ++STORE, 47664389378048, 47664389382143, ++STORE, 47664389066752, 47664389378047, ++ERASE, 47664389066752, 47664389066752, ++STORE, 47664389066752, 47664389378047, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389406720, 47664389406720, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389423104, 47664389558271, ++ERASE, 47664389423104, 47664389423104, ++STORE, 47664389423104, 47664389447679, ++STORE, 47664389447680, 47664389558271, ++STORE, 47664389509120, 47664389558271, ++STORE, 47664389447680, 47664389509119, ++ERASE, 47664389447680, 47664389447680, ++STORE, 47664389447680, 47664389509119, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389509120, 47664389533695, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389558271, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389533695, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389541888, 47664389541888, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389558272, 47664389578751, ++ERASE, 47664389558272, 47664389558272, ++STORE, 47664389558272, 47664389562367, ++STORE, 47664389562368, 47664389578751, ++STORE, 47664389566464, 47664389578751, ++STORE, 47664389562368, 47664389566463, ++ERASE, 47664389562368, 47664389562368, ++STORE, 47664389562368, 47664389566463, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389566464, 47664389570559, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389578751, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389570559, ++STORE, 47664389570560, 47664389578751, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389578752, 47664389586943, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389398527, ++STORE, 47664389398528, 47664389406719, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389574655, ++STORE, 47664389574656, 47664389578751, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389537791, ++STORE, 47664389537792, 47664389541887, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387559423, ++STORE, 47664387559424, 47664387567615, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384528383, ++STORE, 47664384528384, 47664384532479, ++ERASE, 94071213555712, 94071213555712, ++STORE, 94071213555712, 94071213563903, ++STORE, 94071213563904, 94071213567999, ++ERASE, 139968410804224, 139968410804224, ++STORE, 139968410804224, 139968410808319, ++STORE, 139968410808320, 139968410812415, ++ERASE, 47664384352256, 47664384352256, ++STORE, 94071244402688, 94071244537855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728271503360, 140737488351231, ++ERASE, 140728271503360, 140728271503360, ++STORE, 140728271503360, 140728271507455, ++STORE, 94410361982976, 94410362155007, ++ERASE, 94410361982976, 94410361982976, ++STORE, 94410361982976, 94410361999359, ++STORE, 94410361999360, 94410362155007, ++ERASE, 94410361999360, 94410361999360, ++STORE, 94410361999360, 94410362101759, ++STORE, 94410362101760, 94410362142719, ++STORE, 94410362142720, 94410362155007, ++STORE, 140351953997824, 140351954169855, ++ERASE, 140351953997824, 140351953997824, ++STORE, 140351953997824, 140351954001919, ++STORE, 140351954001920, 140351954169855, ++ERASE, 140351954001920, 140351954001920, ++STORE, 140351954001920, 140351954124799, ++STORE, 140351954124800, 140351954157567, ++STORE, 140351954157568, 140351954165759, ++STORE, 140351954165760, 140351954169855, ++STORE, 140728272429056, 140728272433151, ++STORE, 140728272416768, 140728272429055, ++STORE, 47280840998912, 47280841007103, ++STORE, 47280841007104, 47280841015295, ++STORE, 47280841015296, 47280841179135, ++ERASE, 47280841015296, 47280841015296, ++STORE, 47280841015296, 47280841027583, ++STORE, 47280841027584, 47280841179135, ++STORE, 47280841125888, 47280841179135, ++STORE, 47280841027584, 47280841125887, ++ERASE, 47280841027584, 47280841027584, ++STORE, 47280841027584, 47280841125887, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841125888, 47280841170943, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841179135, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841170943, ++STORE, 47280841170944, 47280841179135, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841179136, 47280844230655, ++STORE, 47280841723904, 47280844230655, ++STORE, 47280841179136, 47280841723903, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280844009471, ++STORE, 47280844009472, 47280844230655, ++STORE, 47280843419648, 47280844009471, ++STORE, 47280841723904, 47280843419647, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280843419647, ++STORE, 47280844005376, 47280844009471, ++STORE, 47280843419648, 47280844005375, ++ERASE, 47280843419648, 47280843419648, ++STORE, 47280843419648, 47280844005375, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844214272, 47280844214272, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844230656, 47280846069759, ++STORE, 47280844369920, 47280846069759, ++STORE, 47280844230656, 47280844369919, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280846028799, ++STORE, 47280846028800, 47280846069759, ++STORE, 47280845713408, 47280846028799, ++STORE, 47280844369920, 47280845713407, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280845713407, ++STORE, 47280846024704, 47280846028799, ++STORE, 47280845713408, 47280846024703, ++ERASE, 47280845713408, 47280845713408, ++STORE, 47280845713408, 47280846024703, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846053376, 47280846053376, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846069760, 47280846204927, ++ERASE, 47280846069760, 47280846069760, ++STORE, 47280846069760, 47280846094335, ++STORE, 47280846094336, 47280846204927, ++STORE, 47280846155776, 47280846204927, ++STORE, 47280846094336, 47280846155775, ++ERASE, 47280846094336, 47280846094336, ++STORE, 47280846094336, 47280846155775, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846155776, 47280846180351, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846204927, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846180351, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846188544, 47280846188544, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846204928, 47280846225407, ++ERASE, 47280846204928, 47280846204928, ++STORE, 47280846204928, 47280846209023, ++STORE, 47280846209024, 47280846225407, ++STORE, 47280846213120, 47280846225407, ++STORE, 47280846209024, 47280846213119, ++ERASE, 47280846209024, 47280846209024, ++STORE, 47280846209024, 47280846213119, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846213120, 47280846217215, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846225407, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846217215, ++STORE, 47280846217216, 47280846225407, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846225408, 47280846233599, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846045183, ++STORE, 47280846045184, 47280846053375, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846221311, ++STORE, 47280846221312, 47280846225407, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846184447, ++STORE, 47280846184448, 47280846188543, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844206079, ++STORE, 47280844206080, 47280844214271, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841175039, ++STORE, 47280841175040, 47280841179135, ++ERASE, 94410362142720, 94410362142720, ++STORE, 94410362142720, 94410362150911, ++STORE, 94410362150912, 94410362155007, ++ERASE, 140351954157568, 140351954157568, ++STORE, 140351954157568, 140351954161663, ++STORE, 140351954161664, 140351954165759, ++ERASE, 47280840998912, 47280840998912, ++STORE, 94410379456512, 94410379591679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732946362368, 140737488351231, ++ERASE, 140732946362368, 140732946362368, ++STORE, 140732946362368, 140732946366463, ++STORE, 94352937934848, 94352938106879, ++ERASE, 94352937934848, 94352937934848, ++STORE, 94352937934848, 94352937951231, ++STORE, 94352937951232, 94352938106879, ++ERASE, 94352937951232, 94352937951232, ++STORE, 94352937951232, 94352938053631, ++STORE, 94352938053632, 94352938094591, ++STORE, 94352938094592, 94352938106879, ++STORE, 140595518742528, 140595518914559, ++ERASE, 140595518742528, 140595518742528, ++STORE, 140595518742528, 140595518746623, ++STORE, 140595518746624, 140595518914559, ++ERASE, 140595518746624, 140595518746624, ++STORE, 140595518746624, 140595518869503, ++STORE, 140595518869504, 140595518902271, ++STORE, 140595518902272, 140595518910463, ++STORE, 140595518910464, 140595518914559, ++STORE, 140732947468288, 140732947472383, ++STORE, 140732947456000, 140732947468287, ++STORE, 47037276254208, 47037276262399, ++STORE, 47037276262400, 47037276270591, ++STORE, 47037276270592, 47037276434431, ++ERASE, 47037276270592, 47037276270592, ++STORE, 47037276270592, 47037276282879, ++STORE, 47037276282880, 47037276434431, ++STORE, 47037276381184, 47037276434431, ++STORE, 47037276282880, 47037276381183, ++ERASE, 47037276282880, 47037276282880, ++STORE, 47037276282880, 47037276381183, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276381184, 47037276426239, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276434431, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276426239, ++STORE, 47037276426240, 47037276434431, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276434432, 47037279485951, ++STORE, 47037276979200, 47037279485951, ++STORE, 47037276434432, 47037276979199, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037279264767, ++STORE, 47037279264768, 47037279485951, ++STORE, 47037278674944, 47037279264767, ++STORE, 47037276979200, 47037278674943, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037278674943, ++STORE, 47037279260672, 47037279264767, ++STORE, 47037278674944, 47037279260671, ++ERASE, 47037278674944, 47037278674944, ++STORE, 47037278674944, 47037279260671, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279469568, 47037279469568, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279485952, 47037281325055, ++STORE, 47037279625216, 47037281325055, ++STORE, 47037279485952, 47037279625215, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037281284095, ++STORE, 47037281284096, 47037281325055, ++STORE, 47037280968704, 47037281284095, ++STORE, 47037279625216, 47037280968703, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037280968703, ++STORE, 47037281280000, 47037281284095, ++STORE, 47037280968704, 47037281279999, ++ERASE, 47037280968704, 47037280968704, ++STORE, 47037280968704, 47037281279999, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281308672, 47037281308672, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281325056, 47037281460223, ++ERASE, 47037281325056, 47037281325056, ++STORE, 47037281325056, 47037281349631, ++STORE, 47037281349632, 47037281460223, ++STORE, 47037281411072, 47037281460223, ++STORE, 47037281349632, 47037281411071, ++ERASE, 47037281349632, 47037281349632, ++STORE, 47037281349632, 47037281411071, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281411072, 47037281435647, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281460223, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281435647, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281443840, 47037281443840, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281460224, 47037281480703, ++ERASE, 47037281460224, 47037281460224, ++STORE, 47037281460224, 47037281464319, ++STORE, 47037281464320, 47037281480703, ++STORE, 47037281468416, 47037281480703, ++STORE, 47037281464320, 47037281468415, ++ERASE, 47037281464320, 47037281464320, ++STORE, 47037281464320, 47037281468415, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281468416, 47037281472511, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281480703, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281472511, ++STORE, 47037281472512, 47037281480703, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281480704, 47037281488895, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281300479, ++STORE, 47037281300480, 47037281308671, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281476607, ++STORE, 47037281476608, 47037281480703, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281439743, ++STORE, 47037281439744, 47037281443839, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279461375, ++STORE, 47037279461376, 47037279469567, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276430335, ++STORE, 47037276430336, 47037276434431, ++ERASE, 94352938094592, 94352938094592, ++STORE, 94352938094592, 94352938102783, ++STORE, 94352938102784, 94352938106879, ++ERASE, 140595518902272, 140595518902272, ++STORE, 140595518902272, 140595518906367, ++STORE, 140595518906368, 140595518910463, ++ERASE, 47037276254208, 47037276254208, ++STORE, 94352938438656, 94352938573823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733506027520, 140737488351231, ++ERASE, 140733506027520, 140733506027520, ++STORE, 140733506027520, 140733506031615, ++STORE, 94150123073536, 94150123245567, ++ERASE, 94150123073536, 94150123073536, ++STORE, 94150123073536, 94150123089919, ++STORE, 94150123089920, 94150123245567, ++ERASE, 94150123089920, 94150123089920, ++STORE, 94150123089920, 94150123192319, ++STORE, 94150123192320, 94150123233279, ++STORE, 94150123233280, 94150123245567, ++STORE, 140081290375168, 140081290547199, ++ERASE, 140081290375168, 140081290375168, ++STORE, 140081290375168, 140081290379263, ++STORE, 140081290379264, 140081290547199, ++ERASE, 140081290379264, 140081290379264, ++STORE, 140081290379264, 140081290502143, ++STORE, 140081290502144, 140081290534911, ++STORE, 140081290534912, 140081290543103, ++STORE, 140081290543104, 140081290547199, ++STORE, 140733506707456, 140733506711551, ++STORE, 140733506695168, 140733506707455, ++STORE, 47551504621568, 47551504629759, ++STORE, 47551504629760, 47551504637951, ++STORE, 47551504637952, 47551504801791, ++ERASE, 47551504637952, 47551504637952, ++STORE, 47551504637952, 47551504650239, ++STORE, 47551504650240, 47551504801791, ++STORE, 47551504748544, 47551504801791, ++STORE, 47551504650240, 47551504748543, ++ERASE, 47551504650240, 47551504650240, ++STORE, 47551504650240, 47551504748543, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504748544, 47551504793599, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504801791, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504793599, ++STORE, 47551504793600, 47551504801791, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504801792, 47551507853311, ++STORE, 47551505346560, 47551507853311, ++STORE, 47551504801792, 47551505346559, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507632127, ++STORE, 47551507632128, 47551507853311, ++STORE, 47551507042304, 47551507632127, ++STORE, 47551505346560, 47551507042303, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507042303, ++STORE, 47551507628032, 47551507632127, ++STORE, 47551507042304, 47551507628031, ++ERASE, 47551507042304, 47551507042304, ++STORE, 47551507042304, 47551507628031, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507836928, 47551507836928, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507853312, 47551509692415, ++STORE, 47551507992576, 47551509692415, ++STORE, 47551507853312, 47551507992575, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509651455, ++STORE, 47551509651456, 47551509692415, ++STORE, 47551509336064, 47551509651455, ++STORE, 47551507992576, 47551509336063, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509336063, ++STORE, 47551509647360, 47551509651455, ++STORE, 47551509336064, 47551509647359, ++ERASE, 47551509336064, 47551509336064, ++STORE, 47551509336064, 47551509647359, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509676032, 47551509676032, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509692416, 47551509827583, ++ERASE, 47551509692416, 47551509692416, ++STORE, 47551509692416, 47551509716991, ++STORE, 47551509716992, 47551509827583, ++STORE, 47551509778432, 47551509827583, ++STORE, 47551509716992, 47551509778431, ++ERASE, 47551509716992, 47551509716992, ++STORE, 47551509716992, 47551509778431, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509778432, 47551509803007, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509827583, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509803007, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509811200, 47551509811200, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509827584, 47551509848063, ++ERASE, 47551509827584, 47551509827584, ++STORE, 47551509827584, 47551509831679, ++STORE, 47551509831680, 47551509848063, ++STORE, 47551509835776, 47551509848063, ++STORE, 47551509831680, 47551509835775, ++ERASE, 47551509831680, 47551509831680, ++STORE, 47551509831680, 47551509835775, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509835776, 47551509839871, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509848063, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509839871, ++STORE, 47551509839872, 47551509848063, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509848064, 47551509856255, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509667839, ++STORE, 47551509667840, 47551509676031, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509843967, ++STORE, 47551509843968, 47551509848063, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509807103, ++STORE, 47551509807104, 47551509811199, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507828735, ++STORE, 47551507828736, 47551507836927, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504797695, ++STORE, 47551504797696, 47551504801791, ++ERASE, 94150123233280, 94150123233280, ++STORE, 94150123233280, 94150123241471, ++STORE, 94150123241472, 94150123245567, ++ERASE, 140081290534912, 140081290534912, ++STORE, 140081290534912, 140081290539007, ++STORE, 140081290539008, 140081290543103, ++ERASE, 47551504621568, 47551504621568, ++STORE, 94150148112384, 94150148247551, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734389334016, 140737488351231, ++ERASE, 140734389334016, 140734389334016, ++STORE, 140734389334016, 140734389338111, ++STORE, 94844636606464, 94844636778495, ++ERASE, 94844636606464, 94844636606464, ++STORE, 94844636606464, 94844636622847, ++STORE, 94844636622848, 94844636778495, ++ERASE, 94844636622848, 94844636622848, ++STORE, 94844636622848, 94844636725247, ++STORE, 94844636725248, 94844636766207, ++STORE, 94844636766208, 94844636778495, ++STORE, 139922765217792, 139922765389823, ++ERASE, 139922765217792, 139922765217792, ++STORE, 139922765217792, 139922765221887, ++STORE, 139922765221888, 139922765389823, ++ERASE, 139922765221888, 139922765221888, ++STORE, 139922765221888, 139922765344767, ++STORE, 139922765344768, 139922765377535, ++STORE, 139922765377536, 139922765385727, ++STORE, 139922765385728, 139922765389823, ++STORE, 140734389678080, 140734389682175, ++STORE, 140734389665792, 140734389678079, ++STORE, 47710029778944, 47710029787135, ++STORE, 47710029787136, 47710029795327, ++STORE, 47710029795328, 47710029959167, ++ERASE, 47710029795328, 47710029795328, ++STORE, 47710029795328, 47710029807615, ++STORE, 47710029807616, 47710029959167, ++STORE, 47710029905920, 47710029959167, ++STORE, 47710029807616, 47710029905919, ++ERASE, 47710029807616, 47710029807616, ++STORE, 47710029807616, 47710029905919, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029905920, 47710029950975, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029959167, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029950975, ++STORE, 47710029950976, 47710029959167, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029959168, 47710033010687, ++STORE, 47710030503936, 47710033010687, ++STORE, 47710029959168, 47710030503935, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032789503, ++STORE, 47710032789504, 47710033010687, ++STORE, 47710032199680, 47710032789503, ++STORE, 47710030503936, 47710032199679, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032199679, ++STORE, 47710032785408, 47710032789503, ++STORE, 47710032199680, 47710032785407, ++ERASE, 47710032199680, 47710032199680, ++STORE, 47710032199680, 47710032785407, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032994304, 47710032994304, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710033010688, 47710034849791, ++STORE, 47710033149952, 47710034849791, ++STORE, 47710033010688, 47710033149951, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034808831, ++STORE, 47710034808832, 47710034849791, ++STORE, 47710034493440, 47710034808831, ++STORE, 47710033149952, 47710034493439, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034493439, ++STORE, 47710034804736, 47710034808831, ++STORE, 47710034493440, 47710034804735, ++ERASE, 47710034493440, 47710034493440, ++STORE, 47710034493440, 47710034804735, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034833408, 47710034833408, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034849792, 47710034984959, ++ERASE, 47710034849792, 47710034849792, ++STORE, 47710034849792, 47710034874367, ++STORE, 47710034874368, 47710034984959, ++STORE, 47710034935808, 47710034984959, ++STORE, 47710034874368, 47710034935807, ++ERASE, 47710034874368, 47710034874368, ++STORE, 47710034874368, 47710034935807, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034935808, 47710034960383, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034984959, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034960383, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034968576, 47710034968576, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034984960, 47710035005439, ++ERASE, 47710034984960, 47710034984960, ++STORE, 47710034984960, 47710034989055, ++STORE, 47710034989056, 47710035005439, ++STORE, 47710034993152, 47710035005439, ++STORE, 47710034989056, 47710034993151, ++ERASE, 47710034989056, 47710034989056, ++STORE, 47710034989056, 47710034993151, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710034993152, 47710034997247, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710035005439, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710034997247, ++STORE, 47710034997248, 47710035005439, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710035005440, 47710035013631, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034825215, ++STORE, 47710034825216, 47710034833407, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035001343, ++STORE, 47710035001344, 47710035005439, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034964479, ++STORE, 47710034964480, 47710034968575, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032986111, ++STORE, 47710032986112, 47710032994303, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029955071, ++STORE, 47710029955072, 47710029959167, ++ERASE, 94844636766208, 94844636766208, ++STORE, 94844636766208, 94844636774399, ++STORE, 94844636774400, 94844636778495, ++ERASE, 139922765377536, 139922765377536, ++STORE, 139922765377536, 139922765381631, ++STORE, 139922765381632, 139922765385727, ++ERASE, 47710029778944, 47710029778944, ++STORE, 94844641775616, 94844641910783, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732213886976, 140737488351231, ++ERASE, 140732213886976, 140732213886976, ++STORE, 140732213886976, 140732213891071, ++STORE, 94240508887040, 94240509059071, ++ERASE, 94240508887040, 94240508887040, ++STORE, 94240508887040, 94240508903423, ++STORE, 94240508903424, 94240509059071, ++ERASE, 94240508903424, 94240508903424, ++STORE, 94240508903424, 94240509005823, ++STORE, 94240509005824, 94240509046783, ++STORE, 94240509046784, 94240509059071, ++STORE, 140275106516992, 140275106689023, ++ERASE, 140275106516992, 140275106516992, ++STORE, 140275106516992, 140275106521087, ++STORE, 140275106521088, 140275106689023, ++ERASE, 140275106521088, 140275106521088, ++STORE, 140275106521088, 140275106643967, ++STORE, 140275106643968, 140275106676735, ++STORE, 140275106676736, 140275106684927, ++STORE, 140275106684928, 140275106689023, ++STORE, 140732213977088, 140732213981183, ++STORE, 140732213964800, 140732213977087, ++STORE, 47357688479744, 47357688487935, ++STORE, 47357688487936, 47357688496127, ++STORE, 47357688496128, 47357688659967, ++ERASE, 47357688496128, 47357688496128, ++STORE, 47357688496128, 47357688508415, ++STORE, 47357688508416, 47357688659967, ++STORE, 47357688606720, 47357688659967, ++STORE, 47357688508416, 47357688606719, ++ERASE, 47357688508416, 47357688508416, ++STORE, 47357688508416, 47357688606719, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688606720, 47357688651775, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688659967, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688651775, ++STORE, 47357688651776, 47357688659967, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688659968, 47357691711487, ++STORE, 47357689204736, 47357691711487, ++STORE, 47357688659968, 47357689204735, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357691490303, ++STORE, 47357691490304, 47357691711487, ++STORE, 47357690900480, 47357691490303, ++STORE, 47357689204736, 47357690900479, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357690900479, ++STORE, 47357691486208, 47357691490303, ++STORE, 47357690900480, 47357691486207, ++ERASE, 47357690900480, 47357690900480, ++STORE, 47357690900480, 47357691486207, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691695104, 47357691695104, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691711488, 47357693550591, ++STORE, 47357691850752, 47357693550591, ++STORE, 47357691711488, 47357691850751, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693509631, ++STORE, 47357693509632, 47357693550591, ++STORE, 47357693194240, 47357693509631, ++STORE, 47357691850752, 47357693194239, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693194239, ++STORE, 47357693505536, 47357693509631, ++STORE, 47357693194240, 47357693505535, ++ERASE, 47357693194240, 47357693194240, ++STORE, 47357693194240, 47357693505535, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693534208, 47357693534208, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693550592, 47357693685759, ++ERASE, 47357693550592, 47357693550592, ++STORE, 47357693550592, 47357693575167, ++STORE, 47357693575168, 47357693685759, ++STORE, 47357693636608, 47357693685759, ++STORE, 47357693575168, 47357693636607, ++ERASE, 47357693575168, 47357693575168, ++STORE, 47357693575168, 47357693636607, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693636608, 47357693661183, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693685759, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693661183, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693669376, 47357693669376, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693685760, 47357693706239, ++ERASE, 47357693685760, 47357693685760, ++STORE, 47357693685760, 47357693689855, ++STORE, 47357693689856, 47357693706239, ++STORE, 47357693693952, 47357693706239, ++STORE, 47357693689856, 47357693693951, ++ERASE, 47357693689856, 47357693689856, ++STORE, 47357693689856, 47357693693951, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693693952, 47357693698047, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693706239, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693698047, ++STORE, 47357693698048, 47357693706239, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693706240, 47357693714431, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693526015, ++STORE, 47357693526016, 47357693534207, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693702143, ++STORE, 47357693702144, 47357693706239, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693665279, ++STORE, 47357693665280, 47357693669375, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691686911, ++STORE, 47357691686912, 47357691695103, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688655871, ++STORE, 47357688655872, 47357688659967, ++ERASE, 94240509046784, 94240509046784, ++STORE, 94240509046784, 94240509054975, ++STORE, 94240509054976, 94240509059071, ++ERASE, 140275106676736, 140275106676736, ++STORE, 140275106676736, 140275106680831, ++STORE, 140275106680832, 140275106684927, ++ERASE, 47357688479744, 47357688479744, ++STORE, 94240518361088, 94240518496255, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732688277504, 140737488351231, ++ERASE, 140732688277504, 140732688277504, ++STORE, 140732688277504, 140732688281599, ++STORE, 94629171351552, 94629172064255, ++ERASE, 94629171351552, 94629171351552, ++STORE, 94629171351552, 94629171400703, ++STORE, 94629171400704, 94629172064255, ++ERASE, 94629171400704, 94629171400704, ++STORE, 94629171400704, 94629171945471, ++STORE, 94629171945472, 94629172043775, ++STORE, 94629172043776, 94629172064255, ++STORE, 139770707644416, 139770707816447, ++ERASE, 139770707644416, 139770707644416, ++STORE, 139770707644416, 139770707648511, ++STORE, 139770707648512, 139770707816447, ++ERASE, 139770707648512, 139770707648512, ++STORE, 139770707648512, 139770707771391, ++STORE, 139770707771392, 139770707804159, ++STORE, 139770707804160, 139770707812351, ++STORE, 139770707812352, 139770707816447, ++STORE, 140732689121280, 140732689125375, ++STORE, 140732689108992, 140732689121279, ++STORE, 47862087352320, 47862087360511, ++STORE, 47862087360512, 47862087368703, ++STORE, 47862087368704, 47862087475199, ++STORE, 47862087385088, 47862087475199, ++STORE, 47862087368704, 47862087385087, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087458815, ++STORE, 47862087458816, 47862087475199, ++STORE, 47862087438336, 47862087458815, ++STORE, 47862087385088, 47862087438335, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087438335, ++STORE, 47862087454720, 47862087458815, ++STORE, 47862087438336, 47862087454719, ++ERASE, 47862087438336, 47862087438336, ++STORE, 47862087438336, 47862087454719, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087467008, 47862087467008, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087475200, 47862089314303, ++STORE, 47862087614464, 47862089314303, ++STORE, 47862087475200, 47862087614463, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862089273343, ++STORE, 47862089273344, 47862089314303, ++STORE, 47862088957952, 47862089273343, ++STORE, 47862087614464, 47862088957951, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862088957951, ++STORE, 47862089269248, 47862089273343, ++STORE, 47862088957952, 47862089269247, ++ERASE, 47862088957952, 47862088957952, ++STORE, 47862088957952, 47862089269247, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089297920, 47862089297920, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089297920, 47862089326591, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089289727, ++STORE, 47862089289728, 47862089297919, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087462911, ++STORE, 47862087462912, 47862087467007, ++ERASE, 94629172043776, 94629172043776, ++STORE, 94629172043776, 94629172060159, ++STORE, 94629172060160, 94629172064255, ++ERASE, 139770707804160, 139770707804160, ++STORE, 139770707804160, 139770707808255, ++STORE, 139770707808256, 139770707812351, ++ERASE, 47862087352320, 47862087352320, ++STORE, 94629197533184, 94629197668351, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727540711424, 140737488351231, ++ERASE, 140727540711424, 140727540711424, ++STORE, 140727540711424, 140727540715519, ++STORE, 94299865313280, 94299866025983, ++ERASE, 94299865313280, 94299865313280, ++STORE, 94299865313280, 94299865362431, ++STORE, 94299865362432, 94299866025983, ++ERASE, 94299865362432, 94299865362432, ++STORE, 94299865362432, 94299865907199, ++STORE, 94299865907200, 94299866005503, ++STORE, 94299866005504, 94299866025983, ++STORE, 140680268763136, 140680268935167, ++ERASE, 140680268763136, 140680268763136, ++STORE, 140680268763136, 140680268767231, ++STORE, 140680268767232, 140680268935167, ++ERASE, 140680268767232, 140680268767232, ++STORE, 140680268767232, 140680268890111, ++STORE, 140680268890112, 140680268922879, ++STORE, 140680268922880, 140680268931071, ++STORE, 140680268931072, 140680268935167, ++STORE, 140727541424128, 140727541428223, ++STORE, 140727541411840, 140727541424127, ++STORE, 46952526233600, 46952526241791, ++STORE, 46952526241792, 46952526249983, ++STORE, 46952526249984, 46952526356479, ++STORE, 46952526266368, 46952526356479, ++STORE, 46952526249984, 46952526266367, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526340095, ++STORE, 46952526340096, 46952526356479, ++STORE, 46952526319616, 46952526340095, ++STORE, 46952526266368, 46952526319615, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526319615, ++STORE, 46952526336000, 46952526340095, ++STORE, 46952526319616, 46952526335999, ++ERASE, 46952526319616, 46952526319616, ++STORE, 46952526319616, 46952526335999, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526348288, 46952526348288, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526356480, 46952528195583, ++STORE, 46952526495744, 46952528195583, ++STORE, 46952526356480, 46952526495743, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952528154623, ++STORE, 46952528154624, 46952528195583, ++STORE, 46952527839232, 46952528154623, ++STORE, 46952526495744, 46952527839231, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952527839231, ++STORE, 46952528150528, 46952528154623, ++STORE, 46952527839232, 46952528150527, ++ERASE, 46952527839232, 46952527839232, ++STORE, 46952527839232, 46952528150527, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528179200, 46952528179200, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528179200, 46952528207871, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528171007, ++STORE, 46952528171008, 46952528179199, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526344191, ++STORE, 46952526344192, 46952526348287, ++ERASE, 94299866005504, 94299866005504, ++STORE, 94299866005504, 94299866021887, ++STORE, 94299866021888, 94299866025983, ++ERASE, 140680268922880, 140680268922880, ++STORE, 140680268922880, 140680268926975, ++STORE, 140680268926976, 140680268931071, ++ERASE, 46952526233600, 46952526233600, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722874793984, 140737488351231, ++ERASE, 140722874793984, 140722874793984, ++STORE, 140722874793984, 140722874798079, ++STORE, 94448916213760, 94448916926463, ++ERASE, 94448916213760, 94448916213760, ++STORE, 94448916213760, 94448916262911, ++STORE, 94448916262912, 94448916926463, ++ERASE, 94448916262912, 94448916262912, ++STORE, 94448916262912, 94448916807679, ++STORE, 94448916807680, 94448916905983, ++STORE, 94448916905984, 94448916926463, ++STORE, 140389117046784, 140389117218815, ++ERASE, 140389117046784, 140389117046784, ++STORE, 140389117046784, 140389117050879, ++STORE, 140389117050880, 140389117218815, ++ERASE, 140389117050880, 140389117050880, ++STORE, 140389117050880, 140389117173759, ++STORE, 140389117173760, 140389117206527, ++STORE, 140389117206528, 140389117214719, ++STORE, 140389117214720, 140389117218815, ++STORE, 140722875297792, 140722875301887, ++STORE, 140722875285504, 140722875297791, ++STORE, 47243677949952, 47243677958143, ++STORE, 47243677958144, 47243677966335, ++STORE, 47243677966336, 47243678072831, ++STORE, 47243677982720, 47243678072831, ++STORE, 47243677966336, 47243677982719, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678056447, ++STORE, 47243678056448, 47243678072831, ++STORE, 47243678035968, 47243678056447, ++STORE, 47243677982720, 47243678035967, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678035967, ++STORE, 47243678052352, 47243678056447, ++STORE, 47243678035968, 47243678052351, ++ERASE, 47243678035968, 47243678035968, ++STORE, 47243678035968, 47243678052351, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678064640, 47243678064640, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678072832, 47243679911935, ++STORE, 47243678212096, 47243679911935, ++STORE, 47243678072832, 47243678212095, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679870975, ++STORE, 47243679870976, 47243679911935, ++STORE, 47243679555584, 47243679870975, ++STORE, 47243678212096, 47243679555583, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679555583, ++STORE, 47243679866880, 47243679870975, ++STORE, 47243679555584, 47243679866879, ++ERASE, 47243679555584, 47243679555584, ++STORE, 47243679555584, 47243679866879, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679895552, 47243679895552, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679895552, 47243679924223, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679887359, ++STORE, 47243679887360, 47243679895551, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678060543, ++STORE, 47243678060544, 47243678064639, ++ERASE, 94448916905984, 94448916905984, ++STORE, 94448916905984, 94448916922367, ++STORE, 94448916922368, 94448916926463, ++ERASE, 140389117206528, 140389117206528, ++STORE, 140389117206528, 140389117210623, ++STORE, 140389117210624, 140389117214719, ++ERASE, 47243677949952, 47243677949952, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733068505088, 140737488351231, ++ERASE, 140733068505088, 140733068505088, ++STORE, 140733068505088, 140733068509183, ++STORE, 94207145750528, 94207146463231, ++ERASE, 94207145750528, 94207145750528, ++STORE, 94207145750528, 94207145799679, ++STORE, 94207145799680, 94207146463231, ++ERASE, 94207145799680, 94207145799680, ++STORE, 94207145799680, 94207146344447, ++STORE, 94207146344448, 94207146442751, ++STORE, 94207146442752, 94207146463231, ++STORE, 140684504911872, 140684505083903, ++ERASE, 140684504911872, 140684504911872, ++STORE, 140684504911872, 140684504915967, ++STORE, 140684504915968, 140684505083903, ++ERASE, 140684504915968, 140684504915968, ++STORE, 140684504915968, 140684505038847, ++STORE, 140684505038848, 140684505071615, ++STORE, 140684505071616, 140684505079807, ++STORE, 140684505079808, 140684505083903, ++STORE, 140733068607488, 140733068611583, ++STORE, 140733068595200, 140733068607487, ++STORE, 46948290084864, 46948290093055, ++STORE, 46948290093056, 46948290101247, ++STORE, 46948290101248, 46948290207743, ++STORE, 46948290117632, 46948290207743, ++STORE, 46948290101248, 46948290117631, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290191359, ++STORE, 46948290191360, 46948290207743, ++STORE, 46948290170880, 46948290191359, ++STORE, 46948290117632, 46948290170879, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290170879, ++STORE, 46948290187264, 46948290191359, ++STORE, 46948290170880, 46948290187263, ++ERASE, 46948290170880, 46948290170880, ++STORE, 46948290170880, 46948290187263, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290199552, 46948290199552, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290207744, 46948292046847, ++STORE, 46948290347008, 46948292046847, ++STORE, 46948290207744, 46948290347007, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948292005887, ++STORE, 46948292005888, 46948292046847, ++STORE, 46948291690496, 46948292005887, ++STORE, 46948290347008, 46948291690495, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948291690495, ++STORE, 46948292001792, 46948292005887, ++STORE, 46948291690496, 46948292001791, ++ERASE, 46948291690496, 46948291690496, ++STORE, 46948291690496, 46948292001791, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292030464, 46948292030464, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292030464, 46948292059135, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292022271, ++STORE, 46948292022272, 46948292030463, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290195455, ++STORE, 46948290195456, 46948290199551, ++ERASE, 94207146442752, 94207146442752, ++STORE, 94207146442752, 94207146459135, ++STORE, 94207146459136, 94207146463231, ++ERASE, 140684505071616, 140684505071616, ++STORE, 140684505071616, 140684505075711, ++STORE, 140684505075712, 140684505079807, ++ERASE, 46948290084864, 46948290084864, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726367158272, 140737488351231, ++ERASE, 140726367158272, 140726367158272, ++STORE, 140726367158272, 140726367162367, ++STORE, 94436124106752, 94436124819455, ++ERASE, 94436124106752, 94436124106752, ++STORE, 94436124106752, 94436124155903, ++STORE, 94436124155904, 94436124819455, ++ERASE, 94436124155904, 94436124155904, ++STORE, 94436124155904, 94436124700671, ++STORE, 94436124700672, 94436124798975, ++STORE, 94436124798976, 94436124819455, ++STORE, 140049025044480, 140049025216511, ++ERASE, 140049025044480, 140049025044480, ++STORE, 140049025044480, 140049025048575, ++STORE, 140049025048576, 140049025216511, ++ERASE, 140049025048576, 140049025048576, ++STORE, 140049025048576, 140049025171455, ++STORE, 140049025171456, 140049025204223, ++STORE, 140049025204224, 140049025212415, ++STORE, 140049025212416, 140049025216511, ++STORE, 140726367256576, 140726367260671, ++STORE, 140726367244288, 140726367256575, ++STORE, 47583769952256, 47583769960447, ++STORE, 47583769960448, 47583769968639, ++STORE, 47583769968640, 47583770075135, ++STORE, 47583769985024, 47583770075135, ++STORE, 47583769968640, 47583769985023, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770058751, ++STORE, 47583770058752, 47583770075135, ++STORE, 47583770038272, 47583770058751, ++STORE, 47583769985024, 47583770038271, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770038271, ++STORE, 47583770054656, 47583770058751, ++STORE, 47583770038272, 47583770054655, ++ERASE, 47583770038272, 47583770038272, ++STORE, 47583770038272, 47583770054655, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770066944, 47583770066944, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770075136, 47583771914239, ++STORE, 47583770214400, 47583771914239, ++STORE, 47583770075136, 47583770214399, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771873279, ++STORE, 47583771873280, 47583771914239, ++STORE, 47583771557888, 47583771873279, ++STORE, 47583770214400, 47583771557887, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771557887, ++STORE, 47583771869184, 47583771873279, ++STORE, 47583771557888, 47583771869183, ++ERASE, 47583771557888, 47583771557888, ++STORE, 47583771557888, 47583771869183, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771897856, 47583771897856, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771897856, 47583771926527, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771889663, ++STORE, 47583771889664, 47583771897855, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770062847, ++STORE, 47583770062848, 47583770066943, ++ERASE, 94436124798976, 94436124798976, ++STORE, 94436124798976, 94436124815359, ++STORE, 94436124815360, 94436124819455, ++ERASE, 140049025204224, 140049025204224, ++STORE, 140049025204224, 140049025208319, ++STORE, 140049025208320, 140049025212415, ++ERASE, 47583769952256, 47583769952256, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727116099584, 140737488351231, ++ERASE, 140727116099584, 140727116099584, ++STORE, 140727116099584, 140727116103679, ++STORE, 94166319734784, 94166320447487, ++ERASE, 94166319734784, 94166319734784, ++STORE, 94166319734784, 94166319783935, ++STORE, 94166319783936, 94166320447487, ++ERASE, 94166319783936, 94166319783936, ++STORE, 94166319783936, 94166320328703, ++STORE, 94166320328704, 94166320427007, ++STORE, 94166320427008, 94166320447487, ++STORE, 139976559542272, 139976559714303, ++ERASE, 139976559542272, 139976559542272, ++STORE, 139976559542272, 139976559546367, ++STORE, 139976559546368, 139976559714303, ++ERASE, 139976559546368, 139976559546368, ++STORE, 139976559546368, 139976559669247, ++STORE, 139976559669248, 139976559702015, ++STORE, 139976559702016, 139976559710207, ++STORE, 139976559710208, 139976559714303, ++STORE, 140727116222464, 140727116226559, ++STORE, 140727116210176, 140727116222463, ++STORE, 47656235454464, 47656235462655, ++STORE, 47656235462656, 47656235470847, ++STORE, 47656235470848, 47656235577343, ++STORE, 47656235487232, 47656235577343, ++STORE, 47656235470848, 47656235487231, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235560959, ++STORE, 47656235560960, 47656235577343, ++STORE, 47656235540480, 47656235560959, ++STORE, 47656235487232, 47656235540479, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235540479, ++STORE, 47656235556864, 47656235560959, ++STORE, 47656235540480, 47656235556863, ++ERASE, 47656235540480, 47656235540480, ++STORE, 47656235540480, 47656235556863, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235569152, 47656235569152, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235577344, 47656237416447, ++STORE, 47656235716608, 47656237416447, ++STORE, 47656235577344, 47656235716607, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237375487, ++STORE, 47656237375488, 47656237416447, ++STORE, 47656237060096, 47656237375487, ++STORE, 47656235716608, 47656237060095, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237060095, ++STORE, 47656237371392, 47656237375487, ++STORE, 47656237060096, 47656237371391, ++ERASE, 47656237060096, 47656237060096, ++STORE, 47656237060096, 47656237371391, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237400064, 47656237400064, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237400064, 47656237428735, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237391871, ++STORE, 47656237391872, 47656237400063, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235565055, ++STORE, 47656235565056, 47656235569151, ++ERASE, 94166320427008, 94166320427008, ++STORE, 94166320427008, 94166320443391, ++STORE, 94166320443392, 94166320447487, ++ERASE, 139976559702016, 139976559702016, ++STORE, 139976559702016, 139976559706111, ++STORE, 139976559706112, 139976559710207, ++ERASE, 47656235454464, 47656235454464, ++STORE, 94166332153856, 94166332289023, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726412816384, 140737488351231, ++ERASE, 140726412816384, 140726412816384, ++STORE, 140726412816384, 140726412820479, ++STORE, 94094884507648, 94094885220351, ++ERASE, 94094884507648, 94094884507648, ++STORE, 94094884507648, 94094884556799, ++STORE, 94094884556800, 94094885220351, ++ERASE, 94094884556800, 94094884556800, ++STORE, 94094884556800, 94094885101567, ++STORE, 94094885101568, 94094885199871, ++STORE, 94094885199872, 94094885220351, ++STORE, 139773773938688, 139773774110719, ++ERASE, 139773773938688, 139773773938688, ++STORE, 139773773938688, 139773773942783, ++STORE, 139773773942784, 139773774110719, ++ERASE, 139773773942784, 139773773942784, ++STORE, 139773773942784, 139773774065663, ++STORE, 139773774065664, 139773774098431, ++STORE, 139773774098432, 139773774106623, ++STORE, 139773774106624, 139773774110719, ++STORE, 140726412963840, 140726412967935, ++STORE, 140726412951552, 140726412963839, ++STORE, 47859021058048, 47859021066239, ++STORE, 47859021066240, 47859021074431, ++STORE, 47859021074432, 47859021180927, ++STORE, 47859021090816, 47859021180927, ++STORE, 47859021074432, 47859021090815, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021164543, ++STORE, 47859021164544, 47859021180927, ++STORE, 47859021144064, 47859021164543, ++STORE, 47859021090816, 47859021144063, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021144063, ++STORE, 47859021160448, 47859021164543, ++STORE, 47859021144064, 47859021160447, ++ERASE, 47859021144064, 47859021144064, ++STORE, 47859021144064, 47859021160447, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021172736, 47859021172736, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021180928, 47859023020031, ++STORE, 47859021320192, 47859023020031, ++STORE, 47859021180928, 47859021320191, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022979071, ++STORE, 47859022979072, 47859023020031, ++STORE, 47859022663680, 47859022979071, ++STORE, 47859021320192, 47859022663679, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022663679, ++STORE, 47859022974976, 47859022979071, ++STORE, 47859022663680, 47859022974975, ++ERASE, 47859022663680, 47859022663680, ++STORE, 47859022663680, 47859022974975, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859023003648, 47859023003648, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859023003648, 47859023032319, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859022995455, ++STORE, 47859022995456, 47859023003647, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021168639, ++STORE, 47859021168640, 47859021172735, ++ERASE, 94094885199872, 94094885199872, ++STORE, 94094885199872, 94094885216255, ++STORE, 94094885216256, 94094885220351, ++ERASE, 139773774098432, 139773774098432, ++STORE, 139773774098432, 139773774102527, ++STORE, 139773774102528, 139773774106623, ++ERASE, 47859021058048, 47859021058048, ++STORE, 94094901108736, 94094901243903, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736567963648, 140737488351231, ++ERASE, 140736567963648, 140736567963648, ++STORE, 140736567963648, 140736567967743, ++STORE, 94924425748480, 94924426461183, ++ERASE, 94924425748480, 94924425748480, ++STORE, 94924425748480, 94924425797631, ++STORE, 94924425797632, 94924426461183, ++ERASE, 94924425797632, 94924425797632, ++STORE, 94924425797632, 94924426342399, ++STORE, 94924426342400, 94924426440703, ++STORE, 94924426440704, 94924426461183, ++STORE, 140042126319616, 140042126491647, ++ERASE, 140042126319616, 140042126319616, ++STORE, 140042126319616, 140042126323711, ++STORE, 140042126323712, 140042126491647, ++ERASE, 140042126323712, 140042126323712, ++STORE, 140042126323712, 140042126446591, ++STORE, 140042126446592, 140042126479359, ++STORE, 140042126479360, 140042126487551, ++STORE, 140042126487552, 140042126491647, ++STORE, 140736568672256, 140736568676351, ++STORE, 140736568659968, 140736568672255, ++STORE, 47590668677120, 47590668685311, ++STORE, 47590668685312, 47590668693503, ++STORE, 47590668693504, 47590668799999, ++STORE, 47590668709888, 47590668799999, ++STORE, 47590668693504, 47590668709887, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668783615, ++STORE, 47590668783616, 47590668799999, ++STORE, 47590668763136, 47590668783615, ++STORE, 47590668709888, 47590668763135, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668763135, ++STORE, 47590668779520, 47590668783615, ++STORE, 47590668763136, 47590668779519, ++ERASE, 47590668763136, 47590668763136, ++STORE, 47590668763136, 47590668779519, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668791808, 47590668791808, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668800000, 47590670639103, ++STORE, 47590668939264, 47590670639103, ++STORE, 47590668800000, 47590668939263, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670598143, ++STORE, 47590670598144, 47590670639103, ++STORE, 47590670282752, 47590670598143, ++STORE, 47590668939264, 47590670282751, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670282751, ++STORE, 47590670594048, 47590670598143, ++STORE, 47590670282752, 47590670594047, ++ERASE, 47590670282752, 47590670282752, ++STORE, 47590670282752, 47590670594047, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670622720, 47590670622720, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670622720, 47590670651391, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670614527, ++STORE, 47590670614528, 47590670622719, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668787711, ++STORE, 47590668787712, 47590668791807, ++ERASE, 94924426440704, 94924426440704, ++STORE, 94924426440704, 94924426457087, ++STORE, 94924426457088, 94924426461183, ++ERASE, 140042126479360, 140042126479360, ++STORE, 140042126479360, 140042126483455, ++STORE, 140042126483456, 140042126487551, ++ERASE, 47590668677120, 47590668677120, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733281439744, 140737488351231, ++ERASE, 140733281439744, 140733281439744, ++STORE, 140733281439744, 140733281443839, ++STORE, 94490667069440, 94490667782143, ++ERASE, 94490667069440, 94490667069440, ++STORE, 94490667069440, 94490667118591, ++STORE, 94490667118592, 94490667782143, ++ERASE, 94490667118592, 94490667118592, ++STORE, 94490667118592, 94490667663359, ++STORE, 94490667663360, 94490667761663, ++STORE, 94490667761664, 94490667782143, ++STORE, 139878215118848, 139878215290879, ++ERASE, 139878215118848, 139878215118848, ++STORE, 139878215118848, 139878215122943, ++STORE, 139878215122944, 139878215290879, ++ERASE, 139878215122944, 139878215122944, ++STORE, 139878215122944, 139878215245823, ++STORE, 139878215245824, 139878215278591, ++STORE, 139878215278592, 139878215286783, ++STORE, 139878215286784, 139878215290879, ++STORE, 140733281464320, 140733281468415, ++STORE, 140733281452032, 140733281464319, ++STORE, 47754579877888, 47754579886079, ++STORE, 47754579886080, 47754579894271, ++STORE, 47754579894272, 47754580000767, ++STORE, 47754579910656, 47754580000767, ++STORE, 47754579894272, 47754579910655, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579984383, ++STORE, 47754579984384, 47754580000767, ++STORE, 47754579963904, 47754579984383, ++STORE, 47754579910656, 47754579963903, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579963903, ++STORE, 47754579980288, 47754579984383, ++STORE, 47754579963904, 47754579980287, ++ERASE, 47754579963904, 47754579963904, ++STORE, 47754579963904, 47754579980287, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579992576, 47754579992576, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754580000768, 47754581839871, ++STORE, 47754580140032, 47754581839871, ++STORE, 47754580000768, 47754580140031, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581798911, ++STORE, 47754581798912, 47754581839871, ++STORE, 47754581483520, 47754581798911, ++STORE, 47754580140032, 47754581483519, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581483519, ++STORE, 47754581794816, 47754581798911, ++STORE, 47754581483520, 47754581794815, ++ERASE, 47754581483520, 47754581483520, ++STORE, 47754581483520, 47754581794815, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581823488, 47754581823488, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581823488, 47754581852159, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581815295, ++STORE, 47754581815296, 47754581823487, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579988479, ++STORE, 47754579988480, 47754579992575, ++ERASE, 94490667761664, 94490667761664, ++STORE, 94490667761664, 94490667778047, ++STORE, 94490667778048, 94490667782143, ++ERASE, 139878215278592, 139878215278592, ++STORE, 139878215278592, 139878215282687, ++STORE, 139878215282688, 139878215286783, ++ERASE, 47754579877888, 47754579877888, ++STORE, 94490669649920, 94490669785087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140735382188032, 140737488351231, ++ERASE, 140735382188032, 140735382188032, ++STORE, 140735382188032, 140735382192127, ++STORE, 94150181302272, 94150182014975, ++ERASE, 94150181302272, 94150181302272, ++STORE, 94150181302272, 94150181351423, ++STORE, 94150181351424, 94150182014975, ++ERASE, 94150181351424, 94150181351424, ++STORE, 94150181351424, 94150181896191, ++STORE, 94150181896192, 94150181994495, ++STORE, 94150181994496, 94150182014975, ++STORE, 139679752458240, 139679752630271, ++ERASE, 139679752458240, 139679752458240, ++STORE, 139679752458240, 139679752462335, ++STORE, 139679752462336, 139679752630271, ++ERASE, 139679752462336, 139679752462336, ++STORE, 139679752462336, 139679752585215, ++STORE, 139679752585216, 139679752617983, ++STORE, 139679752617984, 139679752626175, ++STORE, 139679752626176, 139679752630271, ++STORE, 140735382536192, 140735382540287, ++STORE, 140735382523904, 140735382536191, ++STORE, 47953042538496, 47953042546687, ++STORE, 47953042546688, 47953042554879, ++STORE, 47953042554880, 47953042661375, ++STORE, 47953042571264, 47953042661375, ++STORE, 47953042554880, 47953042571263, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042644991, ++STORE, 47953042644992, 47953042661375, ++STORE, 47953042624512, 47953042644991, ++STORE, 47953042571264, 47953042624511, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042624511, ++STORE, 47953042640896, 47953042644991, ++STORE, 47953042624512, 47953042640895, ++ERASE, 47953042624512, 47953042624512, ++STORE, 47953042624512, 47953042640895, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042653184, 47953042653184, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042661376, 47953044500479, ++STORE, 47953042800640, 47953044500479, ++STORE, 47953042661376, 47953042800639, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044459519, ++STORE, 47953044459520, 47953044500479, ++STORE, 47953044144128, 47953044459519, ++STORE, 47953042800640, 47953044144127, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044144127, ++STORE, 47953044455424, 47953044459519, ++STORE, 47953044144128, 47953044455423, ++ERASE, 47953044144128, 47953044144128, ++STORE, 47953044144128, 47953044455423, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044484096, 47953044484096, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044484096, 47953044512767, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044475903, ++STORE, 47953044475904, 47953044484095, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042649087, ++STORE, 47953042649088, 47953042653183, ++ERASE, 94150181994496, 94150181994496, ++STORE, 94150181994496, 94150182010879, ++STORE, 94150182010880, 94150182014975, ++ERASE, 139679752617984, 139679752617984, ++STORE, 139679752617984, 139679752622079, ++STORE, 139679752622080, 139679752626175, ++ERASE, 47953042538496, 47953042538496, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737044123648, 140737488351231, ++ERASE, 140737044123648, 140737044123648, ++STORE, 140737044123648, 140737044127743, ++STORE, 94425324294144, 94425325006847, ++ERASE, 94425324294144, 94425324294144, ++STORE, 94425324294144, 94425324343295, ++STORE, 94425324343296, 94425325006847, ++ERASE, 94425324343296, 94425324343296, ++STORE, 94425324343296, 94425324888063, ++STORE, 94425324888064, 94425324986367, ++STORE, 94425324986368, 94425325006847, ++STORE, 140382015016960, 140382015188991, ++ERASE, 140382015016960, 140382015016960, ++STORE, 140382015016960, 140382015021055, ++STORE, 140382015021056, 140382015188991, ++ERASE, 140382015021056, 140382015021056, ++STORE, 140382015021056, 140382015143935, ++STORE, 140382015143936, 140382015176703, ++STORE, 140382015176704, 140382015184895, ++STORE, 140382015184896, 140382015188991, ++STORE, 140737045585920, 140737045590015, ++STORE, 140737045573632, 140737045585919, ++STORE, 47250779979776, 47250779987967, ++STORE, 47250779987968, 47250779996159, ++STORE, 47250779996160, 47250780102655, ++STORE, 47250780012544, 47250780102655, ++STORE, 47250779996160, 47250780012543, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780086271, ++STORE, 47250780086272, 47250780102655, ++STORE, 47250780065792, 47250780086271, ++STORE, 47250780012544, 47250780065791, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780065791, ++STORE, 47250780082176, 47250780086271, ++STORE, 47250780065792, 47250780082175, ++ERASE, 47250780065792, 47250780065792, ++STORE, 47250780065792, 47250780082175, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780094464, 47250780094464, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780102656, 47250781941759, ++STORE, 47250780241920, 47250781941759, ++STORE, 47250780102656, 47250780241919, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781900799, ++STORE, 47250781900800, 47250781941759, ++STORE, 47250781585408, 47250781900799, ++STORE, 47250780241920, 47250781585407, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781585407, ++STORE, 47250781896704, 47250781900799, ++STORE, 47250781585408, 47250781896703, ++ERASE, 47250781585408, 47250781585408, ++STORE, 47250781585408, 47250781896703, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781925376, 47250781925376, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781925376, 47250781954047, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781917183, ++STORE, 47250781917184, 47250781925375, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780090367, ++STORE, 47250780090368, 47250780094463, ++ERASE, 94425324986368, 94425324986368, ++STORE, 94425324986368, 94425325002751, ++STORE, 94425325002752, 94425325006847, ++ERASE, 140382015176704, 140382015176704, ++STORE, 140382015176704, 140382015180799, ++STORE, 140382015180800, 140382015184895, ++ERASE, 47250779979776, 47250779979776, ++STORE, 94425351438336, 94425351573503, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736801144832, 140737488351231, ++ERASE, 140736801144832, 140736801144832, ++STORE, 140736801144832, 140736801148927, ++STORE, 94629429358592, 94629430071295, ++ERASE, 94629429358592, 94629429358592, ++STORE, 94629429358592, 94629429407743, ++STORE, 94629429407744, 94629430071295, ++ERASE, 94629429407744, 94629429407744, ++STORE, 94629429407744, 94629429952511, ++STORE, 94629429952512, 94629430050815, ++STORE, 94629430050816, 94629430071295, ++STORE, 139801685483520, 139801685655551, ++ERASE, 139801685483520, 139801685483520, ++STORE, 139801685483520, 139801685487615, ++STORE, 139801685487616, 139801685655551, ++ERASE, 139801685487616, 139801685487616, ++STORE, 139801685487616, 139801685610495, ++STORE, 139801685610496, 139801685643263, ++STORE, 139801685643264, 139801685651455, ++STORE, 139801685651456, 139801685655551, ++STORE, 140736801198080, 140736801202175, ++STORE, 140736801185792, 140736801198079, ++STORE, 47831109513216, 47831109521407, ++STORE, 47831109521408, 47831109529599, ++STORE, 47831109529600, 47831109636095, ++STORE, 47831109545984, 47831109636095, ++STORE, 47831109529600, 47831109545983, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109619711, ++STORE, 47831109619712, 47831109636095, ++STORE, 47831109599232, 47831109619711, ++STORE, 47831109545984, 47831109599231, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109599231, ++STORE, 47831109615616, 47831109619711, ++STORE, 47831109599232, 47831109615615, ++ERASE, 47831109599232, 47831109599232, ++STORE, 47831109599232, 47831109615615, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109627904, 47831109627904, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109636096, 47831111475199, ++STORE, 47831109775360, 47831111475199, ++STORE, 47831109636096, 47831109775359, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111434239, ++STORE, 47831111434240, 47831111475199, ++STORE, 47831111118848, 47831111434239, ++STORE, 47831109775360, 47831111118847, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111118847, ++STORE, 47831111430144, 47831111434239, ++STORE, 47831111118848, 47831111430143, ++ERASE, 47831111118848, 47831111118848, ++STORE, 47831111118848, 47831111430143, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111458816, 47831111458816, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111458816, 47831111487487, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111450623, ++STORE, 47831111450624, 47831111458815, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109623807, ++STORE, 47831109623808, 47831109627903, ++ERASE, 94629430050816, 94629430050816, ++STORE, 94629430050816, 94629430067199, ++STORE, 94629430067200, 94629430071295, ++ERASE, 139801685643264, 139801685643264, ++STORE, 139801685643264, 139801685647359, ++STORE, 139801685647360, 139801685651455, ++ERASE, 47831109513216, 47831109513216, ++STORE, 140737488347136, 140737488351231, ++STORE, 140729419612160, 140737488351231, ++ERASE, 140729419612160, 140729419612160, ++STORE, 140729419612160, 140729419616255, ++STORE, 94443354148864, 94443354861567, ++ERASE, 94443354148864, 94443354148864, ++STORE, 94443354148864, 94443354198015, ++STORE, 94443354198016, 94443354861567, ++ERASE, 94443354198016, 94443354198016, ++STORE, 94443354198016, 94443354742783, ++STORE, 94443354742784, 94443354841087, ++STORE, 94443354841088, 94443354861567, ++STORE, 139741700038656, 139741700210687, ++ERASE, 139741700038656, 139741700038656, ++STORE, 139741700038656, 139741700042751, ++STORE, 139741700042752, 139741700210687, ++ERASE, 139741700042752, 139741700042752, ++STORE, 139741700042752, 139741700165631, ++STORE, 139741700165632, 139741700198399, ++STORE, 139741700198400, 139741700206591, ++STORE, 139741700206592, 139741700210687, ++STORE, 140729420574720, 140729420578815, ++STORE, 140729420562432, 140729420574719, ++STORE, 47891094958080, 47891094966271, ++STORE, 47891094966272, 47891094974463, ++STORE, 47891094974464, 47891095080959, ++STORE, 47891094990848, 47891095080959, ++STORE, 47891094974464, 47891094990847, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095064575, ++STORE, 47891095064576, 47891095080959, ++STORE, 47891095044096, 47891095064575, ++STORE, 47891094990848, 47891095044095, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095044095, ++STORE, 47891095060480, 47891095064575, ++STORE, 47891095044096, 47891095060479, ++ERASE, 47891095044096, 47891095044096, ++STORE, 47891095044096, 47891095060479, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095072768, 47891095072768, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095080960, 47891096920063, ++STORE, 47891095220224, 47891096920063, ++STORE, 47891095080960, 47891095220223, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096879103, ++STORE, 47891096879104, 47891096920063, ++STORE, 47891096563712, 47891096879103, ++STORE, 47891095220224, 47891096563711, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096563711, ++STORE, 47891096875008, 47891096879103, ++STORE, 47891096563712, 47891096875007, ++ERASE, 47891096563712, 47891096563712, ++STORE, 47891096563712, 47891096875007, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096903680, 47891096903680, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096903680, 47891096932351, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096895487, ++STORE, 47891096895488, 47891096903679, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095068671, ++STORE, 47891095068672, 47891095072767, ++ERASE, 94443354841088, 94443354841088, ++STORE, 94443354841088, 94443354857471, ++STORE, 94443354857472, 94443354861567, ++ERASE, 139741700198400, 139741700198400, ++STORE, 139741700198400, 139741700202495, ++STORE, 139741700202496, 139741700206591, ++ERASE, 47891094958080, 47891094958080, ++STORE, 94443360825344, 94443360960511, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722961661952, 140737488351231, ++ERASE, 140722961661952, 140722961661952, ++STORE, 140722961661952, 140722961666047, ++STORE, 94878388944896, 94878389657599, ++ERASE, 94878388944896, 94878388944896, ++STORE, 94878388944896, 94878388994047, ++STORE, 94878388994048, 94878389657599, ++ERASE, 94878388994048, 94878388994048, ++STORE, 94878388994048, 94878389538815, ++STORE, 94878389538816, 94878389637119, ++STORE, 94878389637120, 94878389657599, ++STORE, 140210690056192, 140210690228223, ++ERASE, 140210690056192, 140210690056192, ++STORE, 140210690056192, 140210690060287, ++STORE, 140210690060288, 140210690228223, ++ERASE, 140210690060288, 140210690060288, ++STORE, 140210690060288, 140210690183167, ++STORE, 140210690183168, 140210690215935, ++STORE, 140210690215936, 140210690224127, ++STORE, 140210690224128, 140210690228223, ++STORE, 140722963148800, 140722963152895, ++STORE, 140722963136512, 140722963148799, ++STORE, 47422104940544, 47422104948735, ++STORE, 47422104948736, 47422104956927, ++STORE, 47422104956928, 47422105063423, ++STORE, 47422104973312, 47422105063423, ++STORE, 47422104956928, 47422104973311, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105047039, ++STORE, 47422105047040, 47422105063423, ++STORE, 47422105026560, 47422105047039, ++STORE, 47422104973312, 47422105026559, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105026559, ++STORE, 47422105042944, 47422105047039, ++STORE, 47422105026560, 47422105042943, ++ERASE, 47422105026560, 47422105026560, ++STORE, 47422105026560, 47422105042943, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105055232, 47422105055232, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105063424, 47422106902527, ++STORE, 47422105202688, 47422106902527, ++STORE, 47422105063424, 47422105202687, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106861567, ++STORE, 47422106861568, 47422106902527, ++STORE, 47422106546176, 47422106861567, ++STORE, 47422105202688, 47422106546175, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106546175, ++STORE, 47422106857472, 47422106861567, ++STORE, 47422106546176, 47422106857471, ++ERASE, 47422106546176, 47422106546176, ++STORE, 47422106546176, 47422106857471, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106886144, 47422106886144, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106886144, 47422106914815, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106877951, ++STORE, 47422106877952, 47422106886143, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105051135, ++STORE, 47422105051136, 47422105055231, ++ERASE, 94878389637120, 94878389637120, ++STORE, 94878389637120, 94878389653503, ++STORE, 94878389653504, 94878389657599, ++ERASE, 140210690215936, 140210690215936, ++STORE, 140210690215936, 140210690220031, ++STORE, 140210690220032, 140210690224127, ++ERASE, 47422104940544, 47422104940544, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727690309632, 140737488351231, ++ERASE, 140727690309632, 140727690309632, ++STORE, 140727690309632, 140727690313727, ++STORE, 94121892208640, 94121892921343, ++ERASE, 94121892208640, 94121892208640, ++STORE, 94121892208640, 94121892257791, ++STORE, 94121892257792, 94121892921343, ++ERASE, 94121892257792, 94121892257792, ++STORE, 94121892257792, 94121892802559, ++STORE, 94121892802560, 94121892900863, ++STORE, 94121892900864, 94121892921343, ++STORE, 140662438326272, 140662438498303, ++ERASE, 140662438326272, 140662438326272, ++STORE, 140662438326272, 140662438330367, ++STORE, 140662438330368, 140662438498303, ++ERASE, 140662438330368, 140662438330368, ++STORE, 140662438330368, 140662438453247, ++STORE, 140662438453248, 140662438486015, ++STORE, 140662438486016, 140662438494207, ++STORE, 140662438494208, 140662438498303, ++STORE, 140727690379264, 140727690383359, ++STORE, 140727690366976, 140727690379263, ++STORE, 46970356670464, 46970356678655, ++STORE, 46970356678656, 46970356686847, ++STORE, 46970356686848, 46970356793343, ++STORE, 46970356703232, 46970356793343, ++STORE, 46970356686848, 46970356703231, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356776959, ++STORE, 46970356776960, 46970356793343, ++STORE, 46970356756480, 46970356776959, ++STORE, 46970356703232, 46970356756479, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356756479, ++STORE, 46970356772864, 46970356776959, ++STORE, 46970356756480, 46970356772863, ++ERASE, 46970356756480, 46970356756480, ++STORE, 46970356756480, 46970356772863, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356785152, 46970356785152, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356793344, 46970358632447, ++STORE, 46970356932608, 46970358632447, ++STORE, 46970356793344, 46970356932607, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358591487, ++STORE, 46970358591488, 46970358632447, ++STORE, 46970358276096, 46970358591487, ++STORE, 46970356932608, 46970358276095, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358276095, ++STORE, 46970358587392, 46970358591487, ++STORE, 46970358276096, 46970358587391, ++ERASE, 46970358276096, 46970358276096, ++STORE, 46970358276096, 46970358587391, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358616064, 46970358616064, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358616064, 46970358644735, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358607871, ++STORE, 46970358607872, 46970358616063, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356781055, ++STORE, 46970356781056, 46970356785151, ++ERASE, 94121892900864, 94121892900864, ++STORE, 94121892900864, 94121892917247, ++STORE, 94121892917248, 94121892921343, ++ERASE, 140662438486016, 140662438486016, ++STORE, 140662438486016, 140662438490111, ++STORE, 140662438490112, 140662438494207, ++ERASE, 46970356670464, 46970356670464, ++STORE, 94121898610688, 94121898745855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737189351424, 140737488351231, ++ERASE, 140737189351424, 140737189351424, ++STORE, 140737189351424, 140737189355519, ++STORE, 93847948832768, 93847949545471, ++ERASE, 93847948832768, 93847948832768, ++STORE, 93847948832768, 93847948881919, ++STORE, 93847948881920, 93847949545471, ++ERASE, 93847948881920, 93847948881920, ++STORE, 93847948881920, 93847949426687, ++STORE, 93847949426688, 93847949524991, ++STORE, 93847949524992, 93847949545471, ++STORE, 139698989985792, 139698990157823, ++ERASE, 139698989985792, 139698989985792, ++STORE, 139698989985792, 139698989989887, ++STORE, 139698989989888, 139698990157823, ++ERASE, 139698989989888, 139698989989888, ++STORE, 139698989989888, 139698990112767, ++STORE, 139698990112768, 139698990145535, ++STORE, 139698990145536, 139698990153727, ++STORE, 139698990153728, 139698990157823, ++STORE, 140737189744640, 140737189748735, ++STORE, 140737189732352, 140737189744639, ++STORE, 47933805010944, 47933805019135, ++STORE, 47933805019136, 47933805027327, ++STORE, 47933805027328, 47933805133823, ++STORE, 47933805043712, 47933805133823, ++STORE, 47933805027328, 47933805043711, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805117439, ++STORE, 47933805117440, 47933805133823, ++STORE, 47933805096960, 47933805117439, ++STORE, 47933805043712, 47933805096959, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805096959, ++STORE, 47933805113344, 47933805117439, ++STORE, 47933805096960, 47933805113343, ++ERASE, 47933805096960, 47933805096960, ++STORE, 47933805096960, 47933805113343, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805125632, 47933805125632, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805133824, 47933806972927, ++STORE, 47933805273088, 47933806972927, ++STORE, 47933805133824, 47933805273087, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806931967, ++STORE, 47933806931968, 47933806972927, ++STORE, 47933806616576, 47933806931967, ++STORE, 47933805273088, 47933806616575, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806616575, ++STORE, 47933806927872, 47933806931967, ++STORE, 47933806616576, 47933806927871, ++ERASE, 47933806616576, 47933806616576, ++STORE, 47933806616576, 47933806927871, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806956544, 47933806956544, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806956544, 47933806985215, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806948351, ++STORE, 47933806948352, 47933806956543, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805121535, ++STORE, 47933805121536, 47933805125631, ++ERASE, 93847949524992, 93847949524992, ++STORE, 93847949524992, 93847949541375, ++STORE, 93847949541376, 93847949545471, ++ERASE, 139698990145536, 139698990145536, ++STORE, 139698990145536, 139698990149631, ++STORE, 139698990149632, 139698990153727, ++ERASE, 47933805010944, 47933805010944, ++STORE, 140737488347136, 140737488351231, ++STORE, 140725553991680, 140737488351231, ++ERASE, 140725553991680, 140725553991680, ++STORE, 140725553991680, 140725553995775, ++STORE, 93980056248320, 93980056961023, ++ERASE, 93980056248320, 93980056248320, ++STORE, 93980056248320, 93980056297471, ++STORE, 93980056297472, 93980056961023, ++ERASE, 93980056297472, 93980056297472, ++STORE, 93980056297472, 93980056842239, ++STORE, 93980056842240, 93980056940543, ++STORE, 93980056940544, 93980056961023, ++STORE, 140146588971008, 140146589143039, ++ERASE, 140146588971008, 140146588971008, ++STORE, 140146588971008, 140146588975103, ++STORE, 140146588975104, 140146589143039, ++ERASE, 140146588975104, 140146588975104, ++STORE, 140146588975104, 140146589097983, ++STORE, 140146589097984, 140146589130751, ++STORE, 140146589130752, 140146589138943, ++STORE, 140146589138944, 140146589143039, ++STORE, 140725554860032, 140725554864127, ++STORE, 140725554847744, 140725554860031, ++STORE, 47486206025728, 47486206033919, ++STORE, 47486206033920, 47486206042111, ++STORE, 47486206042112, 47486206148607, ++STORE, 47486206058496, 47486206148607, ++STORE, 47486206042112, 47486206058495, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206132223, ++STORE, 47486206132224, 47486206148607, ++STORE, 47486206111744, 47486206132223, ++STORE, 47486206058496, 47486206111743, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206111743, ++STORE, 47486206128128, 47486206132223, ++STORE, 47486206111744, 47486206128127, ++ERASE, 47486206111744, 47486206111744, ++STORE, 47486206111744, 47486206128127, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206140416, 47486206140416, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206148608, 47486207987711, ++STORE, 47486206287872, 47486207987711, ++STORE, 47486206148608, 47486206287871, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207946751, ++STORE, 47486207946752, 47486207987711, ++STORE, 47486207631360, 47486207946751, ++STORE, 47486206287872, 47486207631359, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207631359, ++STORE, 47486207942656, 47486207946751, ++STORE, 47486207631360, 47486207942655, ++ERASE, 47486207631360, 47486207631360, ++STORE, 47486207631360, 47486207942655, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207971328, 47486207971328, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207971328, 47486207999999, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207963135, ++STORE, 47486207963136, 47486207971327, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206136319, ++STORE, 47486206136320, 47486206140415, ++ERASE, 93980056940544, 93980056940544, ++STORE, 93980056940544, 93980056956927, ++STORE, 93980056956928, 93980056961023, ++ERASE, 140146589130752, 140146589130752, ++STORE, 140146589130752, 140146589134847, ++STORE, 140146589134848, 140146589138943, ++ERASE, 47486206025728, 47486206025728, ++STORE, 93980070006784, 93980070141951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727334776832, 140737488351231, ++ERASE, 140727334776832, 140727334776832, ++STORE, 140727334776832, 140727334780927, ++STORE, 94049747247104, 94049747959807, ++ERASE, 94049747247104, 94049747247104, ++STORE, 94049747247104, 94049747296255, ++STORE, 94049747296256, 94049747959807, ++ERASE, 94049747296256, 94049747296256, ++STORE, 94049747296256, 94049747841023, ++STORE, 94049747841024, 94049747939327, ++STORE, 94049747939328, 94049747959807, ++STORE, 140227307216896, 140227307388927, ++ERASE, 140227307216896, 140227307216896, ++STORE, 140227307216896, 140227307220991, ++STORE, 140227307220992, 140227307388927, ++ERASE, 140227307220992, 140227307220992, ++STORE, 140227307220992, 140227307343871, ++STORE, 140227307343872, 140227307376639, ++STORE, 140227307376640, 140227307384831, ++STORE, 140227307384832, 140227307388927, ++STORE, 140727335337984, 140727335342079, ++STORE, 140727335325696, 140727335337983, ++STORE, 47405487779840, 47405487788031, ++STORE, 47405487788032, 47405487796223, ++STORE, 47405487796224, 47405487902719, ++STORE, 47405487812608, 47405487902719, ++STORE, 47405487796224, 47405487812607, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487886335, ++STORE, 47405487886336, 47405487902719, ++STORE, 47405487865856, 47405487886335, ++STORE, 47405487812608, 47405487865855, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487865855, ++STORE, 47405487882240, 47405487886335, ++STORE, 47405487865856, 47405487882239, ++ERASE, 47405487865856, 47405487865856, ++STORE, 47405487865856, 47405487882239, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487894528, 47405487894528, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487902720, 47405489741823, ++STORE, 47405488041984, 47405489741823, ++STORE, 47405487902720, 47405488041983, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489700863, ++STORE, 47405489700864, 47405489741823, ++STORE, 47405489385472, 47405489700863, ++STORE, 47405488041984, 47405489385471, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489385471, ++STORE, 47405489696768, 47405489700863, ++STORE, 47405489385472, 47405489696767, ++ERASE, 47405489385472, 47405489385472, ++STORE, 47405489385472, 47405489696767, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489725440, 47405489725440, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489725440, 47405489754111, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489717247, ++STORE, 47405489717248, 47405489725439, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487890431, ++STORE, 47405487890432, 47405487894527, ++ERASE, 94049747939328, 94049747939328, ++STORE, 94049747939328, 94049747955711, ++STORE, 94049747955712, 94049747959807, ++ERASE, 140227307376640, 140227307376640, ++STORE, 140227307376640, 140227307380735, ++STORE, 140227307380736, 140227307384831, ++ERASE, 47405487779840, 47405487779840, ++STORE, 94049758810112, 94049758945279, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727079718912, 140737488351231, ++ERASE, 140727079718912, 140727079718912, ++STORE, 140727079718912, 140727079723007, ++STORE, 94250996527104, 94250997239807, ++ERASE, 94250996527104, 94250996527104, ++STORE, 94250996527104, 94250996576255, ++STORE, 94250996576256, 94250997239807, ++ERASE, 94250996576256, 94250996576256, ++STORE, 94250996576256, 94250997121023, ++STORE, 94250997121024, 94250997219327, ++STORE, 94250997219328, 94250997239807, ++STORE, 140060022587392, 140060022759423, ++ERASE, 140060022587392, 140060022587392, ++STORE, 140060022587392, 140060022591487, ++STORE, 140060022591488, 140060022759423, ++ERASE, 140060022591488, 140060022591488, ++STORE, 140060022591488, 140060022714367, ++STORE, 140060022714368, 140060022747135, ++STORE, 140060022747136, 140060022755327, ++STORE, 140060022755328, 140060022759423, ++STORE, 140727079788544, 140727079792639, ++STORE, 140727079776256, 140727079788543, ++/* this next one caused issues when lowering the efficiency */ ++STORE, 47572772409344, 47572772417535, ++STORE, 47572772417536, 47572772425727, ++STORE, 47572772425728, 47572772532223, ++STORE, 47572772442112, 47572772532223, ++STORE, 47572772425728, 47572772442111, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772515839, ++STORE, 47572772515840, 47572772532223, ++STORE, 47572772495360, 47572772515839, ++STORE, 47572772442112, 47572772495359, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772495359, ++STORE, 47572772511744, 47572772515839, ++STORE, 47572772495360, 47572772511743, ++ERASE, 47572772495360, 47572772495360, ++STORE, 47572772495360, 47572772511743, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772524032, 47572772524032, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772532224, 47572774371327, ++STORE, 47572772671488, 47572774371327, ++STORE, 47572772532224, 47572772671487, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774330367, ++STORE, 47572774330368, 47572774371327, ++STORE, 47572774014976, 47572774330367, ++STORE, 47572772671488, 47572774014975, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774014975, ++STORE, 47572774326272, 47572774330367, ++STORE, 47572774014976, 47572774326271, ++ERASE, 47572774014976, 47572774014976, ++STORE, 47572774014976, 47572774326271, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774354944, 47572774354944, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774354944, 47572774383615, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774346751, ++STORE, 47572774346752, 47572774354943, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772519935, ++STORE, 47572772519936, 47572772524031, ++ERASE, 94250997219328, 94250997219328, ++STORE, 94250997219328, 94250997235711, ++STORE, 94250997235712, 94250997239807, ++ERASE, 140060022747136, 140060022747136, ++STORE, 140060022747136, 140060022751231, ++STORE, 140060022751232, 140060022755327, ++ERASE, 47572772409344, 47572772409344, ++STORE, 94251018305536, 94251018440703, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730012389376, 140737488351231, ++ERASE, 140730012389376, 140730012389376, ++STORE, 140730012389376, 140730012393471, ++STORE, 94382607675392, 94382607695871, ++ERASE, 94382607675392, 94382607675392, ++STORE, 94382607675392, 94382607679487, ++STORE, 94382607679488, 94382607695871, ++ERASE, 94382607679488, 94382607679488, ++STORE, 94382607679488, 94382607683583, ++STORE, 94382607683584, 94382607687679, ++STORE, 94382607687680, 94382607695871, ++STORE, 140252451454976, 140252451627007, ++ERASE, 140252451454976, 140252451454976, ++STORE, 140252451454976, 140252451459071, ++STORE, 140252451459072, 140252451627007, ++ERASE, 140252451459072, 140252451459072, ++STORE, 140252451459072, 140252451581951, ++STORE, 140252451581952, 140252451614719, ++STORE, 140252451614720, 140252451622911, ++STORE, 140252451622912, 140252451627007, ++STORE, 140730013548544, 140730013552639, ++STORE, 140730013536256, 140730013548543, ++STORE, 47380343541760, 47380343549951, ++STORE, 47380343549952, 47380343558143, ++STORE, 47380343558144, 47380345397247, ++STORE, 47380343697408, 47380345397247, ++STORE, 47380343558144, 47380343697407, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345356287, ++STORE, 47380345356288, 47380345397247, ++STORE, 47380345040896, 47380345356287, ++STORE, 47380343697408, 47380345040895, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345040895, ++STORE, 47380345352192, 47380345356287, ++STORE, 47380345040896, 47380345352191, ++ERASE, 47380345040896, 47380345040896, ++STORE, 47380345040896, 47380345352191, ++STORE, 47380345380864, 47380345397247, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345380864, 47380345380864, ++STORE, 47380345380864, 47380345397247, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345372671, ++STORE, 47380345372672, 47380345380863, ++ERASE, 94382607687680, 94382607687680, ++STORE, 94382607687680, 94382607691775, ++STORE, 94382607691776, 94382607695871, ++ERASE, 140252451614720, 140252451614720, ++STORE, 140252451614720, 140252451618815, ++STORE, 140252451618816, 140252451622911, ++ERASE, 47380343541760, 47380343541760, ++STORE, 94382626803712, 94382626938879, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730900271104, 140737488351231, ++ERASE, 140730900271104, 140730900271104, ++STORE, 140730900271104, 140730900275199, ++STORE, 93855478120448, 93855478337535, ++ERASE, 93855478120448, 93855478120448, ++STORE, 93855478120448, 93855478198271, ++STORE, 93855478198272, 93855478337535, ++ERASE, 93855478198272, 93855478198272, ++STORE, 93855478198272, 93855478243327, ++STORE, 93855478243328, 93855478288383, ++STORE, 93855478288384, 93855478337535, ++STORE, 140092686573568, 140092686745599, ++ERASE, 140092686573568, 140092686573568, ++STORE, 140092686573568, 140092686577663, ++STORE, 140092686577664, 140092686745599, ++ERASE, 140092686577664, 140092686577664, ++STORE, 140092686577664, 140092686700543, ++STORE, 140092686700544, 140092686733311, ++STORE, 140092686733312, 140092686741503, ++STORE, 140092686741504, 140092686745599, ++STORE, 140730900537344, 140730900541439, ++STORE, 140730900525056, 140730900537343, ++STORE, 47540108423168, 47540108431359, ++STORE, 47540108431360, 47540108439551, ++STORE, 47540108439552, 47540110278655, ++STORE, 47540108578816, 47540110278655, ++STORE, 47540108439552, 47540108578815, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540110237695, ++STORE, 47540110237696, 47540110278655, ++STORE, 47540109922304, 47540110237695, ++STORE, 47540108578816, 47540109922303, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540109922303, ++STORE, 47540110233600, 47540110237695, ++STORE, 47540109922304, 47540110233599, ++ERASE, 47540109922304, 47540109922304, ++STORE, 47540109922304, 47540110233599, ++STORE, 47540110262272, 47540110278655, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110262272, 47540110262272, ++STORE, 47540110262272, 47540110278655, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110254079, ++STORE, 47540110254080, 47540110262271, ++ERASE, 93855478288384, 93855478288384, ++STORE, 93855478288384, 93855478333439, ++STORE, 93855478333440, 93855478337535, ++ERASE, 140092686733312, 140092686733312, ++STORE, 140092686733312, 140092686737407, ++STORE, 140092686737408, 140092686741503, ++ERASE, 47540108423168, 47540108423168, ++STORE, 93855492222976, 93855492358143, ++STORE, 93855492222976, 93855492493311, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733498146816, 140737488351231, ++ERASE, 140733498146816, 140733498146816, ++STORE, 140733498146816, 140733498150911, ++STORE, 94170739654656, 94170740367359, ++ERASE, 94170739654656, 94170739654656, ++STORE, 94170739654656, 94170739703807, ++STORE, 94170739703808, 94170740367359, ++ERASE, 94170739703808, 94170739703808, ++STORE, 94170739703808, 94170740248575, ++STORE, 94170740248576, 94170740346879, ++STORE, 94170740346880, 94170740367359, ++STORE, 140024788877312, 140024789049343, ++ERASE, 140024788877312, 140024788877312, ++STORE, 140024788877312, 140024788881407, ++STORE, 140024788881408, 140024789049343, ++ERASE, 140024788881408, 140024788881408, ++STORE, 140024788881408, 140024789004287, ++STORE, 140024789004288, 140024789037055, ++STORE, 140024789037056, 140024789045247, ++STORE, 140024789045248, 140024789049343, ++STORE, 140733499023360, 140733499027455, ++STORE, 140733499011072, 140733499023359, ++STORE, 47608006119424, 47608006127615, ++STORE, 47608006127616, 47608006135807, ++STORE, 47608006135808, 47608006242303, ++STORE, 47608006152192, 47608006242303, ++STORE, 47608006135808, 47608006152191, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006225919, ++STORE, 47608006225920, 47608006242303, ++STORE, 47608006205440, 47608006225919, ++STORE, 47608006152192, 47608006205439, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006205439, ++STORE, 47608006221824, 47608006225919, ++STORE, 47608006205440, 47608006221823, ++ERASE, 47608006205440, 47608006205440, ++STORE, 47608006205440, 47608006221823, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006234112, 47608006234112, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006242304, 47608008081407, ++STORE, 47608006381568, 47608008081407, ++STORE, 47608006242304, 47608006381567, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608008040447, ++STORE, 47608008040448, 47608008081407, ++STORE, 47608007725056, 47608008040447, ++STORE, 47608006381568, 47608007725055, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608007725055, ++STORE, 47608008036352, 47608008040447, ++STORE, 47608007725056, 47608008036351, ++ERASE, 47608007725056, 47608007725056, ++STORE, 47608007725056, 47608008036351, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008065024, 47608008065024, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008065024, 47608008093695, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008056831, ++STORE, 47608008056832, 47608008065023, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006230015, ++STORE, 47608006230016, 47608006234111, ++ERASE, 94170740346880, 94170740346880, ++STORE, 94170740346880, 94170740363263, ++STORE, 94170740363264, 94170740367359, ++ERASE, 140024789037056, 140024789037056, ++STORE, 140024789037056, 140024789041151, ++STORE, 140024789041152, 140024789045247, ++ERASE, 47608006119424, 47608006119424, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730264326144, 140737488351231, ++ERASE, 140730264326144, 140730264326144, ++STORE, 140730264326144, 140730264330239, ++STORE, 94653216407552, 94653217120255, ++ERASE, 94653216407552, 94653216407552, ++STORE, 94653216407552, 94653216456703, ++STORE, 94653216456704, 94653217120255, ++ERASE, 94653216456704, 94653216456704, ++STORE, 94653216456704, 94653217001471, ++STORE, 94653217001472, 94653217099775, ++STORE, 94653217099776, 94653217120255, ++STORE, 140103617011712, 140103617183743, ++ERASE, 140103617011712, 140103617011712, ++STORE, 140103617011712, 140103617015807, ++STORE, 140103617015808, 140103617183743, ++ERASE, 140103617015808, 140103617015808, ++STORE, 140103617015808, 140103617138687, ++STORE, 140103617138688, 140103617171455, ++STORE, 140103617171456, 140103617179647, ++STORE, 140103617179648, 140103617183743, ++STORE, 140730265427968, 140730265432063, ++STORE, 140730265415680, 140730265427967, ++STORE, 47529177985024, 47529177993215, ++STORE, 47529177993216, 47529178001407, ++STORE, 47529178001408, 47529178107903, ++STORE, 47529178017792, 47529178107903, ++STORE, 47529178001408, 47529178017791, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178091519, ++STORE, 47529178091520, 47529178107903, ++STORE, 47529178071040, 47529178091519, ++STORE, 47529178017792, 47529178071039, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178071039, ++STORE, 47529178087424, 47529178091519, ++STORE, 47529178071040, 47529178087423, ++ERASE, 47529178071040, 47529178071040, ++STORE, 47529178071040, 47529178087423, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178099712, 47529178099712, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178107904, 47529179947007, ++STORE, 47529178247168, 47529179947007, ++STORE, 47529178107904, 47529178247167, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179906047, ++STORE, 47529179906048, 47529179947007, ++STORE, 47529179590656, 47529179906047, ++STORE, 47529178247168, 47529179590655, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179590655, ++STORE, 47529179901952, 47529179906047, ++STORE, 47529179590656, 47529179901951, ++ERASE, 47529179590656, 47529179590656, ++STORE, 47529179590656, 47529179901951, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179930624, 47529179930624, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179930624, 47529179959295, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179922431, ++STORE, 47529179922432, 47529179930623, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178095615, ++STORE, 47529178095616, 47529178099711, ++ERASE, 94653217099776, 94653217099776, ++STORE, 94653217099776, 94653217116159, ++STORE, 94653217116160, 94653217120255, ++ERASE, 140103617171456, 140103617171456, ++STORE, 140103617171456, 140103617175551, ++STORE, 140103617175552, 140103617179647, ++ERASE, 47529177985024, 47529177985024, ++STORE, 94653241135104, 94653241270271, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736284549120, 140737488351231, ++ERASE, 140736284549120, 140736284549120, ++STORE, 140736284549120, 140736284553215, ++STORE, 93963663822848, 93963664506879, ++ERASE, 93963663822848, 93963663822848, ++STORE, 93963663822848, 93963663884287, ++STORE, 93963663884288, 93963664506879, ++ERASE, 93963663884288, 93963663884288, ++STORE, 93963663884288, 93963664240639, ++STORE, 93963664240640, 93963664379903, ++STORE, 93963664379904, 93963664506879, ++STORE, 140450188439552, 140450188611583, ++ERASE, 140450188439552, 140450188439552, ++STORE, 140450188439552, 140450188443647, ++STORE, 140450188443648, 140450188611583, ++ERASE, 140450188443648, 140450188443648, ++STORE, 140450188443648, 140450188566527, ++STORE, 140450188566528, 140450188599295, ++STORE, 140450188599296, 140450188607487, ++STORE, 140450188607488, 140450188611583, ++STORE, 140736284577792, 140736284581887, ++STORE, 140736284565504, 140736284577791, ++STORE, 47182606557184, 47182606565375, ++STORE, 47182606565376, 47182606573567, ++STORE, 47182606573568, 47182608412671, ++STORE, 47182606712832, 47182608412671, ++STORE, 47182606573568, 47182606712831, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608371711, ++STORE, 47182608371712, 47182608412671, ++STORE, 47182608056320, 47182608371711, ++STORE, 47182606712832, 47182608056319, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608056319, ++STORE, 47182608367616, 47182608371711, ++STORE, 47182608056320, 47182608367615, ++ERASE, 47182608056320, 47182608056320, ++STORE, 47182608056320, 47182608367615, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608396288, 47182608396288, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608412672, 47182608523263, ++STORE, 47182608429056, 47182608523263, ++STORE, 47182608412672, 47182608429055, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608515071, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608490496, 47182608515071, ++STORE, 47182608429056, 47182608490495, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608490495, ++STORE, 47182608510976, 47182608515071, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608490496, 47182608490496, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608523264, 47182608568319, ++ERASE, 47182608523264, 47182608523264, ++STORE, 47182608523264, 47182608531455, ++STORE, 47182608531456, 47182608568319, ++STORE, 47182608551936, 47182608568319, ++STORE, 47182608531456, 47182608551935, ++ERASE, 47182608531456, 47182608531456, ++STORE, 47182608531456, 47182608551935, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608551936, 47182608560127, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608568319, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608560127, ++STORE, 47182608560128, 47182608568319, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608568320, 47182608916479, ++STORE, 47182608609280, 47182608916479, ++STORE, 47182608568320, 47182608609279, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608891903, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608822272, 47182608891903, ++STORE, 47182608609280, 47182608822271, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608822271, ++STORE, 47182608887808, 47182608891903, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608822272, 47182608822272, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608916480, 47182611177471, ++STORE, 47182609068032, 47182611177471, ++STORE, 47182608916480, 47182609068031, ++ERASE, 47182609068032, 47182609068032, ++STORE, 47182609068032, 47182611161087, ++STORE, 47182611161088, 47182611177471, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611169280, 47182611169280, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611177472, 47182611312639, ++ERASE, 47182611177472, 47182611177472, ++STORE, 47182611177472, 47182611202047, ++STORE, 47182611202048, 47182611312639, ++STORE, 47182611263488, 47182611312639, ++STORE, 47182611202048, 47182611263487, ++ERASE, 47182611202048, 47182611202048, ++STORE, 47182611202048, 47182611263487, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611263488, 47182611288063, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611312639, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611288063, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611296256, 47182611296256, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611296256, 47182611320831, ++STORE, 47182611320832, 47182611484671, ++ERASE, 47182611320832, 47182611320832, ++STORE, 47182611320832, 47182611333119, ++STORE, 47182611333120, 47182611484671, ++STORE, 47182611431424, 47182611484671, ++STORE, 47182611333120, 47182611431423, ++ERASE, 47182611333120, 47182611333120, ++STORE, 47182611333120, 47182611431423, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611431424, 47182611476479, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611484671, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611476479, ++STORE, 47182611476480, 47182611484671, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611484672, 47182612082687, ++STORE, 47182611603456, 47182612082687, ++STORE, 47182611484672, 47182611603455, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182612029439, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182611918848, 47182612029439, ++STORE, 47182611603456, 47182611918847, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182611918847, ++STORE, 47182612025344, 47182612029439, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182611918848, 47182611918848, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182612082688, 47182615134207, ++STORE, 47182612627456, 47182615134207, ++STORE, 47182612082688, 47182612627455, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614913023, ++STORE, 47182614913024, 47182615134207, ++STORE, 47182614323200, 47182614913023, ++STORE, 47182612627456, 47182614323199, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614323199, ++STORE, 47182614908928, 47182614913023, ++STORE, 47182614323200, 47182614908927, ++ERASE, 47182614323200, 47182614323200, ++STORE, 47182614323200, 47182614908927, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182615117824, 47182615117824, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182615134208, 47182615166975, ++ERASE, 47182615134208, 47182615134208, ++STORE, 47182615134208, 47182615142399, ++STORE, 47182615142400, 47182615166975, ++STORE, 47182615154688, 47182615166975, ++STORE, 47182615142400, 47182615154687, ++ERASE, 47182615142400, 47182615142400, ++STORE, 47182615142400, 47182615154687, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615154688, 47182615158783, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615166975, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615158783, ++STORE, 47182615158784, 47182615166975, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615166976, 47182615203839, ++ERASE, 47182615166976, 47182615166976, ++STORE, 47182615166976, 47182615175167, ++STORE, 47182615175168, 47182615203839, ++STORE, 47182615191552, 47182615203839, ++STORE, 47182615175168, 47182615191551, ++ERASE, 47182615175168, 47182615175168, ++STORE, 47182615175168, 47182615191551, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615191552, 47182615195647, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615203839, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615195647, ++STORE, 47182615195648, 47182615203839, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615203840, 47182615678975, ++ERASE, 47182615203840, 47182615203840, ++STORE, 47182615203840, 47182615212031, ++STORE, 47182615212032, 47182615678975, ++STORE, 47182615547904, 47182615678975, ++STORE, 47182615212032, 47182615547903, ++ERASE, 47182615212032, 47182615212032, ++STORE, 47182615212032, 47182615547903, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615547904, 47182615670783, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615678975, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615670783, ++STORE, 47182615670784, 47182615678975, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615678976, 47182615687167, ++STORE, 47182615687168, 47182615707647, ++ERASE, 47182615687168, 47182615687168, ++STORE, 47182615687168, 47182615691263, ++STORE, 47182615691264, 47182615707647, ++STORE, 47182615695360, 47182615707647, ++STORE, 47182615691264, 47182615695359, ++ERASE, 47182615691264, 47182615691264, ++STORE, 47182615691264, 47182615695359, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615695360, 47182615699455, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615707647, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615699455, ++STORE, 47182615699456, 47182615707647, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615707648, 47182615715839, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608388095, ++STORE, 47182608388096, 47182608396287, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615703551, ++STORE, 47182615703552, 47182615707647, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611292159, ++STORE, 47182611292160, 47182611296255, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615674879, ++STORE, 47182615674880, 47182615678975, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615199743, ++STORE, 47182615199744, 47182615203839, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615162879, ++STORE, 47182615162880, 47182615166975, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615109631, ++STORE, 47182615109632, 47182615117823, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612066303, ++STORE, 47182612066304, 47182612082687, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611480575, ++STORE, 47182611480576, 47182611484671, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611165183, ++STORE, 47182611165184, 47182611169279, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608912383, ++STORE, 47182608912384, 47182608916479, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608564223, ++STORE, 47182608564224, 47182608568319, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608519167, ++STORE, 47182608519168, 47182608523263, ++ERASE, 93963664379904, 93963664379904, ++STORE, 93963664379904, 93963664502783, ++STORE, 93963664502784, 93963664506879, ++ERASE, 140450188599296, 140450188599296, ++STORE, 140450188599296, 140450188603391, ++STORE, 140450188603392, 140450188607487, ++ERASE, 47182606557184, 47182606557184, ++STORE, 93963694723072, 93963694858239, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730313261056, 140737488351231, ++ERASE, 140730313261056, 140730313261056, ++STORE, 140730313261056, 140730313265151, ++STORE, 94386579017728, 94386579697663, ++ERASE, 94386579017728, 94386579017728, ++STORE, 94386579017728, 94386579083263, ++STORE, 94386579083264, 94386579697663, ++ERASE, 94386579083264, 94386579083264, ++STORE, 94386579083264, 94386579431423, ++STORE, 94386579431424, 94386579570687, ++STORE, 94386579570688, 94386579697663, ++STORE, 140124810838016, 140124811010047, ++ERASE, 140124810838016, 140124810838016, ++STORE, 140124810838016, 140124810842111, ++STORE, 140124810842112, 140124811010047, ++ERASE, 140124810842112, 140124810842112, ++STORE, 140124810842112, 140124810964991, ++STORE, 140124810964992, 140124810997759, ++STORE, 140124810997760, 140124811005951, ++STORE, 140124811005952, 140124811010047, ++STORE, 140730313601024, 140730313605119, ++STORE, 140730313588736, 140730313601023, ++STORE, 47507984158720, 47507984166911, ++STORE, 47507984166912, 47507984175103, ++STORE, 47507984175104, 47507986014207, ++STORE, 47507984314368, 47507986014207, ++STORE, 47507984175104, 47507984314367, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985973247, ++STORE, 47507985973248, 47507986014207, ++STORE, 47507985657856, 47507985973247, ++STORE, 47507984314368, 47507985657855, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985657855, ++STORE, 47507985969152, 47507985973247, ++STORE, 47507985657856, 47507985969151, ++ERASE, 47507985657856, 47507985657856, ++STORE, 47507985657856, 47507985969151, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985997824, 47507985997824, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507986014208, 47507986124799, ++STORE, 47507986030592, 47507986124799, ++STORE, 47507986014208, 47507986030591, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986116607, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986092032, 47507986116607, ++STORE, 47507986030592, 47507986092031, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986092031, ++STORE, 47507986112512, 47507986116607, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986092032, 47507986092032, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986124800, 47507986169855, ++ERASE, 47507986124800, 47507986124800, ++STORE, 47507986124800, 47507986132991, ++STORE, 47507986132992, 47507986169855, ++STORE, 47507986153472, 47507986169855, ++STORE, 47507986132992, 47507986153471, ++ERASE, 47507986132992, 47507986132992, ++STORE, 47507986132992, 47507986153471, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986153472, 47507986161663, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986169855, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986161663, ++STORE, 47507986161664, 47507986169855, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986169856, 47507986518015, ++STORE, 47507986210816, 47507986518015, ++STORE, 47507986169856, 47507986210815, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986493439, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986423808, 47507986493439, ++STORE, 47507986210816, 47507986423807, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986423807, ++STORE, 47507986489344, 47507986493439, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986423808, 47507986423808, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986518016, 47507988779007, ++STORE, 47507986669568, 47507988779007, ++STORE, 47507986518016, 47507986669567, ++ERASE, 47507986669568, 47507986669568, ++STORE, 47507986669568, 47507988762623, ++STORE, 47507988762624, 47507988779007, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988770816, 47507988770816, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988779008, 47507988914175, ++ERASE, 47507988779008, 47507988779008, ++STORE, 47507988779008, 47507988803583, ++STORE, 47507988803584, 47507988914175, ++STORE, 47507988865024, 47507988914175, ++STORE, 47507988803584, 47507988865023, ++ERASE, 47507988803584, 47507988803584, ++STORE, 47507988803584, 47507988865023, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988865024, 47507988889599, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988914175, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988889599, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988897792, 47507988897792, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988897792, 47507988922367, ++STORE, 47507988922368, 47507989086207, ++ERASE, 47507988922368, 47507988922368, ++STORE, 47507988922368, 47507988934655, ++STORE, 47507988934656, 47507989086207, ++STORE, 47507989032960, 47507989086207, ++STORE, 47507988934656, 47507989032959, ++ERASE, 47507988934656, 47507988934656, ++STORE, 47507988934656, 47507989032959, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989032960, 47507989078015, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989086207, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989078015, ++STORE, 47507989078016, 47507989086207, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989086208, 47507989684223, ++STORE, 47507989204992, 47507989684223, ++STORE, 47507989086208, 47507989204991, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989630975, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989520384, 47507989630975, ++STORE, 47507989204992, 47507989520383, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989520383, ++STORE, 47507989626880, 47507989630975, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989520384, 47507989520384, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989684224, 47507992735743, ++STORE, 47507990228992, 47507992735743, ++STORE, 47507989684224, 47507990228991, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507992514559, ++STORE, 47507992514560, 47507992735743, ++STORE, 47507991924736, 47507992514559, ++STORE, 47507990228992, 47507991924735, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507991924735, ++STORE, 47507992510464, 47507992514559, ++STORE, 47507991924736, 47507992510463, ++ERASE, 47507991924736, 47507991924736, ++STORE, 47507991924736, 47507992510463, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992719360, 47507992719360, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992735744, 47507992768511, ++ERASE, 47507992735744, 47507992735744, ++STORE, 47507992735744, 47507992743935, ++STORE, 47507992743936, 47507992768511, ++STORE, 47507992756224, 47507992768511, ++STORE, 47507992743936, 47507992756223, ++ERASE, 47507992743936, 47507992743936, ++STORE, 47507992743936, 47507992756223, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992756224, 47507992760319, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992768511, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992760319, ++STORE, 47507992760320, 47507992768511, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992768512, 47507992805375, ++ERASE, 47507992768512, 47507992768512, ++STORE, 47507992768512, 47507992776703, ++STORE, 47507992776704, 47507992805375, ++STORE, 47507992793088, 47507992805375, ++STORE, 47507992776704, 47507992793087, ++ERASE, 47507992776704, 47507992776704, ++STORE, 47507992776704, 47507992793087, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992793088, 47507992797183, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992805375, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992797183, ++STORE, 47507992797184, 47507992805375, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992805376, 47507993280511, ++ERASE, 47507992805376, 47507992805376, ++STORE, 47507992805376, 47507992813567, ++STORE, 47507992813568, 47507993280511, ++STORE, 47507993149440, 47507993280511, ++STORE, 47507992813568, 47507993149439, ++ERASE, 47507992813568, 47507992813568, ++STORE, 47507992813568, 47507993149439, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993149440, 47507993272319, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993280511, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993272319, ++STORE, 47507993272320, 47507993280511, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993280512, 47507993288703, ++STORE, 47507993288704, 47507993309183, ++ERASE, 47507993288704, 47507993288704, ++STORE, 47507993288704, 47507993292799, ++STORE, 47507993292800, 47507993309183, ++STORE, 47507993296896, 47507993309183, ++STORE, 47507993292800, 47507993296895, ++ERASE, 47507993292800, 47507993292800, ++STORE, 47507993292800, 47507993296895, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993296896, 47507993300991, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993309183, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993300991, ++STORE, 47507993300992, 47507993309183, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993309184, 47507993317375, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985989631, ++STORE, 47507985989632, 47507985997823, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993305087, ++STORE, 47507993305088, 47507993309183, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988893695, ++STORE, 47507988893696, 47507988897791, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993276415, ++STORE, 47507993276416, 47507993280511, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992801279, ++STORE, 47507992801280, 47507992805375, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992764415, ++STORE, 47507992764416, 47507992768511, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992711167, ++STORE, 47507992711168, 47507992719359, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989667839, ++STORE, 47507989667840, 47507989684223, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989082111, ++STORE, 47507989082112, 47507989086207, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988766719, ++STORE, 47507988766720, 47507988770815, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986513919, ++STORE, 47507986513920, 47507986518015, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986165759, ++STORE, 47507986165760, 47507986169855, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986120703, ++STORE, 47507986120704, 47507986124799, ++ERASE, 94386579570688, 94386579570688, ++STORE, 94386579570688, 94386579693567, ++STORE, 94386579693568, 94386579697663, ++ERASE, 140124810997760, 140124810997760, ++STORE, 140124810997760, 140124811001855, ++STORE, 140124811001856, 140124811005951, ++ERASE, 47507984158720, 47507984158720, ++STORE, 94386583982080, 94386584117247, ++STORE, 94386583982080, 94386584256511, ++ERASE, 94386583982080, 94386583982080, ++STORE, 94386583982080, 94386584223743, ++STORE, 94386584223744, 94386584256511, ++ERASE, 94386584223744, 94386584223744, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733763395584, 140737488351231, ++ERASE, 140733763395584, 140733763395584, ++STORE, 140733763395584, 140733763399679, ++STORE, 94011546472448, 94011547152383, ++ERASE, 94011546472448, 94011546472448, ++STORE, 94011546472448, 94011546537983, ++STORE, 94011546537984, 94011547152383, ++ERASE, 94011546537984, 94011546537984, ++STORE, 94011546537984, 94011546886143, ++STORE, 94011546886144, 94011547025407, ++STORE, 94011547025408, 94011547152383, ++STORE, 139757597949952, 139757598121983, ++ERASE, 139757597949952, 139757597949952, ++STORE, 139757597949952, 139757597954047, ++STORE, 139757597954048, 139757598121983, ++ERASE, 139757597954048, 139757597954048, ++STORE, 139757597954048, 139757598076927, ++STORE, 139757598076928, 139757598109695, ++STORE, 139757598109696, 139757598117887, ++STORE, 139757598117888, 139757598121983, ++STORE, 140733763596288, 140733763600383, ++STORE, 140733763584000, 140733763596287, ++STORE, 47875197046784, 47875197054975, ++STORE, 47875197054976, 47875197063167, ++STORE, 47875197063168, 47875198902271, ++STORE, 47875197202432, 47875198902271, ++STORE, 47875197063168, 47875197202431, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198861311, ++STORE, 47875198861312, 47875198902271, ++STORE, 47875198545920, 47875198861311, ++STORE, 47875197202432, 47875198545919, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198545919, ++STORE, 47875198857216, 47875198861311, ++STORE, 47875198545920, 47875198857215, ++ERASE, 47875198545920, 47875198545920, ++STORE, 47875198545920, 47875198857215, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198885888, 47875198885888, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198902272, 47875199012863, ++STORE, 47875198918656, 47875199012863, ++STORE, 47875198902272, 47875198918655, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875199004671, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875198980096, 47875199004671, ++STORE, 47875198918656, 47875198980095, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875198980095, ++STORE, 47875199000576, 47875199004671, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875198980096, 47875198980096, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875199012864, 47875199057919, ++ERASE, 47875199012864, 47875199012864, ++STORE, 47875199012864, 47875199021055, ++STORE, 47875199021056, 47875199057919, ++STORE, 47875199041536, 47875199057919, ++STORE, 47875199021056, 47875199041535, ++ERASE, 47875199021056, 47875199021056, ++STORE, 47875199021056, 47875199041535, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199041536, 47875199049727, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199057919, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199049727, ++STORE, 47875199049728, 47875199057919, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199057920, 47875199406079, ++STORE, 47875199098880, 47875199406079, ++STORE, 47875199057920, 47875199098879, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199381503, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199311872, 47875199381503, ++STORE, 47875199098880, 47875199311871, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199311871, ++STORE, 47875199377408, 47875199381503, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199311872, 47875199311872, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199406080, 47875201667071, ++STORE, 47875199557632, 47875201667071, ++STORE, 47875199406080, 47875199557631, ++ERASE, 47875199557632, 47875199557632, ++STORE, 47875199557632, 47875201650687, ++STORE, 47875201650688, 47875201667071, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201658880, 47875201658880, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201667072, 47875201802239, ++ERASE, 47875201667072, 47875201667072, ++STORE, 47875201667072, 47875201691647, ++STORE, 47875201691648, 47875201802239, ++STORE, 47875201753088, 47875201802239, ++STORE, 47875201691648, 47875201753087, ++ERASE, 47875201691648, 47875201691648, ++STORE, 47875201691648, 47875201753087, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201753088, 47875201777663, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201802239, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201777663, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201785856, 47875201785856, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201785856, 47875201810431, ++STORE, 47875201810432, 47875201974271, ++ERASE, 47875201810432, 47875201810432, ++STORE, 47875201810432, 47875201822719, ++STORE, 47875201822720, 47875201974271, ++STORE, 47875201921024, 47875201974271, ++STORE, 47875201822720, 47875201921023, ++ERASE, 47875201822720, 47875201822720, ++STORE, 47875201822720, 47875201921023, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201921024, 47875201966079, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201974271, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201966079, ++STORE, 47875201966080, 47875201974271, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201974272, 47875202572287, ++STORE, 47875202093056, 47875202572287, ++STORE, 47875201974272, 47875202093055, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202519039, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202408448, 47875202519039, ++STORE, 47875202093056, 47875202408447, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202408447, ++STORE, 47875202514944, 47875202519039, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202408448, 47875202408448, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202572288, 47875205623807, ++STORE, 47875203117056, 47875205623807, ++STORE, 47875202572288, 47875203117055, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875205402623, ++STORE, 47875205402624, 47875205623807, ++STORE, 47875204812800, 47875205402623, ++STORE, 47875203117056, 47875204812799, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875204812799, ++STORE, 47875205398528, 47875205402623, ++STORE, 47875204812800, 47875205398527, ++ERASE, 47875204812800, 47875204812800, ++STORE, 47875204812800, 47875205398527, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205607424, 47875205607424, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205623808, 47875205656575, ++ERASE, 47875205623808, 47875205623808, ++STORE, 47875205623808, 47875205631999, ++STORE, 47875205632000, 47875205656575, ++STORE, 47875205644288, 47875205656575, ++STORE, 47875205632000, 47875205644287, ++ERASE, 47875205632000, 47875205632000, ++STORE, 47875205632000, 47875205644287, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205644288, 47875205648383, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205656575, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205648383, ++STORE, 47875205648384, 47875205656575, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205656576, 47875205693439, ++ERASE, 47875205656576, 47875205656576, ++STORE, 47875205656576, 47875205664767, ++STORE, 47875205664768, 47875205693439, ++STORE, 47875205681152, 47875205693439, ++STORE, 47875205664768, 47875205681151, ++ERASE, 47875205664768, 47875205664768, ++STORE, 47875205664768, 47875205681151, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205681152, 47875205685247, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205693439, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205685247, ++STORE, 47875205685248, 47875205693439, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205693440, 47875206168575, ++ERASE, 47875205693440, 47875205693440, ++STORE, 47875205693440, 47875205701631, ++STORE, 47875205701632, 47875206168575, ++STORE, 47875206037504, 47875206168575, ++STORE, 47875205701632, 47875206037503, ++ERASE, 47875205701632, 47875205701632, ++STORE, 47875205701632, 47875206037503, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206037504, 47875206160383, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206168575, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206160383, ++STORE, 47875206160384, 47875206168575, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206168576, 47875206176767, ++STORE, 47875206176768, 47875206197247, ++ERASE, 47875206176768, 47875206176768, ++STORE, 47875206176768, 47875206180863, ++STORE, 47875206180864, 47875206197247, ++STORE, 47875206184960, 47875206197247, ++STORE, 47875206180864, 47875206184959, ++ERASE, 47875206180864, 47875206180864, ++STORE, 47875206180864, 47875206184959, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206184960, 47875206189055, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206197247, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206189055, ++STORE, 47875206189056, 47875206197247, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206197248, 47875206205439, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198877695, ++STORE, 47875198877696, 47875198885887, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206193151, ++STORE, 47875206193152, 47875206197247, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201781759, ++STORE, 47875201781760, 47875201785855, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206164479, ++STORE, 47875206164480, 47875206168575, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205689343, ++STORE, 47875205689344, 47875205693439, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205652479, ++STORE, 47875205652480, 47875205656575, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205599231, ++STORE, 47875205599232, 47875205607423, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202555903, ++STORE, 47875202555904, 47875202572287, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201970175, ++STORE, 47875201970176, 47875201974271, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201654783, ++STORE, 47875201654784, 47875201658879, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199401983, ++STORE, 47875199401984, 47875199406079, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199053823, ++STORE, 47875199053824, 47875199057919, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199008767, ++STORE, 47875199008768, 47875199012863, ++ERASE, 94011547025408, 94011547025408, ++STORE, 94011547025408, 94011547148287, ++STORE, 94011547148288, 94011547152383, ++ERASE, 139757598109696, 139757598109696, ++STORE, 139757598109696, 139757598113791, ++STORE, 139757598113792, 139757598117887, ++ERASE, 47875197046784, 47875197046784, ++STORE, 94011557584896, 94011557720063, ++STORE, 94011557584896, 94011557855231, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557851135, ++STORE, 94011557851136, 94011557855231, ++ERASE, 94011557851136, 94011557851136, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557847039, ++STORE, 94011557847040, 94011557851135, ++ERASE, 94011557847040, 94011557847040, ++STORE, 94011557584896, 94011557982207, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557978111, ++STORE, 94011557978112, 94011557982207, ++ERASE, 94011557978112, 94011557978112, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557974015, ++STORE, 94011557974016, 94011557978111, ++ERASE, 94011557974016, 94011557974016, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734130360320, 140737488351231, ++ERASE, 140734130360320, 140734130360320, ++STORE, 140734130360320, 140734130364415, ++STORE, 94641232105472, 94641232785407, ++ERASE, 94641232105472, 94641232105472, ++STORE, 94641232105472, 94641232171007, ++STORE, 94641232171008, 94641232785407, ++ERASE, 94641232171008, 94641232171008, ++STORE, 94641232171008, 94641232519167, ++STORE, 94641232519168, 94641232658431, ++STORE, 94641232658432, 94641232785407, ++STORE, 139726599516160, 139726599688191, ++ERASE, 139726599516160, 139726599516160, ++STORE, 139726599516160, 139726599520255, ++STORE, 139726599520256, 139726599688191, ++ERASE, 139726599520256, 139726599520256, ++STORE, 139726599520256, 139726599643135, ++STORE, 139726599643136, 139726599675903, ++STORE, 139726599675904, 139726599684095, ++STORE, 139726599684096, 139726599688191, ++STORE, 140734130446336, 140734130450431, ++STORE, 140734130434048, 140734130446335, ++STORE, 47906195480576, 47906195488767, ++STORE, 47906195488768, 47906195496959, ++STORE, 47906195496960, 47906197336063, ++STORE, 47906195636224, 47906197336063, ++STORE, 47906195496960, 47906195636223, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906197295103, ++STORE, 47906197295104, 47906197336063, ++STORE, 47906196979712, 47906197295103, ++STORE, 47906195636224, 47906196979711, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906196979711, ++STORE, 47906197291008, 47906197295103, ++STORE, 47906196979712, 47906197291007, ++ERASE, 47906196979712, 47906196979712, ++STORE, 47906196979712, 47906197291007, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197319680, 47906197319680, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197336064, 47906197446655, ++STORE, 47906197352448, 47906197446655, ++STORE, 47906197336064, 47906197352447, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197438463, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197413888, 47906197438463, ++STORE, 47906197352448, 47906197413887, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197413887, ++STORE, 47906197434368, 47906197438463, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197413888, 47906197413888, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197446656, 47906197491711, ++ERASE, 47906197446656, 47906197446656, ++STORE, 47906197446656, 47906197454847, ++STORE, 47906197454848, 47906197491711, ++STORE, 47906197475328, 47906197491711, ++STORE, 47906197454848, 47906197475327, ++ERASE, 47906197454848, 47906197454848, ++STORE, 47906197454848, 47906197475327, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197475328, 47906197483519, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197491711, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197483519, ++STORE, 47906197483520, 47906197491711, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197491712, 47906197839871, ++STORE, 47906197532672, 47906197839871, ++STORE, 47906197491712, 47906197532671, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197815295, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197745664, 47906197815295, ++STORE, 47906197532672, 47906197745663, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197745663, ++STORE, 47906197811200, 47906197815295, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197745664, 47906197745664, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197839872, 47906200100863, ++STORE, 47906197991424, 47906200100863, ++STORE, 47906197839872, 47906197991423, ++ERASE, 47906197991424, 47906197991424, ++STORE, 47906197991424, 47906200084479, ++STORE, 47906200084480, 47906200100863, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200092672, 47906200092672, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200100864, 47906200236031, ++ERASE, 47906200100864, 47906200100864, ++STORE, 47906200100864, 47906200125439, ++STORE, 47906200125440, 47906200236031, ++STORE, 47906200186880, 47906200236031, ++STORE, 47906200125440, 47906200186879, ++ERASE, 47906200125440, 47906200125440, ++STORE, 47906200125440, 47906200186879, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200186880, 47906200211455, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200236031, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200211455, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200219648, 47906200219648, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200219648, 47906200244223, ++STORE, 47906200244224, 47906200408063, ++ERASE, 47906200244224, 47906200244224, ++STORE, 47906200244224, 47906200256511, ++STORE, 47906200256512, 47906200408063, ++STORE, 47906200354816, 47906200408063, ++STORE, 47906200256512, 47906200354815, ++ERASE, 47906200256512, 47906200256512, ++STORE, 47906200256512, 47906200354815, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200354816, 47906200399871, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200408063, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200399871, ++STORE, 47906200399872, 47906200408063, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200408064, 47906201006079, ++STORE, 47906200526848, 47906201006079, ++STORE, 47906200408064, 47906200526847, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200952831, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906200842240, 47906200952831, ++STORE, 47906200526848, 47906200842239, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200842239, ++STORE, 47906200948736, 47906200952831, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200842240, 47906200842240, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906201006080, 47906204057599, ++STORE, 47906201550848, 47906204057599, ++STORE, 47906201006080, 47906201550847, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203836415, ++STORE, 47906203836416, 47906204057599, ++STORE, 47906203246592, 47906203836415, ++STORE, 47906201550848, 47906203246591, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203246591, ++STORE, 47906203832320, 47906203836415, ++STORE, 47906203246592, 47906203832319, ++ERASE, 47906203246592, 47906203246592, ++STORE, 47906203246592, 47906203832319, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906204041216, 47906204041216, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906204057600, 47906204090367, ++ERASE, 47906204057600, 47906204057600, ++STORE, 47906204057600, 47906204065791, ++STORE, 47906204065792, 47906204090367, ++STORE, 47906204078080, 47906204090367, ++STORE, 47906204065792, 47906204078079, ++ERASE, 47906204065792, 47906204065792, ++STORE, 47906204065792, 47906204078079, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204078080, 47906204082175, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204090367, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204082175, ++STORE, 47906204082176, 47906204090367, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204090368, 47906204127231, ++ERASE, 47906204090368, 47906204090368, ++STORE, 47906204090368, 47906204098559, ++STORE, 47906204098560, 47906204127231, ++STORE, 47906204114944, 47906204127231, ++STORE, 47906204098560, 47906204114943, ++ERASE, 47906204098560, 47906204098560, ++STORE, 47906204098560, 47906204114943, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204114944, 47906204119039, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204127231, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204119039, ++STORE, 47906204119040, 47906204127231, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204127232, 47906204602367, ++ERASE, 47906204127232, 47906204127232, ++STORE, 47906204127232, 47906204135423, ++STORE, 47906204135424, 47906204602367, ++STORE, 47906204471296, 47906204602367, ++STORE, 47906204135424, 47906204471295, ++ERASE, 47906204135424, 47906204135424, ++STORE, 47906204135424, 47906204471295, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204471296, 47906204594175, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204602367, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204594175, ++STORE, 47906204594176, 47906204602367, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204602368, 47906204610559, ++STORE, 47906204610560, 47906204631039, ++ERASE, 47906204610560, 47906204610560, ++STORE, 47906204610560, 47906204614655, ++STORE, 47906204614656, 47906204631039, ++STORE, 47906204618752, 47906204631039, ++STORE, 47906204614656, 47906204618751, ++ERASE, 47906204614656, 47906204614656, ++STORE, 47906204614656, 47906204618751, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204618752, 47906204622847, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204631039, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204622847, ++STORE, 47906204622848, 47906204631039, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204631040, 47906204639231, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197311487, ++STORE, 47906197311488, 47906197319679, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204626943, ++STORE, 47906204626944, 47906204631039, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200215551, ++STORE, 47906200215552, 47906200219647, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204598271, ++STORE, 47906204598272, 47906204602367, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204123135, ++STORE, 47906204123136, 47906204127231, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204086271, ++STORE, 47906204086272, 47906204090367, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204033023, ++STORE, 47906204033024, 47906204041215, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906200989695, ++STORE, 47906200989696, 47906201006079, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200403967, ++STORE, 47906200403968, 47906200408063, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200088575, ++STORE, 47906200088576, 47906200092671, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197835775, ++STORE, 47906197835776, 47906197839871, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197487615, ++STORE, 47906197487616, 47906197491711, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197442559, ++STORE, 47906197442560, 47906197446655, ++ERASE, 94641232658432, 94641232658432, ++STORE, 94641232658432, 94641232781311, ++STORE, 94641232781312, 94641232785407, ++ERASE, 139726599675904, 139726599675904, ++STORE, 139726599675904, 139726599679999, ++STORE, 139726599680000, 139726599684095, ++ERASE, 47906195480576, 47906195480576, ++STORE, 94641242615808, 94641242750975, ++ }; ++ ++ unsigned long set10[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140736427839488, 140737488351231, ++ERASE, 140736427839488, 140736427839488, ++STORE, 140736427839488, 140736427843583, ++STORE, 94071213395968, 94071213567999, ++ERASE, 94071213395968, 94071213395968, ++STORE, 94071213395968, 94071213412351, ++STORE, 94071213412352, 94071213567999, ++ERASE, 94071213412352, 94071213412352, ++STORE, 94071213412352, 94071213514751, ++STORE, 94071213514752, 94071213555711, ++STORE, 94071213555712, 94071213567999, ++STORE, 139968410644480, 139968410816511, ++ERASE, 139968410644480, 139968410644480, ++STORE, 139968410644480, 139968410648575, ++STORE, 139968410648576, 139968410816511, ++ERASE, 139968410648576, 139968410648576, ++STORE, 139968410648576, 139968410771455, ++STORE, 139968410771456, 139968410804223, ++STORE, 139968410804224, 139968410812415, ++STORE, 139968410812416, 139968410816511, ++STORE, 140736429277184, 140736429281279, ++STORE, 140736429264896, 140736429277183, ++STORE, 47664384352256, 47664384360447, ++STORE, 47664384360448, 47664384368639, ++STORE, 47664384368640, 47664384532479, ++ERASE, 47664384368640, 47664384368640, ++STORE, 47664384368640, 47664384380927, ++STORE, 47664384380928, 47664384532479, ++STORE, 47664384479232, 47664384532479, ++STORE, 47664384380928, 47664384479231, ++ERASE, 47664384380928, 47664384380928, ++STORE, 47664384380928, 47664384479231, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384479232, 47664384524287, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384532479, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384524287, ++STORE, 47664384524288, 47664384532479, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384532480, 47664387583999, ++STORE, 47664385077248, 47664387583999, ++STORE, 47664384532480, 47664385077247, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664387362815, ++STORE, 47664387362816, 47664387583999, ++STORE, 47664386772992, 47664387362815, ++STORE, 47664385077248, 47664386772991, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664386772991, ++STORE, 47664387358720, 47664387362815, ++STORE, 47664386772992, 47664387358719, ++ERASE, 47664386772992, 47664386772992, ++STORE, 47664386772992, 47664387358719, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387567616, 47664387567616, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387584000, 47664389423103, ++STORE, 47664387723264, 47664389423103, ++STORE, 47664387584000, 47664387723263, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389382143, ++STORE, 47664389382144, 47664389423103, ++STORE, 47664389066752, 47664389382143, ++STORE, 47664387723264, 47664389066751, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389066751, ++STORE, 47664389378048, 47664389382143, ++STORE, 47664389066752, 47664389378047, ++ERASE, 47664389066752, 47664389066752, ++STORE, 47664389066752, 47664389378047, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389406720, 47664389406720, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389423104, 47664389558271, ++ERASE, 47664389423104, 47664389423104, ++STORE, 47664389423104, 47664389447679, ++STORE, 47664389447680, 47664389558271, ++STORE, 47664389509120, 47664389558271, ++STORE, 47664389447680, 47664389509119, ++ERASE, 47664389447680, 47664389447680, ++STORE, 47664389447680, 47664389509119, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389509120, 47664389533695, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389558271, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389533695, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389541888, 47664389541888, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389558272, 47664389578751, ++ERASE, 47664389558272, 47664389558272, ++STORE, 47664389558272, 47664389562367, ++STORE, 47664389562368, 47664389578751, ++STORE, 47664389566464, 47664389578751, ++STORE, 47664389562368, 47664389566463, ++ERASE, 47664389562368, 47664389562368, ++STORE, 47664389562368, 47664389566463, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389566464, 47664389570559, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389578751, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389570559, ++STORE, 47664389570560, 47664389578751, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389578752, 47664389586943, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389398527, ++STORE, 47664389398528, 47664389406719, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389574655, ++STORE, 47664389574656, 47664389578751, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389537791, ++STORE, 47664389537792, 47664389541887, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387559423, ++STORE, 47664387559424, 47664387567615, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384528383, ++STORE, 47664384528384, 47664384532479, ++ERASE, 94071213555712, 94071213555712, ++STORE, 94071213555712, 94071213563903, ++STORE, 94071213563904, 94071213567999, ++ERASE, 139968410804224, 139968410804224, ++STORE, 139968410804224, 139968410808319, ++STORE, 139968410808320, 139968410812415, ++ERASE, 47664384352256, 47664384352256, ++STORE, 94071244402688, 94071244537855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728271503360, 140737488351231, ++ERASE, 140728271503360, 140728271503360, ++STORE, 140728271503360, 140728271507455, ++STORE, 94410361982976, 94410362155007, ++ERASE, 94410361982976, 94410361982976, ++STORE, 94410361982976, 94410361999359, ++STORE, 94410361999360, 94410362155007, ++ERASE, 94410361999360, 94410361999360, ++STORE, 94410361999360, 94410362101759, ++STORE, 94410362101760, 94410362142719, ++STORE, 94410362142720, 94410362155007, ++STORE, 140351953997824, 140351954169855, ++ERASE, 140351953997824, 140351953997824, ++STORE, 140351953997824, 140351954001919, ++STORE, 140351954001920, 140351954169855, ++ERASE, 140351954001920, 140351954001920, ++STORE, 140351954001920, 140351954124799, ++STORE, 140351954124800, 140351954157567, ++STORE, 140351954157568, 140351954165759, ++STORE, 140351954165760, 140351954169855, ++STORE, 140728272429056, 140728272433151, ++STORE, 140728272416768, 140728272429055, ++STORE, 47280840998912, 47280841007103, ++STORE, 47280841007104, 47280841015295, ++STORE, 47280841015296, 47280841179135, ++ERASE, 47280841015296, 47280841015296, ++STORE, 47280841015296, 47280841027583, ++STORE, 47280841027584, 47280841179135, ++STORE, 47280841125888, 47280841179135, ++STORE, 47280841027584, 47280841125887, ++ERASE, 47280841027584, 47280841027584, ++STORE, 47280841027584, 47280841125887, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841125888, 47280841170943, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841179135, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841170943, ++STORE, 47280841170944, 47280841179135, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841179136, 47280844230655, ++STORE, 47280841723904, 47280844230655, ++STORE, 47280841179136, 47280841723903, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280844009471, ++STORE, 47280844009472, 47280844230655, ++STORE, 47280843419648, 47280844009471, ++STORE, 47280841723904, 47280843419647, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280843419647, ++STORE, 47280844005376, 47280844009471, ++STORE, 47280843419648, 47280844005375, ++ERASE, 47280843419648, 47280843419648, ++STORE, 47280843419648, 47280844005375, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844214272, 47280844214272, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844230656, 47280846069759, ++STORE, 47280844369920, 47280846069759, ++STORE, 47280844230656, 47280844369919, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280846028799, ++STORE, 47280846028800, 47280846069759, ++STORE, 47280845713408, 47280846028799, ++STORE, 47280844369920, 47280845713407, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280845713407, ++STORE, 47280846024704, 47280846028799, ++STORE, 47280845713408, 47280846024703, ++ERASE, 47280845713408, 47280845713408, ++STORE, 47280845713408, 47280846024703, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846053376, 47280846053376, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846069760, 47280846204927, ++ERASE, 47280846069760, 47280846069760, ++STORE, 47280846069760, 47280846094335, ++STORE, 47280846094336, 47280846204927, ++STORE, 47280846155776, 47280846204927, ++STORE, 47280846094336, 47280846155775, ++ERASE, 47280846094336, 47280846094336, ++STORE, 47280846094336, 47280846155775, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846155776, 47280846180351, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846204927, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846180351, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846188544, 47280846188544, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846204928, 47280846225407, ++ERASE, 47280846204928, 47280846204928, ++STORE, 47280846204928, 47280846209023, ++STORE, 47280846209024, 47280846225407, ++STORE, 47280846213120, 47280846225407, ++STORE, 47280846209024, 47280846213119, ++ERASE, 47280846209024, 47280846209024, ++STORE, 47280846209024, 47280846213119, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846213120, 47280846217215, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846225407, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846217215, ++STORE, 47280846217216, 47280846225407, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846225408, 47280846233599, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846045183, ++STORE, 47280846045184, 47280846053375, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846221311, ++STORE, 47280846221312, 47280846225407, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846184447, ++STORE, 47280846184448, 47280846188543, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844206079, ++STORE, 47280844206080, 47280844214271, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841175039, ++STORE, 47280841175040, 47280841179135, ++ERASE, 94410362142720, 94410362142720, ++STORE, 94410362142720, 94410362150911, ++STORE, 94410362150912, 94410362155007, ++ERASE, 140351954157568, 140351954157568, ++STORE, 140351954157568, 140351954161663, ++STORE, 140351954161664, 140351954165759, ++ERASE, 47280840998912, 47280840998912, ++STORE, 94410379456512, 94410379591679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732946362368, 140737488351231, ++ERASE, 140732946362368, 140732946362368, ++STORE, 140732946362368, 140732946366463, ++STORE, 94352937934848, 94352938106879, ++ERASE, 94352937934848, 94352937934848, ++STORE, 94352937934848, 94352937951231, ++STORE, 94352937951232, 94352938106879, ++ERASE, 94352937951232, 94352937951232, ++STORE, 94352937951232, 94352938053631, ++STORE, 94352938053632, 94352938094591, ++STORE, 94352938094592, 94352938106879, ++STORE, 140595518742528, 140595518914559, ++ERASE, 140595518742528, 140595518742528, ++STORE, 140595518742528, 140595518746623, ++STORE, 140595518746624, 140595518914559, ++ERASE, 140595518746624, 140595518746624, ++STORE, 140595518746624, 140595518869503, ++STORE, 140595518869504, 140595518902271, ++STORE, 140595518902272, 140595518910463, ++STORE, 140595518910464, 140595518914559, ++STORE, 140732947468288, 140732947472383, ++STORE, 140732947456000, 140732947468287, ++STORE, 47037276254208, 47037276262399, ++STORE, 47037276262400, 47037276270591, ++STORE, 47037276270592, 47037276434431, ++ERASE, 47037276270592, 47037276270592, ++STORE, 47037276270592, 47037276282879, ++STORE, 47037276282880, 47037276434431, ++STORE, 47037276381184, 47037276434431, ++STORE, 47037276282880, 47037276381183, ++ERASE, 47037276282880, 47037276282880, ++STORE, 47037276282880, 47037276381183, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276381184, 47037276426239, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276434431, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276426239, ++STORE, 47037276426240, 47037276434431, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276434432, 47037279485951, ++STORE, 47037276979200, 47037279485951, ++STORE, 47037276434432, 47037276979199, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037279264767, ++STORE, 47037279264768, 47037279485951, ++STORE, 47037278674944, 47037279264767, ++STORE, 47037276979200, 47037278674943, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037278674943, ++STORE, 47037279260672, 47037279264767, ++STORE, 47037278674944, 47037279260671, ++ERASE, 47037278674944, 47037278674944, ++STORE, 47037278674944, 47037279260671, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279469568, 47037279469568, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279485952, 47037281325055, ++STORE, 47037279625216, 47037281325055, ++STORE, 47037279485952, 47037279625215, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037281284095, ++STORE, 47037281284096, 47037281325055, ++STORE, 47037280968704, 47037281284095, ++STORE, 47037279625216, 47037280968703, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037280968703, ++STORE, 47037281280000, 47037281284095, ++STORE, 47037280968704, 47037281279999, ++ERASE, 47037280968704, 47037280968704, ++STORE, 47037280968704, 47037281279999, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281308672, 47037281308672, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281325056, 47037281460223, ++ERASE, 47037281325056, 47037281325056, ++STORE, 47037281325056, 47037281349631, ++STORE, 47037281349632, 47037281460223, ++STORE, 47037281411072, 47037281460223, ++STORE, 47037281349632, 47037281411071, ++ERASE, 47037281349632, 47037281349632, ++STORE, 47037281349632, 47037281411071, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281411072, 47037281435647, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281460223, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281435647, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281443840, 47037281443840, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281460224, 47037281480703, ++ERASE, 47037281460224, 47037281460224, ++STORE, 47037281460224, 47037281464319, ++STORE, 47037281464320, 47037281480703, ++STORE, 47037281468416, 47037281480703, ++STORE, 47037281464320, 47037281468415, ++ERASE, 47037281464320, 47037281464320, ++STORE, 47037281464320, 47037281468415, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281468416, 47037281472511, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281480703, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281472511, ++STORE, 47037281472512, 47037281480703, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281480704, 47037281488895, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281300479, ++STORE, 47037281300480, 47037281308671, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281476607, ++STORE, 47037281476608, 47037281480703, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281439743, ++STORE, 47037281439744, 47037281443839, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279461375, ++STORE, 47037279461376, 47037279469567, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276430335, ++STORE, 47037276430336, 47037276434431, ++ERASE, 94352938094592, 94352938094592, ++STORE, 94352938094592, 94352938102783, ++STORE, 94352938102784, 94352938106879, ++ERASE, 140595518902272, 140595518902272, ++STORE, 140595518902272, 140595518906367, ++STORE, 140595518906368, 140595518910463, ++ERASE, 47037276254208, 47037276254208, ++STORE, 94352938438656, 94352938573823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733506027520, 140737488351231, ++ERASE, 140733506027520, 140733506027520, ++STORE, 140733506027520, 140733506031615, ++STORE, 94150123073536, 94150123245567, ++ERASE, 94150123073536, 94150123073536, ++STORE, 94150123073536, 94150123089919, ++STORE, 94150123089920, 94150123245567, ++ERASE, 94150123089920, 94150123089920, ++STORE, 94150123089920, 94150123192319, ++STORE, 94150123192320, 94150123233279, ++STORE, 94150123233280, 94150123245567, ++STORE, 140081290375168, 140081290547199, ++ERASE, 140081290375168, 140081290375168, ++STORE, 140081290375168, 140081290379263, ++STORE, 140081290379264, 140081290547199, ++ERASE, 140081290379264, 140081290379264, ++STORE, 140081290379264, 140081290502143, ++STORE, 140081290502144, 140081290534911, ++STORE, 140081290534912, 140081290543103, ++STORE, 140081290543104, 140081290547199, ++STORE, 140733506707456, 140733506711551, ++STORE, 140733506695168, 140733506707455, ++STORE, 47551504621568, 47551504629759, ++STORE, 47551504629760, 47551504637951, ++STORE, 47551504637952, 47551504801791, ++ERASE, 47551504637952, 47551504637952, ++STORE, 47551504637952, 47551504650239, ++STORE, 47551504650240, 47551504801791, ++STORE, 47551504748544, 47551504801791, ++STORE, 47551504650240, 47551504748543, ++ERASE, 47551504650240, 47551504650240, ++STORE, 47551504650240, 47551504748543, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504748544, 47551504793599, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504801791, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504793599, ++STORE, 47551504793600, 47551504801791, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504801792, 47551507853311, ++STORE, 47551505346560, 47551507853311, ++STORE, 47551504801792, 47551505346559, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507632127, ++STORE, 47551507632128, 47551507853311, ++STORE, 47551507042304, 47551507632127, ++STORE, 47551505346560, 47551507042303, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507042303, ++STORE, 47551507628032, 47551507632127, ++STORE, 47551507042304, 47551507628031, ++ERASE, 47551507042304, 47551507042304, ++STORE, 47551507042304, 47551507628031, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507836928, 47551507836928, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507853312, 47551509692415, ++STORE, 47551507992576, 47551509692415, ++STORE, 47551507853312, 47551507992575, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509651455, ++STORE, 47551509651456, 47551509692415, ++STORE, 47551509336064, 47551509651455, ++STORE, 47551507992576, 47551509336063, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509336063, ++STORE, 47551509647360, 47551509651455, ++STORE, 47551509336064, 47551509647359, ++ERASE, 47551509336064, 47551509336064, ++STORE, 47551509336064, 47551509647359, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509676032, 47551509676032, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509692416, 47551509827583, ++ERASE, 47551509692416, 47551509692416, ++STORE, 47551509692416, 47551509716991, ++STORE, 47551509716992, 47551509827583, ++STORE, 47551509778432, 47551509827583, ++STORE, 47551509716992, 47551509778431, ++ERASE, 47551509716992, 47551509716992, ++STORE, 47551509716992, 47551509778431, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509778432, 47551509803007, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509827583, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509803007, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509811200, 47551509811200, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509827584, 47551509848063, ++ERASE, 47551509827584, 47551509827584, ++STORE, 47551509827584, 47551509831679, ++STORE, 47551509831680, 47551509848063, ++STORE, 47551509835776, 47551509848063, ++STORE, 47551509831680, 47551509835775, ++ERASE, 47551509831680, 47551509831680, ++STORE, 47551509831680, 47551509835775, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509835776, 47551509839871, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509848063, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509839871, ++STORE, 47551509839872, 47551509848063, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509848064, 47551509856255, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509667839, ++STORE, 47551509667840, 47551509676031, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509843967, ++STORE, 47551509843968, 47551509848063, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509807103, ++STORE, 47551509807104, 47551509811199, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507828735, ++STORE, 47551507828736, 47551507836927, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504797695, ++STORE, 47551504797696, 47551504801791, ++ERASE, 94150123233280, 94150123233280, ++STORE, 94150123233280, 94150123241471, ++STORE, 94150123241472, 94150123245567, ++ERASE, 140081290534912, 140081290534912, ++STORE, 140081290534912, 140081290539007, ++STORE, 140081290539008, 140081290543103, ++ERASE, 47551504621568, 47551504621568, ++STORE, 94150148112384, 94150148247551, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734389334016, 140737488351231, ++ERASE, 140734389334016, 140734389334016, ++STORE, 140734389334016, 140734389338111, ++STORE, 94844636606464, 94844636778495, ++ERASE, 94844636606464, 94844636606464, ++STORE, 94844636606464, 94844636622847, ++STORE, 94844636622848, 94844636778495, ++ERASE, 94844636622848, 94844636622848, ++STORE, 94844636622848, 94844636725247, ++STORE, 94844636725248, 94844636766207, ++STORE, 94844636766208, 94844636778495, ++STORE, 139922765217792, 139922765389823, ++ERASE, 139922765217792, 139922765217792, ++STORE, 139922765217792, 139922765221887, ++STORE, 139922765221888, 139922765389823, ++ERASE, 139922765221888, 139922765221888, ++STORE, 139922765221888, 139922765344767, ++STORE, 139922765344768, 139922765377535, ++STORE, 139922765377536, 139922765385727, ++STORE, 139922765385728, 139922765389823, ++STORE, 140734389678080, 140734389682175, ++STORE, 140734389665792, 140734389678079, ++STORE, 47710029778944, 47710029787135, ++STORE, 47710029787136, 47710029795327, ++STORE, 47710029795328, 47710029959167, ++ERASE, 47710029795328, 47710029795328, ++STORE, 47710029795328, 47710029807615, ++STORE, 47710029807616, 47710029959167, ++STORE, 47710029905920, 47710029959167, ++STORE, 47710029807616, 47710029905919, ++ERASE, 47710029807616, 47710029807616, ++STORE, 47710029807616, 47710029905919, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029905920, 47710029950975, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029959167, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029950975, ++STORE, 47710029950976, 47710029959167, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029959168, 47710033010687, ++STORE, 47710030503936, 47710033010687, ++STORE, 47710029959168, 47710030503935, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032789503, ++STORE, 47710032789504, 47710033010687, ++STORE, 47710032199680, 47710032789503, ++STORE, 47710030503936, 47710032199679, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032199679, ++STORE, 47710032785408, 47710032789503, ++STORE, 47710032199680, 47710032785407, ++ERASE, 47710032199680, 47710032199680, ++STORE, 47710032199680, 47710032785407, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032994304, 47710032994304, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710033010688, 47710034849791, ++STORE, 47710033149952, 47710034849791, ++STORE, 47710033010688, 47710033149951, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034808831, ++STORE, 47710034808832, 47710034849791, ++STORE, 47710034493440, 47710034808831, ++STORE, 47710033149952, 47710034493439, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034493439, ++STORE, 47710034804736, 47710034808831, ++STORE, 47710034493440, 47710034804735, ++ERASE, 47710034493440, 47710034493440, ++STORE, 47710034493440, 47710034804735, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034833408, 47710034833408, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034849792, 47710034984959, ++ERASE, 47710034849792, 47710034849792, ++STORE, 47710034849792, 47710034874367, ++STORE, 47710034874368, 47710034984959, ++STORE, 47710034935808, 47710034984959, ++STORE, 47710034874368, 47710034935807, ++ERASE, 47710034874368, 47710034874368, ++STORE, 47710034874368, 47710034935807, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034935808, 47710034960383, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034984959, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034960383, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034968576, 47710034968576, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034984960, 47710035005439, ++ERASE, 47710034984960, 47710034984960, ++STORE, 47710034984960, 47710034989055, ++STORE, 47710034989056, 47710035005439, ++STORE, 47710034993152, 47710035005439, ++STORE, 47710034989056, 47710034993151, ++ERASE, 47710034989056, 47710034989056, ++STORE, 47710034989056, 47710034993151, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710034993152, 47710034997247, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710035005439, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710034997247, ++STORE, 47710034997248, 47710035005439, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710035005440, 47710035013631, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034825215, ++STORE, 47710034825216, 47710034833407, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035001343, ++STORE, 47710035001344, 47710035005439, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034964479, ++STORE, 47710034964480, 47710034968575, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032986111, ++STORE, 47710032986112, 47710032994303, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029955071, ++STORE, 47710029955072, 47710029959167, ++ERASE, 94844636766208, 94844636766208, ++STORE, 94844636766208, 94844636774399, ++STORE, 94844636774400, 94844636778495, ++ERASE, 139922765377536, 139922765377536, ++STORE, 139922765377536, 139922765381631, ++STORE, 139922765381632, 139922765385727, ++ERASE, 47710029778944, 47710029778944, ++STORE, 94844641775616, 94844641910783, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732213886976, 140737488351231, ++ERASE, 140732213886976, 140732213886976, ++STORE, 140732213886976, 140732213891071, ++STORE, 94240508887040, 94240509059071, ++ERASE, 94240508887040, 94240508887040, ++STORE, 94240508887040, 94240508903423, ++STORE, 94240508903424, 94240509059071, ++ERASE, 94240508903424, 94240508903424, ++STORE, 94240508903424, 94240509005823, ++STORE, 94240509005824, 94240509046783, ++STORE, 94240509046784, 94240509059071, ++STORE, 140275106516992, 140275106689023, ++ERASE, 140275106516992, 140275106516992, ++STORE, 140275106516992, 140275106521087, ++STORE, 140275106521088, 140275106689023, ++ERASE, 140275106521088, 140275106521088, ++STORE, 140275106521088, 140275106643967, ++STORE, 140275106643968, 140275106676735, ++STORE, 140275106676736, 140275106684927, ++STORE, 140275106684928, 140275106689023, ++STORE, 140732213977088, 140732213981183, ++STORE, 140732213964800, 140732213977087, ++STORE, 47357688479744, 47357688487935, ++STORE, 47357688487936, 47357688496127, ++STORE, 47357688496128, 47357688659967, ++ERASE, 47357688496128, 47357688496128, ++STORE, 47357688496128, 47357688508415, ++STORE, 47357688508416, 47357688659967, ++STORE, 47357688606720, 47357688659967, ++STORE, 47357688508416, 47357688606719, ++ERASE, 47357688508416, 47357688508416, ++STORE, 47357688508416, 47357688606719, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688606720, 47357688651775, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688659967, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688651775, ++STORE, 47357688651776, 47357688659967, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688659968, 47357691711487, ++STORE, 47357689204736, 47357691711487, ++STORE, 47357688659968, 47357689204735, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357691490303, ++STORE, 47357691490304, 47357691711487, ++STORE, 47357690900480, 47357691490303, ++STORE, 47357689204736, 47357690900479, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357690900479, ++STORE, 47357691486208, 47357691490303, ++STORE, 47357690900480, 47357691486207, ++ERASE, 47357690900480, 47357690900480, ++STORE, 47357690900480, 47357691486207, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691695104, 47357691695104, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691711488, 47357693550591, ++STORE, 47357691850752, 47357693550591, ++STORE, 47357691711488, 47357691850751, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693509631, ++STORE, 47357693509632, 47357693550591, ++STORE, 47357693194240, 47357693509631, ++STORE, 47357691850752, 47357693194239, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693194239, ++STORE, 47357693505536, 47357693509631, ++STORE, 47357693194240, 47357693505535, ++ERASE, 47357693194240, 47357693194240, ++STORE, 47357693194240, 47357693505535, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693534208, 47357693534208, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693550592, 47357693685759, ++ERASE, 47357693550592, 47357693550592, ++STORE, 47357693550592, 47357693575167, ++STORE, 47357693575168, 47357693685759, ++STORE, 47357693636608, 47357693685759, ++STORE, 47357693575168, 47357693636607, ++ERASE, 47357693575168, 47357693575168, ++STORE, 47357693575168, 47357693636607, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693636608, 47357693661183, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693685759, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693661183, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693669376, 47357693669376, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693685760, 47357693706239, ++ERASE, 47357693685760, 47357693685760, ++STORE, 47357693685760, 47357693689855, ++STORE, 47357693689856, 47357693706239, ++STORE, 47357693693952, 47357693706239, ++STORE, 47357693689856, 47357693693951, ++ERASE, 47357693689856, 47357693689856, ++STORE, 47357693689856, 47357693693951, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693693952, 47357693698047, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693706239, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693698047, ++STORE, 47357693698048, 47357693706239, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693706240, 47357693714431, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693526015, ++STORE, 47357693526016, 47357693534207, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693702143, ++STORE, 47357693702144, 47357693706239, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693665279, ++STORE, 47357693665280, 47357693669375, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691686911, ++STORE, 47357691686912, 47357691695103, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688655871, ++STORE, 47357688655872, 47357688659967, ++ERASE, 94240509046784, 94240509046784, ++STORE, 94240509046784, 94240509054975, ++STORE, 94240509054976, 94240509059071, ++ERASE, 140275106676736, 140275106676736, ++STORE, 140275106676736, 140275106680831, ++STORE, 140275106680832, 140275106684927, ++ERASE, 47357688479744, 47357688479744, ++STORE, 94240518361088, 94240518496255, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732688277504, 140737488351231, ++ERASE, 140732688277504, 140732688277504, ++STORE, 140732688277504, 140732688281599, ++STORE, 94629171351552, 94629172064255, ++ERASE, 94629171351552, 94629171351552, ++STORE, 94629171351552, 94629171400703, ++STORE, 94629171400704, 94629172064255, ++ERASE, 94629171400704, 94629171400704, ++STORE, 94629171400704, 94629171945471, ++STORE, 94629171945472, 94629172043775, ++STORE, 94629172043776, 94629172064255, ++STORE, 139770707644416, 139770707816447, ++ERASE, 139770707644416, 139770707644416, ++STORE, 139770707644416, 139770707648511, ++STORE, 139770707648512, 139770707816447, ++ERASE, 139770707648512, 139770707648512, ++STORE, 139770707648512, 139770707771391, ++STORE, 139770707771392, 139770707804159, ++STORE, 139770707804160, 139770707812351, ++STORE, 139770707812352, 139770707816447, ++STORE, 140732689121280, 140732689125375, ++STORE, 140732689108992, 140732689121279, ++STORE, 47862087352320, 47862087360511, ++STORE, 47862087360512, 47862087368703, ++STORE, 47862087368704, 47862087475199, ++STORE, 47862087385088, 47862087475199, ++STORE, 47862087368704, 47862087385087, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087458815, ++STORE, 47862087458816, 47862087475199, ++STORE, 47862087438336, 47862087458815, ++STORE, 47862087385088, 47862087438335, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087438335, ++STORE, 47862087454720, 47862087458815, ++STORE, 47862087438336, 47862087454719, ++ERASE, 47862087438336, 47862087438336, ++STORE, 47862087438336, 47862087454719, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087467008, 47862087467008, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087475200, 47862089314303, ++STORE, 47862087614464, 47862089314303, ++STORE, 47862087475200, 47862087614463, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862089273343, ++STORE, 47862089273344, 47862089314303, ++STORE, 47862088957952, 47862089273343, ++STORE, 47862087614464, 47862088957951, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862088957951, ++STORE, 47862089269248, 47862089273343, ++STORE, 47862088957952, 47862089269247, ++ERASE, 47862088957952, 47862088957952, ++STORE, 47862088957952, 47862089269247, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089297920, 47862089297920, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089297920, 47862089326591, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089289727, ++STORE, 47862089289728, 47862089297919, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087462911, ++STORE, 47862087462912, 47862087467007, ++ERASE, 94629172043776, 94629172043776, ++STORE, 94629172043776, 94629172060159, ++STORE, 94629172060160, 94629172064255, ++ERASE, 139770707804160, 139770707804160, ++STORE, 139770707804160, 139770707808255, ++STORE, 139770707808256, 139770707812351, ++ERASE, 47862087352320, 47862087352320, ++STORE, 94629197533184, 94629197668351, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727540711424, 140737488351231, ++ERASE, 140727540711424, 140727540711424, ++STORE, 140727540711424, 140727540715519, ++STORE, 94299865313280, 94299866025983, ++ERASE, 94299865313280, 94299865313280, ++STORE, 94299865313280, 94299865362431, ++STORE, 94299865362432, 94299866025983, ++ERASE, 94299865362432, 94299865362432, ++STORE, 94299865362432, 94299865907199, ++STORE, 94299865907200, 94299866005503, ++STORE, 94299866005504, 94299866025983, ++STORE, 140680268763136, 140680268935167, ++ERASE, 140680268763136, 140680268763136, ++STORE, 140680268763136, 140680268767231, ++STORE, 140680268767232, 140680268935167, ++ERASE, 140680268767232, 140680268767232, ++STORE, 140680268767232, 140680268890111, ++STORE, 140680268890112, 140680268922879, ++STORE, 140680268922880, 140680268931071, ++STORE, 140680268931072, 140680268935167, ++STORE, 140727541424128, 140727541428223, ++STORE, 140727541411840, 140727541424127, ++STORE, 46952526233600, 46952526241791, ++STORE, 46952526241792, 46952526249983, ++STORE, 46952526249984, 46952526356479, ++STORE, 46952526266368, 46952526356479, ++STORE, 46952526249984, 46952526266367, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526340095, ++STORE, 46952526340096, 46952526356479, ++STORE, 46952526319616, 46952526340095, ++STORE, 46952526266368, 46952526319615, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526319615, ++STORE, 46952526336000, 46952526340095, ++STORE, 46952526319616, 46952526335999, ++ERASE, 46952526319616, 46952526319616, ++STORE, 46952526319616, 46952526335999, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526348288, 46952526348288, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526356480, 46952528195583, ++STORE, 46952526495744, 46952528195583, ++STORE, 46952526356480, 46952526495743, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952528154623, ++STORE, 46952528154624, 46952528195583, ++STORE, 46952527839232, 46952528154623, ++STORE, 46952526495744, 46952527839231, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952527839231, ++STORE, 46952528150528, 46952528154623, ++STORE, 46952527839232, 46952528150527, ++ERASE, 46952527839232, 46952527839232, ++STORE, 46952527839232, 46952528150527, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528179200, 46952528179200, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528179200, 46952528207871, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528171007, ++STORE, 46952528171008, 46952528179199, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526344191, ++STORE, 46952526344192, 46952526348287, ++ERASE, 94299866005504, 94299866005504, ++STORE, 94299866005504, 94299866021887, ++STORE, 94299866021888, 94299866025983, ++ERASE, 140680268922880, 140680268922880, ++STORE, 140680268922880, 140680268926975, ++STORE, 140680268926976, 140680268931071, ++ERASE, 46952526233600, 46952526233600, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722874793984, 140737488351231, ++ERASE, 140722874793984, 140722874793984, ++STORE, 140722874793984, 140722874798079, ++STORE, 94448916213760, 94448916926463, ++ERASE, 94448916213760, 94448916213760, ++STORE, 94448916213760, 94448916262911, ++STORE, 94448916262912, 94448916926463, ++ERASE, 94448916262912, 94448916262912, ++STORE, 94448916262912, 94448916807679, ++STORE, 94448916807680, 94448916905983, ++STORE, 94448916905984, 94448916926463, ++STORE, 140389117046784, 140389117218815, ++ERASE, 140389117046784, 140389117046784, ++STORE, 140389117046784, 140389117050879, ++STORE, 140389117050880, 140389117218815, ++ERASE, 140389117050880, 140389117050880, ++STORE, 140389117050880, 140389117173759, ++STORE, 140389117173760, 140389117206527, ++STORE, 140389117206528, 140389117214719, ++STORE, 140389117214720, 140389117218815, ++STORE, 140722875297792, 140722875301887, ++STORE, 140722875285504, 140722875297791, ++STORE, 47243677949952, 47243677958143, ++STORE, 47243677958144, 47243677966335, ++STORE, 47243677966336, 47243678072831, ++STORE, 47243677982720, 47243678072831, ++STORE, 47243677966336, 47243677982719, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678056447, ++STORE, 47243678056448, 47243678072831, ++STORE, 47243678035968, 47243678056447, ++STORE, 47243677982720, 47243678035967, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678035967, ++STORE, 47243678052352, 47243678056447, ++STORE, 47243678035968, 47243678052351, ++ERASE, 47243678035968, 47243678035968, ++STORE, 47243678035968, 47243678052351, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678064640, 47243678064640, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678072832, 47243679911935, ++STORE, 47243678212096, 47243679911935, ++STORE, 47243678072832, 47243678212095, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679870975, ++STORE, 47243679870976, 47243679911935, ++STORE, 47243679555584, 47243679870975, ++STORE, 47243678212096, 47243679555583, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679555583, ++STORE, 47243679866880, 47243679870975, ++STORE, 47243679555584, 47243679866879, ++ERASE, 47243679555584, 47243679555584, ++STORE, 47243679555584, 47243679866879, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679895552, 47243679895552, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679895552, 47243679924223, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679887359, ++STORE, 47243679887360, 47243679895551, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678060543, ++STORE, 47243678060544, 47243678064639, ++ERASE, 94448916905984, 94448916905984, ++STORE, 94448916905984, 94448916922367, ++STORE, 94448916922368, 94448916926463, ++ERASE, 140389117206528, 140389117206528, ++STORE, 140389117206528, 140389117210623, ++STORE, 140389117210624, 140389117214719, ++ERASE, 47243677949952, 47243677949952, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733068505088, 140737488351231, ++ERASE, 140733068505088, 140733068505088, ++STORE, 140733068505088, 140733068509183, ++STORE, 94207145750528, 94207146463231, ++ERASE, 94207145750528, 94207145750528, ++STORE, 94207145750528, 94207145799679, ++STORE, 94207145799680, 94207146463231, ++ERASE, 94207145799680, 94207145799680, ++STORE, 94207145799680, 94207146344447, ++STORE, 94207146344448, 94207146442751, ++STORE, 94207146442752, 94207146463231, ++STORE, 140684504911872, 140684505083903, ++ERASE, 140684504911872, 140684504911872, ++STORE, 140684504911872, 140684504915967, ++STORE, 140684504915968, 140684505083903, ++ERASE, 140684504915968, 140684504915968, ++STORE, 140684504915968, 140684505038847, ++STORE, 140684505038848, 140684505071615, ++STORE, 140684505071616, 140684505079807, ++STORE, 140684505079808, 140684505083903, ++STORE, 140733068607488, 140733068611583, ++STORE, 140733068595200, 140733068607487, ++STORE, 46948290084864, 46948290093055, ++STORE, 46948290093056, 46948290101247, ++STORE, 46948290101248, 46948290207743, ++STORE, 46948290117632, 46948290207743, ++STORE, 46948290101248, 46948290117631, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290191359, ++STORE, 46948290191360, 46948290207743, ++STORE, 46948290170880, 46948290191359, ++STORE, 46948290117632, 46948290170879, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290170879, ++STORE, 46948290187264, 46948290191359, ++STORE, 46948290170880, 46948290187263, ++ERASE, 46948290170880, 46948290170880, ++STORE, 46948290170880, 46948290187263, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290199552, 46948290199552, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290207744, 46948292046847, ++STORE, 46948290347008, 46948292046847, ++STORE, 46948290207744, 46948290347007, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948292005887, ++STORE, 46948292005888, 46948292046847, ++STORE, 46948291690496, 46948292005887, ++STORE, 46948290347008, 46948291690495, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948291690495, ++STORE, 46948292001792, 46948292005887, ++STORE, 46948291690496, 46948292001791, ++ERASE, 46948291690496, 46948291690496, ++STORE, 46948291690496, 46948292001791, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292030464, 46948292030464, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292030464, 46948292059135, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292022271, ++STORE, 46948292022272, 46948292030463, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290195455, ++STORE, 46948290195456, 46948290199551, ++ERASE, 94207146442752, 94207146442752, ++STORE, 94207146442752, 94207146459135, ++STORE, 94207146459136, 94207146463231, ++ERASE, 140684505071616, 140684505071616, ++STORE, 140684505071616, 140684505075711, ++STORE, 140684505075712, 140684505079807, ++ERASE, 46948290084864, 46948290084864, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726367158272, 140737488351231, ++ERASE, 140726367158272, 140726367158272, ++STORE, 140726367158272, 140726367162367, ++STORE, 94436124106752, 94436124819455, ++ERASE, 94436124106752, 94436124106752, ++STORE, 94436124106752, 94436124155903, ++STORE, 94436124155904, 94436124819455, ++ERASE, 94436124155904, 94436124155904, ++STORE, 94436124155904, 94436124700671, ++STORE, 94436124700672, 94436124798975, ++STORE, 94436124798976, 94436124819455, ++STORE, 140049025044480, 140049025216511, ++ERASE, 140049025044480, 140049025044480, ++STORE, 140049025044480, 140049025048575, ++STORE, 140049025048576, 140049025216511, ++ERASE, 140049025048576, 140049025048576, ++STORE, 140049025048576, 140049025171455, ++STORE, 140049025171456, 140049025204223, ++STORE, 140049025204224, 140049025212415, ++STORE, 140049025212416, 140049025216511, ++STORE, 140726367256576, 140726367260671, ++STORE, 140726367244288, 140726367256575, ++STORE, 47583769952256, 47583769960447, ++STORE, 47583769960448, 47583769968639, ++STORE, 47583769968640, 47583770075135, ++STORE, 47583769985024, 47583770075135, ++STORE, 47583769968640, 47583769985023, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770058751, ++STORE, 47583770058752, 47583770075135, ++STORE, 47583770038272, 47583770058751, ++STORE, 47583769985024, 47583770038271, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770038271, ++STORE, 47583770054656, 47583770058751, ++STORE, 47583770038272, 47583770054655, ++ERASE, 47583770038272, 47583770038272, ++STORE, 47583770038272, 47583770054655, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770066944, 47583770066944, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770075136, 47583771914239, ++STORE, 47583770214400, 47583771914239, ++STORE, 47583770075136, 47583770214399, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771873279, ++STORE, 47583771873280, 47583771914239, ++STORE, 47583771557888, 47583771873279, ++STORE, 47583770214400, 47583771557887, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771557887, ++STORE, 47583771869184, 47583771873279, ++STORE, 47583771557888, 47583771869183, ++ERASE, 47583771557888, 47583771557888, ++STORE, 47583771557888, 47583771869183, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771897856, 47583771897856, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771897856, 47583771926527, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771889663, ++STORE, 47583771889664, 47583771897855, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770062847, ++STORE, 47583770062848, 47583770066943, ++ERASE, 94436124798976, 94436124798976, ++STORE, 94436124798976, 94436124815359, ++STORE, 94436124815360, 94436124819455, ++ERASE, 140049025204224, 140049025204224, ++STORE, 140049025204224, 140049025208319, ++STORE, 140049025208320, 140049025212415, ++ERASE, 47583769952256, 47583769952256, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727116099584, 140737488351231, ++ERASE, 140727116099584, 140727116099584, ++STORE, 140727116099584, 140727116103679, ++STORE, 94166319734784, 94166320447487, ++ERASE, 94166319734784, 94166319734784, ++STORE, 94166319734784, 94166319783935, ++STORE, 94166319783936, 94166320447487, ++ERASE, 94166319783936, 94166319783936, ++STORE, 94166319783936, 94166320328703, ++STORE, 94166320328704, 94166320427007, ++STORE, 94166320427008, 94166320447487, ++STORE, 139976559542272, 139976559714303, ++ERASE, 139976559542272, 139976559542272, ++STORE, 139976559542272, 139976559546367, ++STORE, 139976559546368, 139976559714303, ++ERASE, 139976559546368, 139976559546368, ++STORE, 139976559546368, 139976559669247, ++STORE, 139976559669248, 139976559702015, ++STORE, 139976559702016, 139976559710207, ++STORE, 139976559710208, 139976559714303, ++STORE, 140727116222464, 140727116226559, ++STORE, 140727116210176, 140727116222463, ++STORE, 47656235454464, 47656235462655, ++STORE, 47656235462656, 47656235470847, ++STORE, 47656235470848, 47656235577343, ++STORE, 47656235487232, 47656235577343, ++STORE, 47656235470848, 47656235487231, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235560959, ++STORE, 47656235560960, 47656235577343, ++STORE, 47656235540480, 47656235560959, ++STORE, 47656235487232, 47656235540479, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235540479, ++STORE, 47656235556864, 47656235560959, ++STORE, 47656235540480, 47656235556863, ++ERASE, 47656235540480, 47656235540480, ++STORE, 47656235540480, 47656235556863, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235569152, 47656235569152, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235577344, 47656237416447, ++STORE, 47656235716608, 47656237416447, ++STORE, 47656235577344, 47656235716607, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237375487, ++STORE, 47656237375488, 47656237416447, ++STORE, 47656237060096, 47656237375487, ++STORE, 47656235716608, 47656237060095, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237060095, ++STORE, 47656237371392, 47656237375487, ++STORE, 47656237060096, 47656237371391, ++ERASE, 47656237060096, 47656237060096, ++STORE, 47656237060096, 47656237371391, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237400064, 47656237400064, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237400064, 47656237428735, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237391871, ++STORE, 47656237391872, 47656237400063, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235565055, ++STORE, 47656235565056, 47656235569151, ++ERASE, 94166320427008, 94166320427008, ++STORE, 94166320427008, 94166320443391, ++STORE, 94166320443392, 94166320447487, ++ERASE, 139976559702016, 139976559702016, ++STORE, 139976559702016, 139976559706111, ++STORE, 139976559706112, 139976559710207, ++ERASE, 47656235454464, 47656235454464, ++STORE, 94166332153856, 94166332289023, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726412816384, 140737488351231, ++ERASE, 140726412816384, 140726412816384, ++STORE, 140726412816384, 140726412820479, ++STORE, 94094884507648, 94094885220351, ++ERASE, 94094884507648, 94094884507648, ++STORE, 94094884507648, 94094884556799, ++STORE, 94094884556800, 94094885220351, ++ERASE, 94094884556800, 94094884556800, ++STORE, 94094884556800, 94094885101567, ++STORE, 94094885101568, 94094885199871, ++STORE, 94094885199872, 94094885220351, ++STORE, 139773773938688, 139773774110719, ++ERASE, 139773773938688, 139773773938688, ++STORE, 139773773938688, 139773773942783, ++STORE, 139773773942784, 139773774110719, ++ERASE, 139773773942784, 139773773942784, ++STORE, 139773773942784, 139773774065663, ++STORE, 139773774065664, 139773774098431, ++STORE, 139773774098432, 139773774106623, ++STORE, 139773774106624, 139773774110719, ++STORE, 140726412963840, 140726412967935, ++STORE, 140726412951552, 140726412963839, ++STORE, 47859021058048, 47859021066239, ++STORE, 47859021066240, 47859021074431, ++STORE, 47859021074432, 47859021180927, ++STORE, 47859021090816, 47859021180927, ++STORE, 47859021074432, 47859021090815, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021164543, ++STORE, 47859021164544, 47859021180927, ++STORE, 47859021144064, 47859021164543, ++STORE, 47859021090816, 47859021144063, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021144063, ++STORE, 47859021160448, 47859021164543, ++STORE, 47859021144064, 47859021160447, ++ERASE, 47859021144064, 47859021144064, ++STORE, 47859021144064, 47859021160447, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021172736, 47859021172736, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021180928, 47859023020031, ++STORE, 47859021320192, 47859023020031, ++STORE, 47859021180928, 47859021320191, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022979071, ++STORE, 47859022979072, 47859023020031, ++STORE, 47859022663680, 47859022979071, ++STORE, 47859021320192, 47859022663679, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022663679, ++STORE, 47859022974976, 47859022979071, ++STORE, 47859022663680, 47859022974975, ++ERASE, 47859022663680, 47859022663680, ++STORE, 47859022663680, 47859022974975, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859023003648, 47859023003648, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859023003648, 47859023032319, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859022995455, ++STORE, 47859022995456, 47859023003647, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021168639, ++STORE, 47859021168640, 47859021172735, ++ERASE, 94094885199872, 94094885199872, ++STORE, 94094885199872, 94094885216255, ++STORE, 94094885216256, 94094885220351, ++ERASE, 139773774098432, 139773774098432, ++STORE, 139773774098432, 139773774102527, ++STORE, 139773774102528, 139773774106623, ++ERASE, 47859021058048, 47859021058048, ++STORE, 94094901108736, 94094901243903, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736567963648, 140737488351231, ++ERASE, 140736567963648, 140736567963648, ++STORE, 140736567963648, 140736567967743, ++STORE, 94924425748480, 94924426461183, ++ERASE, 94924425748480, 94924425748480, ++STORE, 94924425748480, 94924425797631, ++STORE, 94924425797632, 94924426461183, ++ERASE, 94924425797632, 94924425797632, ++STORE, 94924425797632, 94924426342399, ++STORE, 94924426342400, 94924426440703, ++STORE, 94924426440704, 94924426461183, ++STORE, 140042126319616, 140042126491647, ++ERASE, 140042126319616, 140042126319616, ++STORE, 140042126319616, 140042126323711, ++STORE, 140042126323712, 140042126491647, ++ERASE, 140042126323712, 140042126323712, ++STORE, 140042126323712, 140042126446591, ++STORE, 140042126446592, 140042126479359, ++STORE, 140042126479360, 140042126487551, ++STORE, 140042126487552, 140042126491647, ++STORE, 140736568672256, 140736568676351, ++STORE, 140736568659968, 140736568672255, ++STORE, 47590668677120, 47590668685311, ++STORE, 47590668685312, 47590668693503, ++STORE, 47590668693504, 47590668799999, ++STORE, 47590668709888, 47590668799999, ++STORE, 47590668693504, 47590668709887, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668783615, ++STORE, 47590668783616, 47590668799999, ++STORE, 47590668763136, 47590668783615, ++STORE, 47590668709888, 47590668763135, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668763135, ++STORE, 47590668779520, 47590668783615, ++STORE, 47590668763136, 47590668779519, ++ERASE, 47590668763136, 47590668763136, ++STORE, 47590668763136, 47590668779519, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668791808, 47590668791808, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668800000, 47590670639103, ++STORE, 47590668939264, 47590670639103, ++STORE, 47590668800000, 47590668939263, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670598143, ++STORE, 47590670598144, 47590670639103, ++STORE, 47590670282752, 47590670598143, ++STORE, 47590668939264, 47590670282751, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670282751, ++STORE, 47590670594048, 47590670598143, ++STORE, 47590670282752, 47590670594047, ++ERASE, 47590670282752, 47590670282752, ++STORE, 47590670282752, 47590670594047, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670622720, 47590670622720, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670622720, 47590670651391, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670614527, ++STORE, 47590670614528, 47590670622719, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668787711, ++STORE, 47590668787712, 47590668791807, ++ERASE, 94924426440704, 94924426440704, ++STORE, 94924426440704, 94924426457087, ++STORE, 94924426457088, 94924426461183, ++ERASE, 140042126479360, 140042126479360, ++STORE, 140042126479360, 140042126483455, ++STORE, 140042126483456, 140042126487551, ++ERASE, 47590668677120, 47590668677120, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733281439744, 140737488351231, ++ERASE, 140733281439744, 140733281439744, ++STORE, 140733281439744, 140733281443839, ++STORE, 94490667069440, 94490667782143, ++ERASE, 94490667069440, 94490667069440, ++STORE, 94490667069440, 94490667118591, ++STORE, 94490667118592, 94490667782143, ++ERASE, 94490667118592, 94490667118592, ++STORE, 94490667118592, 94490667663359, ++STORE, 94490667663360, 94490667761663, ++STORE, 94490667761664, 94490667782143, ++STORE, 139878215118848, 139878215290879, ++ERASE, 139878215118848, 139878215118848, ++STORE, 139878215118848, 139878215122943, ++STORE, 139878215122944, 139878215290879, ++ERASE, 139878215122944, 139878215122944, ++STORE, 139878215122944, 139878215245823, ++STORE, 139878215245824, 139878215278591, ++STORE, 139878215278592, 139878215286783, ++STORE, 139878215286784, 139878215290879, ++STORE, 140733281464320, 140733281468415, ++STORE, 140733281452032, 140733281464319, ++STORE, 47754579877888, 47754579886079, ++STORE, 47754579886080, 47754579894271, ++STORE, 47754579894272, 47754580000767, ++STORE, 47754579910656, 47754580000767, ++STORE, 47754579894272, 47754579910655, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579984383, ++STORE, 47754579984384, 47754580000767, ++STORE, 47754579963904, 47754579984383, ++STORE, 47754579910656, 47754579963903, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579963903, ++STORE, 47754579980288, 47754579984383, ++STORE, 47754579963904, 47754579980287, ++ERASE, 47754579963904, 47754579963904, ++STORE, 47754579963904, 47754579980287, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579992576, 47754579992576, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754580000768, 47754581839871, ++STORE, 47754580140032, 47754581839871, ++STORE, 47754580000768, 47754580140031, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581798911, ++STORE, 47754581798912, 47754581839871, ++STORE, 47754581483520, 47754581798911, ++STORE, 47754580140032, 47754581483519, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581483519, ++STORE, 47754581794816, 47754581798911, ++STORE, 47754581483520, 47754581794815, ++ERASE, 47754581483520, 47754581483520, ++STORE, 47754581483520, 47754581794815, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581823488, 47754581823488, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581823488, 47754581852159, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581815295, ++STORE, 47754581815296, 47754581823487, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579988479, ++STORE, 47754579988480, 47754579992575, ++ERASE, 94490667761664, 94490667761664, ++STORE, 94490667761664, 94490667778047, ++STORE, 94490667778048, 94490667782143, ++ERASE, 139878215278592, 139878215278592, ++STORE, 139878215278592, 139878215282687, ++STORE, 139878215282688, 139878215286783, ++ERASE, 47754579877888, 47754579877888, ++STORE, 94490669649920, 94490669785087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140735382188032, 140737488351231, ++ERASE, 140735382188032, 140735382188032, ++STORE, 140735382188032, 140735382192127, ++STORE, 94150181302272, 94150182014975, ++ERASE, 94150181302272, 94150181302272, ++STORE, 94150181302272, 94150181351423, ++STORE, 94150181351424, 94150182014975, ++ERASE, 94150181351424, 94150181351424, ++STORE, 94150181351424, 94150181896191, ++STORE, 94150181896192, 94150181994495, ++STORE, 94150181994496, 94150182014975, ++STORE, 139679752458240, 139679752630271, ++ERASE, 139679752458240, 139679752458240, ++STORE, 139679752458240, 139679752462335, ++STORE, 139679752462336, 139679752630271, ++ERASE, 139679752462336, 139679752462336, ++STORE, 139679752462336, 139679752585215, ++STORE, 139679752585216, 139679752617983, ++STORE, 139679752617984, 139679752626175, ++STORE, 139679752626176, 139679752630271, ++STORE, 140735382536192, 140735382540287, ++STORE, 140735382523904, 140735382536191, ++STORE, 47953042538496, 47953042546687, ++STORE, 47953042546688, 47953042554879, ++STORE, 47953042554880, 47953042661375, ++STORE, 47953042571264, 47953042661375, ++STORE, 47953042554880, 47953042571263, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042644991, ++STORE, 47953042644992, 47953042661375, ++STORE, 47953042624512, 47953042644991, ++STORE, 47953042571264, 47953042624511, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042624511, ++STORE, 47953042640896, 47953042644991, ++STORE, 47953042624512, 47953042640895, ++ERASE, 47953042624512, 47953042624512, ++STORE, 47953042624512, 47953042640895, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042653184, 47953042653184, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042661376, 47953044500479, ++STORE, 47953042800640, 47953044500479, ++STORE, 47953042661376, 47953042800639, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044459519, ++STORE, 47953044459520, 47953044500479, ++STORE, 47953044144128, 47953044459519, ++STORE, 47953042800640, 47953044144127, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044144127, ++STORE, 47953044455424, 47953044459519, ++STORE, 47953044144128, 47953044455423, ++ERASE, 47953044144128, 47953044144128, ++STORE, 47953044144128, 47953044455423, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044484096, 47953044484096, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044484096, 47953044512767, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044475903, ++STORE, 47953044475904, 47953044484095, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042649087, ++STORE, 47953042649088, 47953042653183, ++ERASE, 94150181994496, 94150181994496, ++STORE, 94150181994496, 94150182010879, ++STORE, 94150182010880, 94150182014975, ++ERASE, 139679752617984, 139679752617984, ++STORE, 139679752617984, 139679752622079, ++STORE, 139679752622080, 139679752626175, ++ERASE, 47953042538496, 47953042538496, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737044123648, 140737488351231, ++ERASE, 140737044123648, 140737044123648, ++STORE, 140737044123648, 140737044127743, ++STORE, 94425324294144, 94425325006847, ++ERASE, 94425324294144, 94425324294144, ++STORE, 94425324294144, 94425324343295, ++STORE, 94425324343296, 94425325006847, ++ERASE, 94425324343296, 94425324343296, ++STORE, 94425324343296, 94425324888063, ++STORE, 94425324888064, 94425324986367, ++STORE, 94425324986368, 94425325006847, ++STORE, 140382015016960, 140382015188991, ++ERASE, 140382015016960, 140382015016960, ++STORE, 140382015016960, 140382015021055, ++STORE, 140382015021056, 140382015188991, ++ERASE, 140382015021056, 140382015021056, ++STORE, 140382015021056, 140382015143935, ++STORE, 140382015143936, 140382015176703, ++STORE, 140382015176704, 140382015184895, ++STORE, 140382015184896, 140382015188991, ++STORE, 140737045585920, 140737045590015, ++STORE, 140737045573632, 140737045585919, ++STORE, 47250779979776, 47250779987967, ++STORE, 47250779987968, 47250779996159, ++STORE, 47250779996160, 47250780102655, ++STORE, 47250780012544, 47250780102655, ++STORE, 47250779996160, 47250780012543, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780086271, ++STORE, 47250780086272, 47250780102655, ++STORE, 47250780065792, 47250780086271, ++STORE, 47250780012544, 47250780065791, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780065791, ++STORE, 47250780082176, 47250780086271, ++STORE, 47250780065792, 47250780082175, ++ERASE, 47250780065792, 47250780065792, ++STORE, 47250780065792, 47250780082175, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780094464, 47250780094464, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780102656, 47250781941759, ++STORE, 47250780241920, 47250781941759, ++STORE, 47250780102656, 47250780241919, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781900799, ++STORE, 47250781900800, 47250781941759, ++STORE, 47250781585408, 47250781900799, ++STORE, 47250780241920, 47250781585407, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781585407, ++STORE, 47250781896704, 47250781900799, ++STORE, 47250781585408, 47250781896703, ++ERASE, 47250781585408, 47250781585408, ++STORE, 47250781585408, 47250781896703, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781925376, 47250781925376, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781925376, 47250781954047, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781917183, ++STORE, 47250781917184, 47250781925375, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780090367, ++STORE, 47250780090368, 47250780094463, ++ERASE, 94425324986368, 94425324986368, ++STORE, 94425324986368, 94425325002751, ++STORE, 94425325002752, 94425325006847, ++ERASE, 140382015176704, 140382015176704, ++STORE, 140382015176704, 140382015180799, ++STORE, 140382015180800, 140382015184895, ++ERASE, 47250779979776, 47250779979776, ++STORE, 94425351438336, 94425351573503, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736801144832, 140737488351231, ++ERASE, 140736801144832, 140736801144832, ++STORE, 140736801144832, 140736801148927, ++STORE, 94629429358592, 94629430071295, ++ERASE, 94629429358592, 94629429358592, ++STORE, 94629429358592, 94629429407743, ++STORE, 94629429407744, 94629430071295, ++ERASE, 94629429407744, 94629429407744, ++STORE, 94629429407744, 94629429952511, ++STORE, 94629429952512, 94629430050815, ++STORE, 94629430050816, 94629430071295, ++STORE, 139801685483520, 139801685655551, ++ERASE, 139801685483520, 139801685483520, ++STORE, 139801685483520, 139801685487615, ++STORE, 139801685487616, 139801685655551, ++ERASE, 139801685487616, 139801685487616, ++STORE, 139801685487616, 139801685610495, ++STORE, 139801685610496, 139801685643263, ++STORE, 139801685643264, 139801685651455, ++STORE, 139801685651456, 139801685655551, ++STORE, 140736801198080, 140736801202175, ++STORE, 140736801185792, 140736801198079, ++STORE, 47831109513216, 47831109521407, ++STORE, 47831109521408, 47831109529599, ++STORE, 47831109529600, 47831109636095, ++STORE, 47831109545984, 47831109636095, ++STORE, 47831109529600, 47831109545983, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109619711, ++STORE, 47831109619712, 47831109636095, ++STORE, 47831109599232, 47831109619711, ++STORE, 47831109545984, 47831109599231, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109599231, ++STORE, 47831109615616, 47831109619711, ++STORE, 47831109599232, 47831109615615, ++ERASE, 47831109599232, 47831109599232, ++STORE, 47831109599232, 47831109615615, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109627904, 47831109627904, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109636096, 47831111475199, ++STORE, 47831109775360, 47831111475199, ++STORE, 47831109636096, 47831109775359, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111434239, ++STORE, 47831111434240, 47831111475199, ++STORE, 47831111118848, 47831111434239, ++STORE, 47831109775360, 47831111118847, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111118847, ++STORE, 47831111430144, 47831111434239, ++STORE, 47831111118848, 47831111430143, ++ERASE, 47831111118848, 47831111118848, ++STORE, 47831111118848, 47831111430143, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111458816, 47831111458816, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111458816, 47831111487487, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111450623, ++STORE, 47831111450624, 47831111458815, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109623807, ++STORE, 47831109623808, 47831109627903, ++ERASE, 94629430050816, 94629430050816, ++STORE, 94629430050816, 94629430067199, ++STORE, 94629430067200, 94629430071295, ++ERASE, 139801685643264, 139801685643264, ++STORE, 139801685643264, 139801685647359, ++STORE, 139801685647360, 139801685651455, ++ERASE, 47831109513216, 47831109513216, ++STORE, 140737488347136, 140737488351231, ++STORE, 140729419612160, 140737488351231, ++ERASE, 140729419612160, 140729419612160, ++STORE, 140729419612160, 140729419616255, ++STORE, 94443354148864, 94443354861567, ++ERASE, 94443354148864, 94443354148864, ++STORE, 94443354148864, 94443354198015, ++STORE, 94443354198016, 94443354861567, ++ERASE, 94443354198016, 94443354198016, ++STORE, 94443354198016, 94443354742783, ++STORE, 94443354742784, 94443354841087, ++STORE, 94443354841088, 94443354861567, ++STORE, 139741700038656, 139741700210687, ++ERASE, 139741700038656, 139741700038656, ++STORE, 139741700038656, 139741700042751, ++STORE, 139741700042752, 139741700210687, ++ERASE, 139741700042752, 139741700042752, ++STORE, 139741700042752, 139741700165631, ++STORE, 139741700165632, 139741700198399, ++STORE, 139741700198400, 139741700206591, ++STORE, 139741700206592, 139741700210687, ++STORE, 140729420574720, 140729420578815, ++STORE, 140729420562432, 140729420574719, ++STORE, 47891094958080, 47891094966271, ++STORE, 47891094966272, 47891094974463, ++STORE, 47891094974464, 47891095080959, ++STORE, 47891094990848, 47891095080959, ++STORE, 47891094974464, 47891094990847, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095064575, ++STORE, 47891095064576, 47891095080959, ++STORE, 47891095044096, 47891095064575, ++STORE, 47891094990848, 47891095044095, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095044095, ++STORE, 47891095060480, 47891095064575, ++STORE, 47891095044096, 47891095060479, ++ERASE, 47891095044096, 47891095044096, ++STORE, 47891095044096, 47891095060479, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095072768, 47891095072768, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095080960, 47891096920063, ++STORE, 47891095220224, 47891096920063, ++STORE, 47891095080960, 47891095220223, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096879103, ++STORE, 47891096879104, 47891096920063, ++STORE, 47891096563712, 47891096879103, ++STORE, 47891095220224, 47891096563711, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096563711, ++STORE, 47891096875008, 47891096879103, ++STORE, 47891096563712, 47891096875007, ++ERASE, 47891096563712, 47891096563712, ++STORE, 47891096563712, 47891096875007, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096903680, 47891096903680, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096903680, 47891096932351, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096895487, ++STORE, 47891096895488, 47891096903679, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095068671, ++STORE, 47891095068672, 47891095072767, ++ERASE, 94443354841088, 94443354841088, ++STORE, 94443354841088, 94443354857471, ++STORE, 94443354857472, 94443354861567, ++ERASE, 139741700198400, 139741700198400, ++STORE, 139741700198400, 139741700202495, ++STORE, 139741700202496, 139741700206591, ++ERASE, 47891094958080, 47891094958080, ++STORE, 94443360825344, 94443360960511, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722961661952, 140737488351231, ++ERASE, 140722961661952, 140722961661952, ++STORE, 140722961661952, 140722961666047, ++STORE, 94878388944896, 94878389657599, ++ERASE, 94878388944896, 94878388944896, ++STORE, 94878388944896, 94878388994047, ++STORE, 94878388994048, 94878389657599, ++ERASE, 94878388994048, 94878388994048, ++STORE, 94878388994048, 94878389538815, ++STORE, 94878389538816, 94878389637119, ++STORE, 94878389637120, 94878389657599, ++STORE, 140210690056192, 140210690228223, ++ERASE, 140210690056192, 140210690056192, ++STORE, 140210690056192, 140210690060287, ++STORE, 140210690060288, 140210690228223, ++ERASE, 140210690060288, 140210690060288, ++STORE, 140210690060288, 140210690183167, ++STORE, 140210690183168, 140210690215935, ++STORE, 140210690215936, 140210690224127, ++STORE, 140210690224128, 140210690228223, ++STORE, 140722963148800, 140722963152895, ++STORE, 140722963136512, 140722963148799, ++STORE, 47422104940544, 47422104948735, ++STORE, 47422104948736, 47422104956927, ++STORE, 47422104956928, 47422105063423, ++STORE, 47422104973312, 47422105063423, ++STORE, 47422104956928, 47422104973311, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105047039, ++STORE, 47422105047040, 47422105063423, ++STORE, 47422105026560, 47422105047039, ++STORE, 47422104973312, 47422105026559, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105026559, ++STORE, 47422105042944, 47422105047039, ++STORE, 47422105026560, 47422105042943, ++ERASE, 47422105026560, 47422105026560, ++STORE, 47422105026560, 47422105042943, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105055232, 47422105055232, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105063424, 47422106902527, ++STORE, 47422105202688, 47422106902527, ++STORE, 47422105063424, 47422105202687, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106861567, ++STORE, 47422106861568, 47422106902527, ++STORE, 47422106546176, 47422106861567, ++STORE, 47422105202688, 47422106546175, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106546175, ++STORE, 47422106857472, 47422106861567, ++STORE, 47422106546176, 47422106857471, ++ERASE, 47422106546176, 47422106546176, ++STORE, 47422106546176, 47422106857471, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106886144, 47422106886144, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106886144, 47422106914815, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106877951, ++STORE, 47422106877952, 47422106886143, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105051135, ++STORE, 47422105051136, 47422105055231, ++ERASE, 94878389637120, 94878389637120, ++STORE, 94878389637120, 94878389653503, ++STORE, 94878389653504, 94878389657599, ++ERASE, 140210690215936, 140210690215936, ++STORE, 140210690215936, 140210690220031, ++STORE, 140210690220032, 140210690224127, ++ERASE, 47422104940544, 47422104940544, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727690309632, 140737488351231, ++ERASE, 140727690309632, 140727690309632, ++STORE, 140727690309632, 140727690313727, ++STORE, 94121892208640, 94121892921343, ++ERASE, 94121892208640, 94121892208640, ++STORE, 94121892208640, 94121892257791, ++STORE, 94121892257792, 94121892921343, ++ERASE, 94121892257792, 94121892257792, ++STORE, 94121892257792, 94121892802559, ++STORE, 94121892802560, 94121892900863, ++STORE, 94121892900864, 94121892921343, ++STORE, 140662438326272, 140662438498303, ++ERASE, 140662438326272, 140662438326272, ++STORE, 140662438326272, 140662438330367, ++STORE, 140662438330368, 140662438498303, ++ERASE, 140662438330368, 140662438330368, ++STORE, 140662438330368, 140662438453247, ++STORE, 140662438453248, 140662438486015, ++STORE, 140662438486016, 140662438494207, ++STORE, 140662438494208, 140662438498303, ++STORE, 140727690379264, 140727690383359, ++STORE, 140727690366976, 140727690379263, ++STORE, 46970356670464, 46970356678655, ++STORE, 46970356678656, 46970356686847, ++STORE, 46970356686848, 46970356793343, ++STORE, 46970356703232, 46970356793343, ++STORE, 46970356686848, 46970356703231, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356776959, ++STORE, 46970356776960, 46970356793343, ++STORE, 46970356756480, 46970356776959, ++STORE, 46970356703232, 46970356756479, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356756479, ++STORE, 46970356772864, 46970356776959, ++STORE, 46970356756480, 46970356772863, ++ERASE, 46970356756480, 46970356756480, ++STORE, 46970356756480, 46970356772863, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356785152, 46970356785152, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356793344, 46970358632447, ++STORE, 46970356932608, 46970358632447, ++STORE, 46970356793344, 46970356932607, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358591487, ++STORE, 46970358591488, 46970358632447, ++STORE, 46970358276096, 46970358591487, ++STORE, 46970356932608, 46970358276095, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358276095, ++STORE, 46970358587392, 46970358591487, ++STORE, 46970358276096, 46970358587391, ++ERASE, 46970358276096, 46970358276096, ++STORE, 46970358276096, 46970358587391, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358616064, 46970358616064, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358616064, 46970358644735, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358607871, ++STORE, 46970358607872, 46970358616063, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356781055, ++STORE, 46970356781056, 46970356785151, ++ERASE, 94121892900864, 94121892900864, ++STORE, 94121892900864, 94121892917247, ++STORE, 94121892917248, 94121892921343, ++ERASE, 140662438486016, 140662438486016, ++STORE, 140662438486016, 140662438490111, ++STORE, 140662438490112, 140662438494207, ++ERASE, 46970356670464, 46970356670464, ++STORE, 94121898610688, 94121898745855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737189351424, 140737488351231, ++ERASE, 140737189351424, 140737189351424, ++STORE, 140737189351424, 140737189355519, ++STORE, 93847948832768, 93847949545471, ++ERASE, 93847948832768, 93847948832768, ++STORE, 93847948832768, 93847948881919, ++STORE, 93847948881920, 93847949545471, ++ERASE, 93847948881920, 93847948881920, ++STORE, 93847948881920, 93847949426687, ++STORE, 93847949426688, 93847949524991, ++STORE, 93847949524992, 93847949545471, ++STORE, 139698989985792, 139698990157823, ++ERASE, 139698989985792, 139698989985792, ++STORE, 139698989985792, 139698989989887, ++STORE, 139698989989888, 139698990157823, ++ERASE, 139698989989888, 139698989989888, ++STORE, 139698989989888, 139698990112767, ++STORE, 139698990112768, 139698990145535, ++STORE, 139698990145536, 139698990153727, ++STORE, 139698990153728, 139698990157823, ++STORE, 140737189744640, 140737189748735, ++STORE, 140737189732352, 140737189744639, ++STORE, 47933805010944, 47933805019135, ++STORE, 47933805019136, 47933805027327, ++STORE, 47933805027328, 47933805133823, ++STORE, 47933805043712, 47933805133823, ++STORE, 47933805027328, 47933805043711, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805117439, ++STORE, 47933805117440, 47933805133823, ++STORE, 47933805096960, 47933805117439, ++STORE, 47933805043712, 47933805096959, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805096959, ++STORE, 47933805113344, 47933805117439, ++STORE, 47933805096960, 47933805113343, ++ERASE, 47933805096960, 47933805096960, ++STORE, 47933805096960, 47933805113343, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805125632, 47933805125632, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805133824, 47933806972927, ++STORE, 47933805273088, 47933806972927, ++STORE, 47933805133824, 47933805273087, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806931967, ++STORE, 47933806931968, 47933806972927, ++STORE, 47933806616576, 47933806931967, ++STORE, 47933805273088, 47933806616575, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806616575, ++STORE, 47933806927872, 47933806931967, ++STORE, 47933806616576, 47933806927871, ++ERASE, 47933806616576, 47933806616576, ++STORE, 47933806616576, 47933806927871, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806956544, 47933806956544, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806956544, 47933806985215, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806948351, ++STORE, 47933806948352, 47933806956543, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805121535, ++STORE, 47933805121536, 47933805125631, ++ERASE, 93847949524992, 93847949524992, ++STORE, 93847949524992, 93847949541375, ++STORE, 93847949541376, 93847949545471, ++ERASE, 139698990145536, 139698990145536, ++STORE, 139698990145536, 139698990149631, ++STORE, 139698990149632, 139698990153727, ++ERASE, 47933805010944, 47933805010944, ++STORE, 140737488347136, 140737488351231, ++STORE, 140725553991680, 140737488351231, ++ERASE, 140725553991680, 140725553991680, ++STORE, 140725553991680, 140725553995775, ++STORE, 93980056248320, 93980056961023, ++ERASE, 93980056248320, 93980056248320, ++STORE, 93980056248320, 93980056297471, ++STORE, 93980056297472, 93980056961023, ++ERASE, 93980056297472, 93980056297472, ++STORE, 93980056297472, 93980056842239, ++STORE, 93980056842240, 93980056940543, ++STORE, 93980056940544, 93980056961023, ++STORE, 140146588971008, 140146589143039, ++ERASE, 140146588971008, 140146588971008, ++STORE, 140146588971008, 140146588975103, ++STORE, 140146588975104, 140146589143039, ++ERASE, 140146588975104, 140146588975104, ++STORE, 140146588975104, 140146589097983, ++STORE, 140146589097984, 140146589130751, ++STORE, 140146589130752, 140146589138943, ++STORE, 140146589138944, 140146589143039, ++STORE, 140725554860032, 140725554864127, ++STORE, 140725554847744, 140725554860031, ++STORE, 47486206025728, 47486206033919, ++STORE, 47486206033920, 47486206042111, ++STORE, 47486206042112, 47486206148607, ++STORE, 47486206058496, 47486206148607, ++STORE, 47486206042112, 47486206058495, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206132223, ++STORE, 47486206132224, 47486206148607, ++STORE, 47486206111744, 47486206132223, ++STORE, 47486206058496, 47486206111743, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206111743, ++STORE, 47486206128128, 47486206132223, ++STORE, 47486206111744, 47486206128127, ++ERASE, 47486206111744, 47486206111744, ++STORE, 47486206111744, 47486206128127, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206140416, 47486206140416, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206148608, 47486207987711, ++STORE, 47486206287872, 47486207987711, ++STORE, 47486206148608, 47486206287871, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207946751, ++STORE, 47486207946752, 47486207987711, ++STORE, 47486207631360, 47486207946751, ++STORE, 47486206287872, 47486207631359, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207631359, ++STORE, 47486207942656, 47486207946751, ++STORE, 47486207631360, 47486207942655, ++ERASE, 47486207631360, 47486207631360, ++STORE, 47486207631360, 47486207942655, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207971328, 47486207971328, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207971328, 47486207999999, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207963135, ++STORE, 47486207963136, 47486207971327, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206136319, ++STORE, 47486206136320, 47486206140415, ++ERASE, 93980056940544, 93980056940544, ++STORE, 93980056940544, 93980056956927, ++STORE, 93980056956928, 93980056961023, ++ERASE, 140146589130752, 140146589130752, ++STORE, 140146589130752, 140146589134847, ++STORE, 140146589134848, 140146589138943, ++ERASE, 47486206025728, 47486206025728, ++STORE, 93980070006784, 93980070141951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727334776832, 140737488351231, ++ERASE, 140727334776832, 140727334776832, ++STORE, 140727334776832, 140727334780927, ++STORE, 94049747247104, 94049747959807, ++ERASE, 94049747247104, 94049747247104, ++STORE, 94049747247104, 94049747296255, ++STORE, 94049747296256, 94049747959807, ++ERASE, 94049747296256, 94049747296256, ++STORE, 94049747296256, 94049747841023, ++STORE, 94049747841024, 94049747939327, ++STORE, 94049747939328, 94049747959807, ++STORE, 140227307216896, 140227307388927, ++ERASE, 140227307216896, 140227307216896, ++STORE, 140227307216896, 140227307220991, ++STORE, 140227307220992, 140227307388927, ++ERASE, 140227307220992, 140227307220992, ++STORE, 140227307220992, 140227307343871, ++STORE, 140227307343872, 140227307376639, ++STORE, 140227307376640, 140227307384831, ++STORE, 140227307384832, 140227307388927, ++STORE, 140727335337984, 140727335342079, ++STORE, 140727335325696, 140727335337983, ++STORE, 47405487779840, 47405487788031, ++STORE, 47405487788032, 47405487796223, ++STORE, 47405487796224, 47405487902719, ++STORE, 47405487812608, 47405487902719, ++STORE, 47405487796224, 47405487812607, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487886335, ++STORE, 47405487886336, 47405487902719, ++STORE, 47405487865856, 47405487886335, ++STORE, 47405487812608, 47405487865855, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487865855, ++STORE, 47405487882240, 47405487886335, ++STORE, 47405487865856, 47405487882239, ++ERASE, 47405487865856, 47405487865856, ++STORE, 47405487865856, 47405487882239, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487894528, 47405487894528, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487902720, 47405489741823, ++STORE, 47405488041984, 47405489741823, ++STORE, 47405487902720, 47405488041983, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489700863, ++STORE, 47405489700864, 47405489741823, ++STORE, 47405489385472, 47405489700863, ++STORE, 47405488041984, 47405489385471, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489385471, ++STORE, 47405489696768, 47405489700863, ++STORE, 47405489385472, 47405489696767, ++ERASE, 47405489385472, 47405489385472, ++STORE, 47405489385472, 47405489696767, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489725440, 47405489725440, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489725440, 47405489754111, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489717247, ++STORE, 47405489717248, 47405489725439, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487890431, ++STORE, 47405487890432, 47405487894527, ++ERASE, 94049747939328, 94049747939328, ++STORE, 94049747939328, 94049747955711, ++STORE, 94049747955712, 94049747959807, ++ERASE, 140227307376640, 140227307376640, ++STORE, 140227307376640, 140227307380735, ++STORE, 140227307380736, 140227307384831, ++ERASE, 47405487779840, 47405487779840, ++STORE, 94049758810112, 94049758945279, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727079718912, 140737488351231, ++ERASE, 140727079718912, 140727079718912, ++STORE, 140727079718912, 140727079723007, ++STORE, 94250996527104, 94250997239807, ++ERASE, 94250996527104, 94250996527104, ++STORE, 94250996527104, 94250996576255, ++STORE, 94250996576256, 94250997239807, ++ERASE, 94250996576256, 94250996576256, ++STORE, 94250996576256, 94250997121023, ++STORE, 94250997121024, 94250997219327, ++STORE, 94250997219328, 94250997239807, ++STORE, 140060022587392, 140060022759423, ++ERASE, 140060022587392, 140060022587392, ++STORE, 140060022587392, 140060022591487, ++STORE, 140060022591488, 140060022759423, ++ERASE, 140060022591488, 140060022591488, ++STORE, 140060022591488, 140060022714367, ++STORE, 140060022714368, 140060022747135, ++STORE, 140060022747136, 140060022755327, ++STORE, 140060022755328, 140060022759423, ++STORE, 140727079788544, 140727079792639, ++STORE, 140727079776256, 140727079788543, ++STORE, 47572772409344, 47572772417535, ++STORE, 47572772417536, 47572772425727, ++STORE, 47572772425728, 47572772532223, ++STORE, 47572772442112, 47572772532223, ++STORE, 47572772425728, 47572772442111, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772515839, ++STORE, 47572772515840, 47572772532223, ++STORE, 47572772495360, 47572772515839, ++STORE, 47572772442112, 47572772495359, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772495359, ++STORE, 47572772511744, 47572772515839, ++STORE, 47572772495360, 47572772511743, ++ERASE, 47572772495360, 47572772495360, ++STORE, 47572772495360, 47572772511743, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772524032, 47572772524032, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772532224, 47572774371327, ++STORE, 47572772671488, 47572774371327, ++STORE, 47572772532224, 47572772671487, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774330367, ++STORE, 47572774330368, 47572774371327, ++STORE, 47572774014976, 47572774330367, ++STORE, 47572772671488, 47572774014975, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774014975, ++STORE, 47572774326272, 47572774330367, ++STORE, 47572774014976, 47572774326271, ++ERASE, 47572774014976, 47572774014976, ++STORE, 47572774014976, 47572774326271, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774354944, 47572774354944, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774354944, 47572774383615, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774346751, ++STORE, 47572774346752, 47572774354943, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772519935, ++STORE, 47572772519936, 47572772524031, ++ERASE, 94250997219328, 94250997219328, ++STORE, 94250997219328, 94250997235711, ++STORE, 94250997235712, 94250997239807, ++ERASE, 140060022747136, 140060022747136, ++STORE, 140060022747136, 140060022751231, ++STORE, 140060022751232, 140060022755327, ++ERASE, 47572772409344, 47572772409344, ++STORE, 94251018305536, 94251018440703, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730012389376, 140737488351231, ++ERASE, 140730012389376, 140730012389376, ++STORE, 140730012389376, 140730012393471, ++STORE, 94382607675392, 94382607695871, ++ERASE, 94382607675392, 94382607675392, ++STORE, 94382607675392, 94382607679487, ++STORE, 94382607679488, 94382607695871, ++ERASE, 94382607679488, 94382607679488, ++STORE, 94382607679488, 94382607683583, ++STORE, 94382607683584, 94382607687679, ++STORE, 94382607687680, 94382607695871, ++STORE, 140252451454976, 140252451627007, ++ERASE, 140252451454976, 140252451454976, ++STORE, 140252451454976, 140252451459071, ++STORE, 140252451459072, 140252451627007, ++ERASE, 140252451459072, 140252451459072, ++STORE, 140252451459072, 140252451581951, ++STORE, 140252451581952, 140252451614719, ++STORE, 140252451614720, 140252451622911, ++STORE, 140252451622912, 140252451627007, ++STORE, 140730013548544, 140730013552639, ++STORE, 140730013536256, 140730013548543, ++STORE, 47380343541760, 47380343549951, ++STORE, 47380343549952, 47380343558143, ++STORE, 47380343558144, 47380345397247, ++STORE, 47380343697408, 47380345397247, ++STORE, 47380343558144, 47380343697407, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345356287, ++STORE, 47380345356288, 47380345397247, ++STORE, 47380345040896, 47380345356287, ++STORE, 47380343697408, 47380345040895, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345040895, ++STORE, 47380345352192, 47380345356287, ++STORE, 47380345040896, 47380345352191, ++ERASE, 47380345040896, 47380345040896, ++STORE, 47380345040896, 47380345352191, ++STORE, 47380345380864, 47380345397247, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345380864, 47380345380864, ++STORE, 47380345380864, 47380345397247, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345372671, ++STORE, 47380345372672, 47380345380863, ++ERASE, 94382607687680, 94382607687680, ++STORE, 94382607687680, 94382607691775, ++STORE, 94382607691776, 94382607695871, ++ERASE, 140252451614720, 140252451614720, ++STORE, 140252451614720, 140252451618815, ++STORE, 140252451618816, 140252451622911, ++ERASE, 47380343541760, 47380343541760, ++STORE, 94382626803712, 94382626938879, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730900271104, 140737488351231, ++ERASE, 140730900271104, 140730900271104, ++STORE, 140730900271104, 140730900275199, ++STORE, 93855478120448, 93855478337535, ++ERASE, 93855478120448, 93855478120448, ++STORE, 93855478120448, 93855478198271, ++STORE, 93855478198272, 93855478337535, ++ERASE, 93855478198272, 93855478198272, ++STORE, 93855478198272, 93855478243327, ++STORE, 93855478243328, 93855478288383, ++STORE, 93855478288384, 93855478337535, ++STORE, 140092686573568, 140092686745599, ++ERASE, 140092686573568, 140092686573568, ++STORE, 140092686573568, 140092686577663, ++STORE, 140092686577664, 140092686745599, ++ERASE, 140092686577664, 140092686577664, ++STORE, 140092686577664, 140092686700543, ++STORE, 140092686700544, 140092686733311, ++STORE, 140092686733312, 140092686741503, ++STORE, 140092686741504, 140092686745599, ++STORE, 140730900537344, 140730900541439, ++STORE, 140730900525056, 140730900537343, ++STORE, 47540108423168, 47540108431359, ++STORE, 47540108431360, 47540108439551, ++STORE, 47540108439552, 47540110278655, ++STORE, 47540108578816, 47540110278655, ++STORE, 47540108439552, 47540108578815, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540110237695, ++STORE, 47540110237696, 47540110278655, ++STORE, 47540109922304, 47540110237695, ++STORE, 47540108578816, 47540109922303, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540109922303, ++STORE, 47540110233600, 47540110237695, ++STORE, 47540109922304, 47540110233599, ++ERASE, 47540109922304, 47540109922304, ++STORE, 47540109922304, 47540110233599, ++STORE, 47540110262272, 47540110278655, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110262272, 47540110262272, ++STORE, 47540110262272, 47540110278655, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110254079, ++STORE, 47540110254080, 47540110262271, ++ERASE, 93855478288384, 93855478288384, ++STORE, 93855478288384, 93855478333439, ++STORE, 93855478333440, 93855478337535, ++ERASE, 140092686733312, 140092686733312, ++STORE, 140092686733312, 140092686737407, ++STORE, 140092686737408, 140092686741503, ++ERASE, 47540108423168, 47540108423168, ++STORE, 93855492222976, 93855492358143, ++STORE, 93855492222976, 93855492493311, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733498146816, 140737488351231, ++ERASE, 140733498146816, 140733498146816, ++STORE, 140733498146816, 140733498150911, ++STORE, 94170739654656, 94170740367359, ++ERASE, 94170739654656, 94170739654656, ++STORE, 94170739654656, 94170739703807, ++STORE, 94170739703808, 94170740367359, ++ERASE, 94170739703808, 94170739703808, ++STORE, 94170739703808, 94170740248575, ++STORE, 94170740248576, 94170740346879, ++STORE, 94170740346880, 94170740367359, ++STORE, 140024788877312, 140024789049343, ++ERASE, 140024788877312, 140024788877312, ++STORE, 140024788877312, 140024788881407, ++STORE, 140024788881408, 140024789049343, ++ERASE, 140024788881408, 140024788881408, ++STORE, 140024788881408, 140024789004287, ++STORE, 140024789004288, 140024789037055, ++STORE, 140024789037056, 140024789045247, ++STORE, 140024789045248, 140024789049343, ++STORE, 140733499023360, 140733499027455, ++STORE, 140733499011072, 140733499023359, ++STORE, 47608006119424, 47608006127615, ++STORE, 47608006127616, 47608006135807, ++STORE, 47608006135808, 47608006242303, ++STORE, 47608006152192, 47608006242303, ++STORE, 47608006135808, 47608006152191, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006225919, ++STORE, 47608006225920, 47608006242303, ++STORE, 47608006205440, 47608006225919, ++STORE, 47608006152192, 47608006205439, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006205439, ++STORE, 47608006221824, 47608006225919, ++STORE, 47608006205440, 47608006221823, ++ERASE, 47608006205440, 47608006205440, ++STORE, 47608006205440, 47608006221823, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006234112, 47608006234112, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006242304, 47608008081407, ++STORE, 47608006381568, 47608008081407, ++STORE, 47608006242304, 47608006381567, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608008040447, ++STORE, 47608008040448, 47608008081407, ++STORE, 47608007725056, 47608008040447, ++STORE, 47608006381568, 47608007725055, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608007725055, ++STORE, 47608008036352, 47608008040447, ++STORE, 47608007725056, 47608008036351, ++ERASE, 47608007725056, 47608007725056, ++STORE, 47608007725056, 47608008036351, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008065024, 47608008065024, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008065024, 47608008093695, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008056831, ++STORE, 47608008056832, 47608008065023, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006230015, ++STORE, 47608006230016, 47608006234111, ++ERASE, 94170740346880, 94170740346880, ++STORE, 94170740346880, 94170740363263, ++STORE, 94170740363264, 94170740367359, ++ERASE, 140024789037056, 140024789037056, ++STORE, 140024789037056, 140024789041151, ++STORE, 140024789041152, 140024789045247, ++ERASE, 47608006119424, 47608006119424, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730264326144, 140737488351231, ++ERASE, 140730264326144, 140730264326144, ++STORE, 140730264326144, 140730264330239, ++STORE, 94653216407552, 94653217120255, ++ERASE, 94653216407552, 94653216407552, ++STORE, 94653216407552, 94653216456703, ++STORE, 94653216456704, 94653217120255, ++ERASE, 94653216456704, 94653216456704, ++STORE, 94653216456704, 94653217001471, ++STORE, 94653217001472, 94653217099775, ++STORE, 94653217099776, 94653217120255, ++STORE, 140103617011712, 140103617183743, ++ERASE, 140103617011712, 140103617011712, ++STORE, 140103617011712, 140103617015807, ++STORE, 140103617015808, 140103617183743, ++ERASE, 140103617015808, 140103617015808, ++STORE, 140103617015808, 140103617138687, ++STORE, 140103617138688, 140103617171455, ++STORE, 140103617171456, 140103617179647, ++STORE, 140103617179648, 140103617183743, ++STORE, 140730265427968, 140730265432063, ++STORE, 140730265415680, 140730265427967, ++STORE, 47529177985024, 47529177993215, ++STORE, 47529177993216, 47529178001407, ++STORE, 47529178001408, 47529178107903, ++STORE, 47529178017792, 47529178107903, ++STORE, 47529178001408, 47529178017791, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178091519, ++STORE, 47529178091520, 47529178107903, ++STORE, 47529178071040, 47529178091519, ++STORE, 47529178017792, 47529178071039, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178071039, ++STORE, 47529178087424, 47529178091519, ++STORE, 47529178071040, 47529178087423, ++ERASE, 47529178071040, 47529178071040, ++STORE, 47529178071040, 47529178087423, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178099712, 47529178099712, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178107904, 47529179947007, ++STORE, 47529178247168, 47529179947007, ++STORE, 47529178107904, 47529178247167, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179906047, ++STORE, 47529179906048, 47529179947007, ++STORE, 47529179590656, 47529179906047, ++STORE, 47529178247168, 47529179590655, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179590655, ++STORE, 47529179901952, 47529179906047, ++STORE, 47529179590656, 47529179901951, ++ERASE, 47529179590656, 47529179590656, ++STORE, 47529179590656, 47529179901951, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179930624, 47529179930624, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179930624, 47529179959295, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179922431, ++STORE, 47529179922432, 47529179930623, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178095615, ++STORE, 47529178095616, 47529178099711, ++ERASE, 94653217099776, 94653217099776, ++STORE, 94653217099776, 94653217116159, ++STORE, 94653217116160, 94653217120255, ++ERASE, 140103617171456, 140103617171456, ++STORE, 140103617171456, 140103617175551, ++STORE, 140103617175552, 140103617179647, ++ERASE, 47529177985024, 47529177985024, ++STORE, 94653241135104, 94653241270271, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736284549120, 140737488351231, ++ERASE, 140736284549120, 140736284549120, ++STORE, 140736284549120, 140736284553215, ++STORE, 93963663822848, 93963664506879, ++ERASE, 93963663822848, 93963663822848, ++STORE, 93963663822848, 93963663884287, ++STORE, 93963663884288, 93963664506879, ++ERASE, 93963663884288, 93963663884288, ++STORE, 93963663884288, 93963664240639, ++STORE, 93963664240640, 93963664379903, ++STORE, 93963664379904, 93963664506879, ++STORE, 140450188439552, 140450188611583, ++ERASE, 140450188439552, 140450188439552, ++STORE, 140450188439552, 140450188443647, ++STORE, 140450188443648, 140450188611583, ++ERASE, 140450188443648, 140450188443648, ++STORE, 140450188443648, 140450188566527, ++STORE, 140450188566528, 140450188599295, ++STORE, 140450188599296, 140450188607487, ++STORE, 140450188607488, 140450188611583, ++STORE, 140736284577792, 140736284581887, ++STORE, 140736284565504, 140736284577791, ++STORE, 47182606557184, 47182606565375, ++STORE, 47182606565376, 47182606573567, ++STORE, 47182606573568, 47182608412671, ++STORE, 47182606712832, 47182608412671, ++STORE, 47182606573568, 47182606712831, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608371711, ++STORE, 47182608371712, 47182608412671, ++STORE, 47182608056320, 47182608371711, ++STORE, 47182606712832, 47182608056319, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608056319, ++STORE, 47182608367616, 47182608371711, ++STORE, 47182608056320, 47182608367615, ++ERASE, 47182608056320, 47182608056320, ++STORE, 47182608056320, 47182608367615, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608396288, 47182608396288, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608412672, 47182608523263, ++STORE, 47182608429056, 47182608523263, ++STORE, 47182608412672, 47182608429055, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608515071, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608490496, 47182608515071, ++STORE, 47182608429056, 47182608490495, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608490495, ++STORE, 47182608510976, 47182608515071, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608490496, 47182608490496, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608523264, 47182608568319, ++ERASE, 47182608523264, 47182608523264, ++STORE, 47182608523264, 47182608531455, ++STORE, 47182608531456, 47182608568319, ++STORE, 47182608551936, 47182608568319, ++STORE, 47182608531456, 47182608551935, ++ERASE, 47182608531456, 47182608531456, ++STORE, 47182608531456, 47182608551935, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608551936, 47182608560127, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608568319, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608560127, ++STORE, 47182608560128, 47182608568319, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608568320, 47182608916479, ++STORE, 47182608609280, 47182608916479, ++STORE, 47182608568320, 47182608609279, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608891903, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608822272, 47182608891903, ++STORE, 47182608609280, 47182608822271, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608822271, ++STORE, 47182608887808, 47182608891903, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608822272, 47182608822272, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608916480, 47182611177471, ++STORE, 47182609068032, 47182611177471, ++STORE, 47182608916480, 47182609068031, ++ERASE, 47182609068032, 47182609068032, ++STORE, 47182609068032, 47182611161087, ++STORE, 47182611161088, 47182611177471, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611169280, 47182611169280, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611177472, 47182611312639, ++ERASE, 47182611177472, 47182611177472, ++STORE, 47182611177472, 47182611202047, ++STORE, 47182611202048, 47182611312639, ++STORE, 47182611263488, 47182611312639, ++STORE, 47182611202048, 47182611263487, ++ERASE, 47182611202048, 47182611202048, ++STORE, 47182611202048, 47182611263487, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611263488, 47182611288063, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611312639, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611288063, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611296256, 47182611296256, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611296256, 47182611320831, ++STORE, 47182611320832, 47182611484671, ++ERASE, 47182611320832, 47182611320832, ++STORE, 47182611320832, 47182611333119, ++STORE, 47182611333120, 47182611484671, ++STORE, 47182611431424, 47182611484671, ++STORE, 47182611333120, 47182611431423, ++ERASE, 47182611333120, 47182611333120, ++STORE, 47182611333120, 47182611431423, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611431424, 47182611476479, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611484671, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611476479, ++STORE, 47182611476480, 47182611484671, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611484672, 47182612082687, ++STORE, 47182611603456, 47182612082687, ++STORE, 47182611484672, 47182611603455, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182612029439, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182611918848, 47182612029439, ++STORE, 47182611603456, 47182611918847, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182611918847, ++STORE, 47182612025344, 47182612029439, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182611918848, 47182611918848, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182612082688, 47182615134207, ++STORE, 47182612627456, 47182615134207, ++STORE, 47182612082688, 47182612627455, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614913023, ++STORE, 47182614913024, 47182615134207, ++STORE, 47182614323200, 47182614913023, ++STORE, 47182612627456, 47182614323199, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614323199, ++STORE, 47182614908928, 47182614913023, ++STORE, 47182614323200, 47182614908927, ++ERASE, 47182614323200, 47182614323200, ++STORE, 47182614323200, 47182614908927, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182615117824, 47182615117824, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182615134208, 47182615166975, ++ERASE, 47182615134208, 47182615134208, ++STORE, 47182615134208, 47182615142399, ++STORE, 47182615142400, 47182615166975, ++STORE, 47182615154688, 47182615166975, ++STORE, 47182615142400, 47182615154687, ++ERASE, 47182615142400, 47182615142400, ++STORE, 47182615142400, 47182615154687, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615154688, 47182615158783, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615166975, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615158783, ++STORE, 47182615158784, 47182615166975, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615166976, 47182615203839, ++ERASE, 47182615166976, 47182615166976, ++STORE, 47182615166976, 47182615175167, ++STORE, 47182615175168, 47182615203839, ++STORE, 47182615191552, 47182615203839, ++STORE, 47182615175168, 47182615191551, ++ERASE, 47182615175168, 47182615175168, ++STORE, 47182615175168, 47182615191551, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615191552, 47182615195647, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615203839, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615195647, ++STORE, 47182615195648, 47182615203839, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615203840, 47182615678975, ++ERASE, 47182615203840, 47182615203840, ++STORE, 47182615203840, 47182615212031, ++STORE, 47182615212032, 47182615678975, ++STORE, 47182615547904, 47182615678975, ++STORE, 47182615212032, 47182615547903, ++ERASE, 47182615212032, 47182615212032, ++STORE, 47182615212032, 47182615547903, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615547904, 47182615670783, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615678975, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615670783, ++STORE, 47182615670784, 47182615678975, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615678976, 47182615687167, ++STORE, 47182615687168, 47182615707647, ++ERASE, 47182615687168, 47182615687168, ++STORE, 47182615687168, 47182615691263, ++STORE, 47182615691264, 47182615707647, ++STORE, 47182615695360, 47182615707647, ++STORE, 47182615691264, 47182615695359, ++ERASE, 47182615691264, 47182615691264, ++STORE, 47182615691264, 47182615695359, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615695360, 47182615699455, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615707647, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615699455, ++STORE, 47182615699456, 47182615707647, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615707648, 47182615715839, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608388095, ++STORE, 47182608388096, 47182608396287, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615703551, ++STORE, 47182615703552, 47182615707647, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611292159, ++STORE, 47182611292160, 47182611296255, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615674879, ++STORE, 47182615674880, 47182615678975, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615199743, ++STORE, 47182615199744, 47182615203839, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615162879, ++STORE, 47182615162880, 47182615166975, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615109631, ++STORE, 47182615109632, 47182615117823, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612066303, ++STORE, 47182612066304, 47182612082687, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611480575, ++STORE, 47182611480576, 47182611484671, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611165183, ++STORE, 47182611165184, 47182611169279, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608912383, ++STORE, 47182608912384, 47182608916479, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608564223, ++STORE, 47182608564224, 47182608568319, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608519167, ++STORE, 47182608519168, 47182608523263, ++ERASE, 93963664379904, 93963664379904, ++STORE, 93963664379904, 93963664502783, ++STORE, 93963664502784, 93963664506879, ++ERASE, 140450188599296, 140450188599296, ++STORE, 140450188599296, 140450188603391, ++STORE, 140450188603392, 140450188607487, ++ERASE, 47182606557184, 47182606557184, ++STORE, 93963694723072, 93963694858239, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730313261056, 140737488351231, ++ERASE, 140730313261056, 140730313261056, ++STORE, 140730313261056, 140730313265151, ++STORE, 94386579017728, 94386579697663, ++ERASE, 94386579017728, 94386579017728, ++STORE, 94386579017728, 94386579083263, ++STORE, 94386579083264, 94386579697663, ++ERASE, 94386579083264, 94386579083264, ++STORE, 94386579083264, 94386579431423, ++STORE, 94386579431424, 94386579570687, ++STORE, 94386579570688, 94386579697663, ++STORE, 140124810838016, 140124811010047, ++ERASE, 140124810838016, 140124810838016, ++STORE, 140124810838016, 140124810842111, ++STORE, 140124810842112, 140124811010047, ++ERASE, 140124810842112, 140124810842112, ++STORE, 140124810842112, 140124810964991, ++STORE, 140124810964992, 140124810997759, ++STORE, 140124810997760, 140124811005951, ++STORE, 140124811005952, 140124811010047, ++STORE, 140730313601024, 140730313605119, ++STORE, 140730313588736, 140730313601023, ++STORE, 47507984158720, 47507984166911, ++STORE, 47507984166912, 47507984175103, ++STORE, 47507984175104, 47507986014207, ++STORE, 47507984314368, 47507986014207, ++STORE, 47507984175104, 47507984314367, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985973247, ++STORE, 47507985973248, 47507986014207, ++STORE, 47507985657856, 47507985973247, ++STORE, 47507984314368, 47507985657855, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985657855, ++STORE, 47507985969152, 47507985973247, ++STORE, 47507985657856, 47507985969151, ++ERASE, 47507985657856, 47507985657856, ++STORE, 47507985657856, 47507985969151, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985997824, 47507985997824, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507986014208, 47507986124799, ++STORE, 47507986030592, 47507986124799, ++STORE, 47507986014208, 47507986030591, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986116607, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986092032, 47507986116607, ++STORE, 47507986030592, 47507986092031, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986092031, ++STORE, 47507986112512, 47507986116607, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986092032, 47507986092032, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986124800, 47507986169855, ++ERASE, 47507986124800, 47507986124800, ++STORE, 47507986124800, 47507986132991, ++STORE, 47507986132992, 47507986169855, ++STORE, 47507986153472, 47507986169855, ++STORE, 47507986132992, 47507986153471, ++ERASE, 47507986132992, 47507986132992, ++STORE, 47507986132992, 47507986153471, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986153472, 47507986161663, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986169855, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986161663, ++STORE, 47507986161664, 47507986169855, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986169856, 47507986518015, ++STORE, 47507986210816, 47507986518015, ++STORE, 47507986169856, 47507986210815, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986493439, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986423808, 47507986493439, ++STORE, 47507986210816, 47507986423807, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986423807, ++STORE, 47507986489344, 47507986493439, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986423808, 47507986423808, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986518016, 47507988779007, ++STORE, 47507986669568, 47507988779007, ++STORE, 47507986518016, 47507986669567, ++ERASE, 47507986669568, 47507986669568, ++STORE, 47507986669568, 47507988762623, ++STORE, 47507988762624, 47507988779007, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988770816, 47507988770816, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988779008, 47507988914175, ++ERASE, 47507988779008, 47507988779008, ++STORE, 47507988779008, 47507988803583, ++STORE, 47507988803584, 47507988914175, ++STORE, 47507988865024, 47507988914175, ++STORE, 47507988803584, 47507988865023, ++ERASE, 47507988803584, 47507988803584, ++STORE, 47507988803584, 47507988865023, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988865024, 47507988889599, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988914175, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988889599, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988897792, 47507988897792, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988897792, 47507988922367, ++STORE, 47507988922368, 47507989086207, ++ERASE, 47507988922368, 47507988922368, ++STORE, 47507988922368, 47507988934655, ++STORE, 47507988934656, 47507989086207, ++STORE, 47507989032960, 47507989086207, ++STORE, 47507988934656, 47507989032959, ++ERASE, 47507988934656, 47507988934656, ++STORE, 47507988934656, 47507989032959, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989032960, 47507989078015, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989086207, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989078015, ++STORE, 47507989078016, 47507989086207, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989086208, 47507989684223, ++STORE, 47507989204992, 47507989684223, ++STORE, 47507989086208, 47507989204991, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989630975, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989520384, 47507989630975, ++STORE, 47507989204992, 47507989520383, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989520383, ++STORE, 47507989626880, 47507989630975, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989520384, 47507989520384, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989684224, 47507992735743, ++STORE, 47507990228992, 47507992735743, ++STORE, 47507989684224, 47507990228991, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507992514559, ++STORE, 47507992514560, 47507992735743, ++STORE, 47507991924736, 47507992514559, ++STORE, 47507990228992, 47507991924735, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507991924735, ++STORE, 47507992510464, 47507992514559, ++STORE, 47507991924736, 47507992510463, ++ERASE, 47507991924736, 47507991924736, ++STORE, 47507991924736, 47507992510463, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992719360, 47507992719360, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992735744, 47507992768511, ++ERASE, 47507992735744, 47507992735744, ++STORE, 47507992735744, 47507992743935, ++STORE, 47507992743936, 47507992768511, ++STORE, 47507992756224, 47507992768511, ++STORE, 47507992743936, 47507992756223, ++ERASE, 47507992743936, 47507992743936, ++STORE, 47507992743936, 47507992756223, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992756224, 47507992760319, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992768511, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992760319, ++STORE, 47507992760320, 47507992768511, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992768512, 47507992805375, ++ERASE, 47507992768512, 47507992768512, ++STORE, 47507992768512, 47507992776703, ++STORE, 47507992776704, 47507992805375, ++STORE, 47507992793088, 47507992805375, ++STORE, 47507992776704, 47507992793087, ++ERASE, 47507992776704, 47507992776704, ++STORE, 47507992776704, 47507992793087, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992793088, 47507992797183, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992805375, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992797183, ++STORE, 47507992797184, 47507992805375, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992805376, 47507993280511, ++ERASE, 47507992805376, 47507992805376, ++STORE, 47507992805376, 47507992813567, ++STORE, 47507992813568, 47507993280511, ++STORE, 47507993149440, 47507993280511, ++STORE, 47507992813568, 47507993149439, ++ERASE, 47507992813568, 47507992813568, ++STORE, 47507992813568, 47507993149439, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993149440, 47507993272319, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993280511, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993272319, ++STORE, 47507993272320, 47507993280511, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993280512, 47507993288703, ++STORE, 47507993288704, 47507993309183, ++ERASE, 47507993288704, 47507993288704, ++STORE, 47507993288704, 47507993292799, ++STORE, 47507993292800, 47507993309183, ++STORE, 47507993296896, 47507993309183, ++STORE, 47507993292800, 47507993296895, ++ERASE, 47507993292800, 47507993292800, ++STORE, 47507993292800, 47507993296895, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993296896, 47507993300991, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993309183, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993300991, ++STORE, 47507993300992, 47507993309183, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993309184, 47507993317375, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985989631, ++STORE, 47507985989632, 47507985997823, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993305087, ++STORE, 47507993305088, 47507993309183, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988893695, ++STORE, 47507988893696, 47507988897791, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993276415, ++STORE, 47507993276416, 47507993280511, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992801279, ++STORE, 47507992801280, 47507992805375, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992764415, ++STORE, 47507992764416, 47507992768511, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992711167, ++STORE, 47507992711168, 47507992719359, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989667839, ++STORE, 47507989667840, 47507989684223, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989082111, ++STORE, 47507989082112, 47507989086207, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988766719, ++STORE, 47507988766720, 47507988770815, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986513919, ++STORE, 47507986513920, 47507986518015, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986165759, ++STORE, 47507986165760, 47507986169855, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986120703, ++STORE, 47507986120704, 47507986124799, ++ERASE, 94386579570688, 94386579570688, ++STORE, 94386579570688, 94386579693567, ++STORE, 94386579693568, 94386579697663, ++ERASE, 140124810997760, 140124810997760, ++STORE, 140124810997760, 140124811001855, ++STORE, 140124811001856, 140124811005951, ++ERASE, 47507984158720, 47507984158720, ++STORE, 94386583982080, 94386584117247, ++STORE, 94386583982080, 94386584256511, ++ERASE, 94386583982080, 94386583982080, ++STORE, 94386583982080, 94386584223743, ++STORE, 94386584223744, 94386584256511, ++ERASE, 94386584223744, 94386584223744, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733763395584, 140737488351231, ++ERASE, 140733763395584, 140733763395584, ++STORE, 140733763395584, 140733763399679, ++STORE, 94011546472448, 94011547152383, ++ERASE, 94011546472448, 94011546472448, ++STORE, 94011546472448, 94011546537983, ++STORE, 94011546537984, 94011547152383, ++ERASE, 94011546537984, 94011546537984, ++STORE, 94011546537984, 94011546886143, ++STORE, 94011546886144, 94011547025407, ++STORE, 94011547025408, 94011547152383, ++STORE, 139757597949952, 139757598121983, ++ERASE, 139757597949952, 139757597949952, ++STORE, 139757597949952, 139757597954047, ++STORE, 139757597954048, 139757598121983, ++ERASE, 139757597954048, 139757597954048, ++STORE, 139757597954048, 139757598076927, ++STORE, 139757598076928, 139757598109695, ++STORE, 139757598109696, 139757598117887, ++STORE, 139757598117888, 139757598121983, ++STORE, 140733763596288, 140733763600383, ++STORE, 140733763584000, 140733763596287, ++STORE, 47875197046784, 47875197054975, ++STORE, 47875197054976, 47875197063167, ++STORE, 47875197063168, 47875198902271, ++STORE, 47875197202432, 47875198902271, ++STORE, 47875197063168, 47875197202431, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198861311, ++STORE, 47875198861312, 47875198902271, ++STORE, 47875198545920, 47875198861311, ++STORE, 47875197202432, 47875198545919, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198545919, ++STORE, 47875198857216, 47875198861311, ++STORE, 47875198545920, 47875198857215, ++ERASE, 47875198545920, 47875198545920, ++STORE, 47875198545920, 47875198857215, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198885888, 47875198885888, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198902272, 47875199012863, ++STORE, 47875198918656, 47875199012863, ++STORE, 47875198902272, 47875198918655, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875199004671, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875198980096, 47875199004671, ++STORE, 47875198918656, 47875198980095, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875198980095, ++STORE, 47875199000576, 47875199004671, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875198980096, 47875198980096, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875199012864, 47875199057919, ++ERASE, 47875199012864, 47875199012864, ++STORE, 47875199012864, 47875199021055, ++STORE, 47875199021056, 47875199057919, ++STORE, 47875199041536, 47875199057919, ++STORE, 47875199021056, 47875199041535, ++ERASE, 47875199021056, 47875199021056, ++STORE, 47875199021056, 47875199041535, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199041536, 47875199049727, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199057919, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199049727, ++STORE, 47875199049728, 47875199057919, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199057920, 47875199406079, ++STORE, 47875199098880, 47875199406079, ++STORE, 47875199057920, 47875199098879, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199381503, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199311872, 47875199381503, ++STORE, 47875199098880, 47875199311871, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199311871, ++STORE, 47875199377408, 47875199381503, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199311872, 47875199311872, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199406080, 47875201667071, ++STORE, 47875199557632, 47875201667071, ++STORE, 47875199406080, 47875199557631, ++ERASE, 47875199557632, 47875199557632, ++STORE, 47875199557632, 47875201650687, ++STORE, 47875201650688, 47875201667071, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201658880, 47875201658880, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201667072, 47875201802239, ++ERASE, 47875201667072, 47875201667072, ++STORE, 47875201667072, 47875201691647, ++STORE, 47875201691648, 47875201802239, ++STORE, 47875201753088, 47875201802239, ++STORE, 47875201691648, 47875201753087, ++ERASE, 47875201691648, 47875201691648, ++STORE, 47875201691648, 47875201753087, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201753088, 47875201777663, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201802239, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201777663, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201785856, 47875201785856, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201785856, 47875201810431, ++STORE, 47875201810432, 47875201974271, ++ERASE, 47875201810432, 47875201810432, ++STORE, 47875201810432, 47875201822719, ++STORE, 47875201822720, 47875201974271, ++STORE, 47875201921024, 47875201974271, ++STORE, 47875201822720, 47875201921023, ++ERASE, 47875201822720, 47875201822720, ++STORE, 47875201822720, 47875201921023, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201921024, 47875201966079, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201974271, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201966079, ++STORE, 47875201966080, 47875201974271, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201974272, 47875202572287, ++STORE, 47875202093056, 47875202572287, ++STORE, 47875201974272, 47875202093055, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202519039, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202408448, 47875202519039, ++STORE, 47875202093056, 47875202408447, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202408447, ++STORE, 47875202514944, 47875202519039, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202408448, 47875202408448, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202572288, 47875205623807, ++STORE, 47875203117056, 47875205623807, ++STORE, 47875202572288, 47875203117055, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875205402623, ++STORE, 47875205402624, 47875205623807, ++STORE, 47875204812800, 47875205402623, ++STORE, 47875203117056, 47875204812799, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875204812799, ++STORE, 47875205398528, 47875205402623, ++STORE, 47875204812800, 47875205398527, ++ERASE, 47875204812800, 47875204812800, ++STORE, 47875204812800, 47875205398527, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205607424, 47875205607424, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205623808, 47875205656575, ++ERASE, 47875205623808, 47875205623808, ++STORE, 47875205623808, 47875205631999, ++STORE, 47875205632000, 47875205656575, ++STORE, 47875205644288, 47875205656575, ++STORE, 47875205632000, 47875205644287, ++ERASE, 47875205632000, 47875205632000, ++STORE, 47875205632000, 47875205644287, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205644288, 47875205648383, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205656575, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205648383, ++STORE, 47875205648384, 47875205656575, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205656576, 47875205693439, ++ERASE, 47875205656576, 47875205656576, ++STORE, 47875205656576, 47875205664767, ++STORE, 47875205664768, 47875205693439, ++STORE, 47875205681152, 47875205693439, ++STORE, 47875205664768, 47875205681151, ++ERASE, 47875205664768, 47875205664768, ++STORE, 47875205664768, 47875205681151, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205681152, 47875205685247, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205693439, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205685247, ++STORE, 47875205685248, 47875205693439, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205693440, 47875206168575, ++ERASE, 47875205693440, 47875205693440, ++STORE, 47875205693440, 47875205701631, ++STORE, 47875205701632, 47875206168575, ++STORE, 47875206037504, 47875206168575, ++STORE, 47875205701632, 47875206037503, ++ERASE, 47875205701632, 47875205701632, ++STORE, 47875205701632, 47875206037503, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206037504, 47875206160383, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206168575, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206160383, ++STORE, 47875206160384, 47875206168575, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206168576, 47875206176767, ++STORE, 47875206176768, 47875206197247, ++ERASE, 47875206176768, 47875206176768, ++STORE, 47875206176768, 47875206180863, ++STORE, 47875206180864, 47875206197247, ++STORE, 47875206184960, 47875206197247, ++STORE, 47875206180864, 47875206184959, ++ERASE, 47875206180864, 47875206180864, ++STORE, 47875206180864, 47875206184959, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206184960, 47875206189055, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206197247, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206189055, ++STORE, 47875206189056, 47875206197247, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206197248, 47875206205439, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198877695, ++STORE, 47875198877696, 47875198885887, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206193151, ++STORE, 47875206193152, 47875206197247, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201781759, ++STORE, 47875201781760, 47875201785855, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206164479, ++STORE, 47875206164480, 47875206168575, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205689343, ++STORE, 47875205689344, 47875205693439, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205652479, ++STORE, 47875205652480, 47875205656575, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205599231, ++STORE, 47875205599232, 47875205607423, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202555903, ++STORE, 47875202555904, 47875202572287, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201970175, ++STORE, 47875201970176, 47875201974271, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201654783, ++STORE, 47875201654784, 47875201658879, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199401983, ++STORE, 47875199401984, 47875199406079, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199053823, ++STORE, 47875199053824, 47875199057919, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199008767, ++STORE, 47875199008768, 47875199012863, ++ERASE, 94011547025408, 94011547025408, ++STORE, 94011547025408, 94011547148287, ++STORE, 94011547148288, 94011547152383, ++ERASE, 139757598109696, 139757598109696, ++STORE, 139757598109696, 139757598113791, ++STORE, 139757598113792, 139757598117887, ++ERASE, 47875197046784, 47875197046784, ++STORE, 94011557584896, 94011557720063, ++STORE, 94011557584896, 94011557855231, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557851135, ++STORE, 94011557851136, 94011557855231, ++ERASE, 94011557851136, 94011557851136, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557847039, ++STORE, 94011557847040, 94011557851135, ++ERASE, 94011557847040, 94011557847040, ++STORE, 94011557584896, 94011557982207, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557978111, ++STORE, 94011557978112, 94011557982207, ++ERASE, 94011557978112, 94011557978112, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557974015, ++STORE, 94011557974016, 94011557978111, ++ERASE, 94011557974016, 94011557974016, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734130360320, 140737488351231, ++ERASE, 140734130360320, 140734130360320, ++STORE, 140734130360320, 140734130364415, ++STORE, 94641232105472, 94641232785407, ++ERASE, 94641232105472, 94641232105472, ++STORE, 94641232105472, 94641232171007, ++STORE, 94641232171008, 94641232785407, ++ERASE, 94641232171008, 94641232171008, ++STORE, 94641232171008, 94641232519167, ++STORE, 94641232519168, 94641232658431, ++STORE, 94641232658432, 94641232785407, ++STORE, 139726599516160, 139726599688191, ++ERASE, 139726599516160, 139726599516160, ++STORE, 139726599516160, 139726599520255, ++STORE, 139726599520256, 139726599688191, ++ERASE, 139726599520256, 139726599520256, ++STORE, 139726599520256, 139726599643135, ++STORE, 139726599643136, 139726599675903, ++STORE, 139726599675904, 139726599684095, ++STORE, 139726599684096, 139726599688191, ++STORE, 140734130446336, 140734130450431, ++STORE, 140734130434048, 140734130446335, ++STORE, 47906195480576, 47906195488767, ++STORE, 47906195488768, 47906195496959, ++STORE, 47906195496960, 47906197336063, ++STORE, 47906195636224, 47906197336063, ++STORE, 47906195496960, 47906195636223, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906197295103, ++STORE, 47906197295104, 47906197336063, ++STORE, 47906196979712, 47906197295103, ++STORE, 47906195636224, 47906196979711, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906196979711, ++STORE, 47906197291008, 47906197295103, ++STORE, 47906196979712, 47906197291007, ++ERASE, 47906196979712, 47906196979712, ++STORE, 47906196979712, 47906197291007, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197319680, 47906197319680, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197336064, 47906197446655, ++STORE, 47906197352448, 47906197446655, ++STORE, 47906197336064, 47906197352447, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197438463, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197413888, 47906197438463, ++STORE, 47906197352448, 47906197413887, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197413887, ++STORE, 47906197434368, 47906197438463, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197413888, 47906197413888, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197446656, 47906197491711, ++ERASE, 47906197446656, 47906197446656, ++STORE, 47906197446656, 47906197454847, ++STORE, 47906197454848, 47906197491711, ++STORE, 47906197475328, 47906197491711, ++STORE, 47906197454848, 47906197475327, ++ERASE, 47906197454848, 47906197454848, ++STORE, 47906197454848, 47906197475327, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197475328, 47906197483519, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197491711, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197483519, ++STORE, 47906197483520, 47906197491711, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197491712, 47906197839871, ++STORE, 47906197532672, 47906197839871, ++STORE, 47906197491712, 47906197532671, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197815295, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197745664, 47906197815295, ++STORE, 47906197532672, 47906197745663, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197745663, ++STORE, 47906197811200, 47906197815295, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197745664, 47906197745664, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197839872, 47906200100863, ++STORE, 47906197991424, 47906200100863, ++STORE, 47906197839872, 47906197991423, ++ERASE, 47906197991424, 47906197991424, ++STORE, 47906197991424, 47906200084479, ++STORE, 47906200084480, 47906200100863, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200092672, 47906200092672, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200100864, 47906200236031, ++ERASE, 47906200100864, 47906200100864, ++STORE, 47906200100864, 47906200125439, ++STORE, 47906200125440, 47906200236031, ++STORE, 47906200186880, 47906200236031, ++STORE, 47906200125440, 47906200186879, ++ERASE, 47906200125440, 47906200125440, ++STORE, 47906200125440, 47906200186879, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200186880, 47906200211455, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200236031, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200211455, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200219648, 47906200219648, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200219648, 47906200244223, ++STORE, 47906200244224, 47906200408063, ++ERASE, 47906200244224, 47906200244224, ++STORE, 47906200244224, 47906200256511, ++STORE, 47906200256512, 47906200408063, ++STORE, 47906200354816, 47906200408063, ++STORE, 47906200256512, 47906200354815, ++ERASE, 47906200256512, 47906200256512, ++STORE, 47906200256512, 47906200354815, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200354816, 47906200399871, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200408063, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200399871, ++STORE, 47906200399872, 47906200408063, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200408064, 47906201006079, ++STORE, 47906200526848, 47906201006079, ++STORE, 47906200408064, 47906200526847, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200952831, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906200842240, 47906200952831, ++STORE, 47906200526848, 47906200842239, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200842239, ++STORE, 47906200948736, 47906200952831, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200842240, 47906200842240, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906201006080, 47906204057599, ++STORE, 47906201550848, 47906204057599, ++STORE, 47906201006080, 47906201550847, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203836415, ++STORE, 47906203836416, 47906204057599, ++STORE, 47906203246592, 47906203836415, ++STORE, 47906201550848, 47906203246591, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203246591, ++STORE, 47906203832320, 47906203836415, ++STORE, 47906203246592, 47906203832319, ++ERASE, 47906203246592, 47906203246592, ++STORE, 47906203246592, 47906203832319, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906204041216, 47906204041216, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906204057600, 47906204090367, ++ERASE, 47906204057600, 47906204057600, ++STORE, 47906204057600, 47906204065791, ++STORE, 47906204065792, 47906204090367, ++STORE, 47906204078080, 47906204090367, ++STORE, 47906204065792, 47906204078079, ++ERASE, 47906204065792, 47906204065792, ++STORE, 47906204065792, 47906204078079, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204078080, 47906204082175, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204090367, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204082175, ++STORE, 47906204082176, 47906204090367, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204090368, 47906204127231, ++ERASE, 47906204090368, 47906204090368, ++STORE, 47906204090368, 47906204098559, ++STORE, 47906204098560, 47906204127231, ++STORE, 47906204114944, 47906204127231, ++STORE, 47906204098560, 47906204114943, ++ERASE, 47906204098560, 47906204098560, ++STORE, 47906204098560, 47906204114943, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204114944, 47906204119039, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204127231, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204119039, ++STORE, 47906204119040, 47906204127231, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204127232, 47906204602367, ++ERASE, 47906204127232, 47906204127232, ++STORE, 47906204127232, 47906204135423, ++STORE, 47906204135424, 47906204602367, ++STORE, 47906204471296, 47906204602367, ++STORE, 47906204135424, 47906204471295, ++ERASE, 47906204135424, 47906204135424, ++STORE, 47906204135424, 47906204471295, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204471296, 47906204594175, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204602367, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204594175, ++STORE, 47906204594176, 47906204602367, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204602368, 47906204610559, ++STORE, 47906204610560, 47906204631039, ++ERASE, 47906204610560, 47906204610560, ++STORE, 47906204610560, 47906204614655, ++STORE, 47906204614656, 47906204631039, ++STORE, 47906204618752, 47906204631039, ++STORE, 47906204614656, 47906204618751, ++ERASE, 47906204614656, 47906204614656, ++STORE, 47906204614656, 47906204618751, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204618752, 47906204622847, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204631039, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204622847, ++STORE, 47906204622848, 47906204631039, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204631040, 47906204639231, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197311487, ++STORE, 47906197311488, 47906197319679, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204626943, ++STORE, 47906204626944, 47906204631039, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200215551, ++STORE, 47906200215552, 47906200219647, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204598271, ++STORE, 47906204598272, 47906204602367, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204123135, ++STORE, 47906204123136, 47906204127231, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204086271, ++STORE, 47906204086272, 47906204090367, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204033023, ++STORE, 47906204033024, 47906204041215, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906200989695, ++STORE, 47906200989696, 47906201006079, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200403967, ++STORE, 47906200403968, 47906200408063, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200088575, ++STORE, 47906200088576, 47906200092671, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197835775, ++STORE, 47906197835776, 47906197839871, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197487615, ++STORE, 47906197487616, 47906197491711, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197442559, ++STORE, 47906197442560, 47906197446655, ++ERASE, 94641232658432, 94641232658432, ++STORE, 94641232658432, 94641232781311, ++STORE, 94641232781312, 94641232785407, ++ERASE, 139726599675904, 139726599675904, ++STORE, 139726599675904, 139726599679999, ++STORE, 139726599680000, 139726599684095, ++ERASE, 47906195480576, 47906195480576, ++STORE, 94641242615808, 94641242750975, ++ }; ++ unsigned long set11[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140732658499584, 140737488351231, ++ERASE, 140732658499584, 140732658499584, ++STORE, 140732658499584, 140732658503679, ++STORE, 94029856579584, 94029856751615, ++ERASE, 94029856579584, 94029856579584, ++STORE, 94029856579584, 94029856595967, ++STORE, 94029856595968, 94029856751615, ++ERASE, 94029856595968, 94029856595968, ++STORE, 94029856595968, 94029856698367, ++STORE, 94029856698368, 94029856739327, ++STORE, 94029856739328, 94029856751615, ++STORE, 140014592573440, 140014592745471, ++ERASE, 140014592573440, 140014592573440, ++STORE, 140014592573440, 140014592577535, ++STORE, 140014592577536, 140014592745471, ++ERASE, 140014592577536, 140014592577536, ++STORE, 140014592577536, 140014592700415, ++STORE, 140014592700416, 140014592733183, ++STORE, 140014592733184, 140014592741375, ++STORE, 140014592741376, 140014592745471, ++STORE, 140732658565120, 140732658569215, ++STORE, 140732658552832, 140732658565119, ++ }; ++ ++ unsigned long set12[] = { /* contains 12 values. */ ++STORE, 140737488347136, 140737488351231, ++STORE, 140732658499584, 140737488351231, ++ERASE, 140732658499584, 140732658499584, ++STORE, 140732658499584, 140732658503679, ++STORE, 94029856579584, 94029856751615, ++ERASE, 94029856579584, 94029856579584, ++STORE, 94029856579584, 94029856595967, ++STORE, 94029856595968, 94029856751615, ++ERASE, 94029856595968, 94029856595968, ++STORE, 94029856595968, 94029856698367, ++STORE, 94029856698368, 94029856739327, ++STORE, 94029856739328, 94029856751615, ++STORE, 140014592573440, 140014592745471, ++ERASE, 140014592573440, 140014592573440, ++STORE, 140014592573440, 140014592577535, ++STORE, 140014592577536, 140014592745471, ++ERASE, 140014592577536, 140014592577536, ++STORE, 140014592577536, 140014592700415, ++STORE, 140014592700416, 140014592733183, ++STORE, 140014592733184, 140014592741375, ++STORE, 140014592741376, 140014592745471, ++STORE, 140732658565120, 140732658569215, ++STORE, 140732658552832, 140732658565119, ++STORE, 140014592741375, 140014592741375, /* contrived */ ++STORE, 140014592733184, 140014592741376, /* creates first entry retry. */ ++ }; ++ unsigned long set13[] = { ++STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */ ++STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */ ++STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */ ++STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */ ++STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */ ++STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */ ++STORE, 140373518663680, 140373518684159,/*: deleted (257) */ ++STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */ ++STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */ ++STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */ ++STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */ ++ }; ++ unsigned long set14[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140731667996672, 140737488351231, ++SNULL, 140731668000767, 140737488351231, ++STORE, 140731667996672, 140731668000767, ++STORE, 140731667865600, 140731668000767, ++STORE, 94077521272832, 94077521313791, ++SNULL, 94077521301503, 94077521313791, ++STORE, 94077521272832, 94077521301503, ++STORE, 94077521301504, 94077521313791, ++ERASE, 94077521301504, 94077521313791, ++STORE, 94077521305600, 94077521313791, ++STORE, 139826134630400, 139826136883199, ++SNULL, 139826134773759, 139826136883199, ++STORE, 139826134630400, 139826134773759, ++STORE, 139826134773760, 139826136883199, ++ERASE, 139826134773760, 139826136883199, ++STORE, 139826136870912, 139826136879103, ++STORE, 139826136879104, 139826136883199, ++STORE, 140731668013056, 140731668017151, ++STORE, 140731668000768, 140731668013055, ++STORE, 139826136862720, 139826136870911, ++STORE, 139826132406272, 139826134630399, ++SNULL, 139826134056959, 139826134630399, ++STORE, 139826132406272, 139826134056959, ++STORE, 139826134056960, 139826134630399, ++SNULL, 139826134056960, 139826134626303, ++STORE, 139826134626304, 139826134630399, ++STORE, 139826134056960, 139826134626303, ++ERASE, 139826134056960, 139826134626303, ++STORE, 139826134056960, 139826134626303, ++ERASE, 139826134626304, 139826134630399, ++STORE, 139826134626304, 139826134630399, ++STORE, 139826136842240, 139826136862719, ++STORE, 139826130022400, 139826132406271, ++SNULL, 139826130022400, 139826130288639, ++STORE, 139826130288640, 139826132406271, ++STORE, 139826130022400, 139826130288639, ++SNULL, 139826132381695, 139826132406271, ++STORE, 139826130288640, 139826132381695, ++STORE, 139826132381696, 139826132406271, ++SNULL, 139826132381696, 139826132402175, ++STORE, 139826132402176, 139826132406271, ++STORE, 139826132381696, 139826132402175, ++ERASE, 139826132381696, 139826132402175, ++STORE, 139826132381696, 139826132402175, ++ERASE, 139826132402176, 139826132406271, ++STORE, 139826132402176, 139826132406271, ++STORE, 139826127806464, 139826130022399, ++SNULL, 139826127806464, 139826127904767, ++STORE, 139826127904768, 139826130022399, ++STORE, 139826127806464, 139826127904767, ++SNULL, 139826129997823, 139826130022399, ++STORE, 139826127904768, 139826129997823, ++STORE, 139826129997824, 139826130022399, ++SNULL, 139826129997824, 139826130006015, ++STORE, 139826130006016, 139826130022399, ++STORE, 139826129997824, 139826130006015, ++ERASE, 139826129997824, 139826130006015, ++STORE, 139826129997824, 139826130006015, ++ERASE, 139826130006016, 139826130022399, ++STORE, 139826130006016, 139826130022399, ++STORE, 139826124009472, 139826127806463, ++SNULL, 139826124009472, 139826125668351, ++STORE, 139826125668352, 139826127806463, ++STORE, 139826124009472, 139826125668351, ++SNULL, 139826127765503, 139826127806463, ++STORE, 139826125668352, 139826127765503, ++STORE, 139826127765504, 139826127806463, ++SNULL, 139826127765504, 139826127790079, ++STORE, 139826127790080, 139826127806463, ++STORE, 139826127765504, 139826127790079, ++ERASE, 139826127765504, 139826127790079, ++STORE, 139826127765504, 139826127790079, ++ERASE, 139826127790080, 139826127806463, ++STORE, 139826127790080, 139826127806463, ++STORE, 139826121748480, 139826124009471, ++SNULL, 139826121748480, 139826121900031, ++STORE, 139826121900032, 139826124009471, ++STORE, 139826121748480, 139826121900031, ++SNULL, 139826123993087, 139826124009471, ++STORE, 139826121900032, 139826123993087, ++STORE, 139826123993088, 139826124009471, ++SNULL, 139826123993088, 139826124001279, ++STORE, 139826124001280, 139826124009471, ++STORE, 139826123993088, 139826124001279, ++ERASE, 139826123993088, 139826124001279, ++STORE, 139826123993088, 139826124001279, ++ERASE, 139826124001280, 139826124009471, ++STORE, 139826124001280, 139826124009471, ++STORE, 139826119626752, 139826121748479, ++SNULL, 139826119626752, 139826119643135, ++STORE, 139826119643136, 139826121748479, ++STORE, 139826119626752, 139826119643135, ++SNULL, 139826121740287, 139826121748479, ++STORE, 139826119643136, 139826121740287, ++STORE, 139826121740288, 139826121748479, ++ERASE, 139826121740288, 139826121748479, ++STORE, 139826121740288, 139826121748479, ++STORE, 139826136834048, 139826136842239, ++STORE, 139826117496832, 139826119626751, ++SNULL, 139826117496832, 139826117525503, ++STORE, 139826117525504, 139826119626751, ++STORE, 139826117496832, 139826117525503, ++SNULL, 139826119618559, 139826119626751, ++STORE, 139826117525504, 139826119618559, ++STORE, 139826119618560, 139826119626751, ++ERASE, 139826119618560, 139826119626751, ++STORE, 139826119618560, 139826119626751, ++STORE, 139826115244032, 139826117496831, ++SNULL, 139826115244032, 139826115395583, ++STORE, 139826115395584, 139826117496831, ++STORE, 139826115244032, 139826115395583, ++SNULL, 139826117488639, 139826117496831, ++STORE, 139826115395584, 139826117488639, ++STORE, 139826117488640, 139826117496831, ++ERASE, 139826117488640, 139826117496831, ++STORE, 139826117488640, 139826117496831, ++STORE, 139826113073152, 139826115244031, ++SNULL, 139826113073152, 139826113142783, ++STORE, 139826113142784, 139826115244031, ++STORE, 139826113073152, 139826113142783, ++SNULL, 139826115235839, 139826115244031, ++STORE, 139826113142784, 139826115235839, ++STORE, 139826115235840, 139826115244031, ++ERASE, 139826115235840, 139826115244031, ++STORE, 139826115235840, 139826115244031, ++STORE, 139826109861888, 139826113073151, ++SNULL, 139826109861888, 139826110939135, ++STORE, 139826110939136, 139826113073151, ++STORE, 139826109861888, 139826110939135, ++SNULL, 139826113036287, 139826113073151, ++STORE, 139826110939136, 139826113036287, ++STORE, 139826113036288, 139826113073151, ++ERASE, 139826113036288, 139826113073151, ++STORE, 139826113036288, 139826113073151, ++STORE, 139826107727872, 139826109861887, ++SNULL, 139826107727872, 139826107756543, ++STORE, 139826107756544, 139826109861887, ++STORE, 139826107727872, 139826107756543, ++SNULL, 139826109853695, 139826109861887, ++STORE, 139826107756544, 139826109853695, ++STORE, 139826109853696, 139826109861887, ++ERASE, 139826109853696, 139826109861887, ++STORE, 139826109853696, 139826109861887, ++STORE, 139826105417728, 139826107727871, ++SNULL, 139826105417728, 139826105622527, ++STORE, 139826105622528, 139826107727871, ++STORE, 139826105417728, 139826105622527, ++SNULL, 139826107719679, 139826107727871, ++STORE, 139826105622528, 139826107719679, ++STORE, 139826107719680, 139826107727871, ++ERASE, 139826107719680, 139826107727871, ++STORE, 139826107719680, 139826107727871, ++STORE, 139826136825856, 139826136842239, ++STORE, 139826103033856, 139826105417727, ++SNULL, 139826103033856, 139826103226367, ++STORE, 139826103226368, 139826105417727, ++STORE, 139826103033856, 139826103226367, ++SNULL, 139826105319423, 139826105417727, ++STORE, 139826103226368, 139826105319423, ++STORE, 139826105319424, 139826105417727, ++ERASE, 139826105319424, 139826105417727, ++STORE, 139826105319424, 139826105417727, ++STORE, 139826100916224, 139826103033855, ++SNULL, 139826100916224, 139826100932607, ++STORE, 139826100932608, 139826103033855, ++STORE, 139826100916224, 139826100932607, ++SNULL, 139826103025663, 139826103033855, ++STORE, 139826100932608, 139826103025663, ++STORE, 139826103025664, 139826103033855, ++ERASE, 139826103025664, 139826103033855, ++STORE, 139826103025664, 139826103033855, ++STORE, 139826098348032, 139826100916223, ++SNULL, 139826098348032, 139826098814975, ++STORE, 139826098814976, 139826100916223, ++STORE, 139826098348032, 139826098814975, ++SNULL, 139826100908031, 139826100916223, ++STORE, 139826098814976, 139826100908031, ++STORE, 139826100908032, 139826100916223, ++ERASE, 139826100908032, 139826100916223, ++STORE, 139826100908032, 139826100916223, ++STORE, 139826096234496, 139826098348031, ++SNULL, 139826096234496, 139826096246783, ++STORE, 139826096246784, 139826098348031, ++STORE, 139826096234496, 139826096246783, ++SNULL, 139826098339839, 139826098348031, ++STORE, 139826096246784, 139826098339839, ++STORE, 139826098339840, 139826098348031, ++ERASE, 139826098339840, 139826098348031, ++STORE, 139826098339840, 139826098348031, ++STORE, 139826094055424, 139826096234495, ++SNULL, 139826094055424, 139826094133247, ++STORE, 139826094133248, 139826096234495, ++STORE, 139826094055424, 139826094133247, ++SNULL, 139826096226303, 139826096234495, ++STORE, 139826094133248, 139826096226303, ++STORE, 139826096226304, 139826096234495, ++ERASE, 139826096226304, 139826096234495, ++STORE, 139826096226304, 139826096234495, ++STORE, 139826136817664, 139826136842239, ++STORE, 139826091937792, 139826094055423, ++SNULL, 139826091937792, 139826091954175, ++STORE, 139826091954176, 139826094055423, ++STORE, 139826091937792, 139826091954175, ++SNULL, 139826094047231, 139826094055423, ++STORE, 139826091954176, 139826094047231, ++STORE, 139826094047232, 139826094055423, ++ERASE, 139826094047232, 139826094055423, ++STORE, 139826094047232, 139826094055423, ++STORE, 139826136809472, 139826136842239, ++SNULL, 139826127781887, 139826127790079, ++STORE, 139826127765504, 139826127781887, ++STORE, 139826127781888, 139826127790079, ++SNULL, 139826094051327, 139826094055423, ++STORE, 139826094047232, 139826094051327, ++STORE, 139826094051328, 139826094055423, ++SNULL, 139826096230399, 139826096234495, ++STORE, 139826096226304, 139826096230399, ++STORE, 139826096230400, 139826096234495, ++SNULL, 139826098343935, 139826098348031, ++STORE, 139826098339840, 139826098343935, ++STORE, 139826098343936, 139826098348031, ++SNULL, 139826130001919, 139826130006015, ++STORE, 139826129997824, 139826130001919, ++STORE, 139826130001920, 139826130006015, ++SNULL, 139826100912127, 139826100916223, ++STORE, 139826100908032, 139826100912127, ++STORE, 139826100912128, 139826100916223, ++SNULL, 139826103029759, 139826103033855, ++STORE, 139826103025664, 139826103029759, ++STORE, 139826103029760, 139826103033855, ++SNULL, 139826105413631, 139826105417727, ++STORE, 139826105319424, 139826105413631, ++STORE, 139826105413632, 139826105417727, ++SNULL, 139826107723775, 139826107727871, ++STORE, 139826107719680, 139826107723775, ++STORE, 139826107723776, 139826107727871, ++SNULL, 139826109857791, 139826109861887, ++STORE, 139826109853696, 139826109857791, ++STORE, 139826109857792, 139826109861887, ++SNULL, 139826113044479, 139826113073151, ++STORE, 139826113036288, 139826113044479, ++STORE, 139826113044480, 139826113073151, ++SNULL, 139826115239935, 139826115244031, ++STORE, 139826115235840, 139826115239935, ++STORE, 139826115239936, 139826115244031, ++SNULL, 139826117492735, 139826117496831, ++STORE, 139826117488640, 139826117492735, ++STORE, 139826117492736, 139826117496831, ++SNULL, 139826119622655, 139826119626751, ++STORE, 139826119618560, 139826119622655, ++STORE, 139826119622656, 139826119626751, ++SNULL, 139826121744383, 139826121748479, ++STORE, 139826121740288, 139826121744383, ++STORE, 139826121744384, 139826121748479, ++SNULL, 139826123997183, 139826124001279, ++STORE, 139826123993088, 139826123997183, ++STORE, 139826123997184, 139826124001279, ++SNULL, 139826132398079, 139826132402175, ++STORE, 139826132381696, 139826132398079, ++STORE, 139826132398080, 139826132402175, ++SNULL, 139826134622207, 139826134626303, ++STORE, 139826134056960, 139826134622207, ++STORE, 139826134622208, 139826134626303, ++SNULL, 94077521309695, 94077521313791, ++STORE, 94077521305600, 94077521309695, ++STORE, 94077521309696, 94077521313791, ++SNULL, 139826136875007, 139826136879103, ++STORE, 139826136870912, 139826136875007, ++STORE, 139826136875008, 139826136879103, ++ERASE, 139826136842240, 139826136862719, ++STORE, 94077554049024, 94077554184191, ++STORE, 139826136543232, 139826136842239, ++STORE, 139826136276992, 139826136842239, ++STORE, 139826136010752, 139826136842239, ++STORE, 139826135744512, 139826136842239, ++SNULL, 139826136543231, 139826136842239, ++STORE, 139826135744512, 139826136543231, ++STORE, 139826136543232, 139826136842239, ++SNULL, 139826136543232, 139826136809471, ++STORE, 139826136809472, 139826136842239, ++STORE, 139826136543232, 139826136809471, ++ }; ++ unsigned long set15[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722061451264, 140737488351231, ++SNULL, 140722061455359, 140737488351231, ++STORE, 140722061451264, 140722061455359, ++STORE, 140722061320192, 140722061455359, ++STORE, 94728600248320, 94728600289279, ++SNULL, 94728600276991, 94728600289279, ++STORE, 94728600248320, 94728600276991, ++STORE, 94728600276992, 94728600289279, ++ERASE, 94728600276992, 94728600289279, ++STORE, 94728600281088, 94728600289279, ++STORE, 139906806779904, 139906809032703, ++SNULL, 139906806923263, 139906809032703, ++STORE, 139906806779904, 139906806923263, ++STORE, 139906806923264, 139906809032703, ++ERASE, 139906806923264, 139906809032703, ++STORE, 139906809020416, 139906809028607, ++STORE, 139906809028608, 139906809032703, ++STORE, 140722061692928, 140722061697023, ++STORE, 140722061680640, 140722061692927, ++STORE, 139906809012224, 139906809020415, ++STORE, 139906804555776, 139906806779903, ++SNULL, 139906806206463, 139906806779903, ++STORE, 139906804555776, 139906806206463, ++STORE, 139906806206464, 139906806779903, ++SNULL, 139906806206464, 139906806775807, ++STORE, 139906806775808, 139906806779903, ++STORE, 139906806206464, 139906806775807, ++ERASE, 139906806206464, 139906806775807, ++STORE, 139906806206464, 139906806775807, ++ERASE, 139906806775808, 139906806779903, ++STORE, 139906806775808, 139906806779903, ++STORE, 139906808991744, 139906809012223, ++STORE, 139906802171904, 139906804555775, ++SNULL, 139906802171904, 139906802438143, ++STORE, 139906802438144, 139906804555775, ++STORE, 139906802171904, 139906802438143, ++SNULL, 139906804531199, 139906804555775, ++STORE, 139906802438144, 139906804531199, ++STORE, 139906804531200, 139906804555775, ++SNULL, 139906804531200, 139906804551679, ++STORE, 139906804551680, 139906804555775, ++STORE, 139906804531200, 139906804551679, ++ERASE, 139906804531200, 139906804551679, ++STORE, 139906804531200, 139906804551679, ++ERASE, 139906804551680, 139906804555775, ++STORE, 139906804551680, 139906804555775, ++STORE, 139906799955968, 139906802171903, ++SNULL, 139906799955968, 139906800054271, ++STORE, 139906800054272, 139906802171903, ++STORE, 139906799955968, 139906800054271, ++SNULL, 139906802147327, 139906802171903, ++STORE, 139906800054272, 139906802147327, ++STORE, 139906802147328, 139906802171903, ++SNULL, 139906802147328, 139906802155519, ++STORE, 139906802155520, 139906802171903, ++STORE, 139906802147328, 139906802155519, ++ERASE, 139906802147328, 139906802155519, ++STORE, 139906802147328, 139906802155519, ++ERASE, 139906802155520, 139906802171903, ++STORE, 139906802155520, 139906802171903, ++STORE, 139906796158976, 139906799955967, ++SNULL, 139906796158976, 139906797817855, ++STORE, 139906797817856, 139906799955967, ++STORE, 139906796158976, 139906797817855, ++SNULL, 139906799915007, 139906799955967, ++STORE, 139906797817856, 139906799915007, ++STORE, 139906799915008, 139906799955967, ++SNULL, 139906799915008, 139906799939583, ++STORE, 139906799939584, 139906799955967, ++STORE, 139906799915008, 139906799939583, ++ERASE, 139906799915008, 139906799939583, ++STORE, 139906799915008, 139906799939583, ++ERASE, 139906799939584, 139906799955967, ++STORE, 139906799939584, 139906799955967, ++STORE, 139906793897984, 139906796158975, ++SNULL, 139906793897984, 139906794049535, ++STORE, 139906794049536, 139906796158975, ++STORE, 139906793897984, 139906794049535, ++SNULL, 139906796142591, 139906796158975, ++STORE, 139906794049536, 139906796142591, ++STORE, 139906796142592, 139906796158975, ++SNULL, 139906796142592, 139906796150783, ++STORE, 139906796150784, 139906796158975, ++STORE, 139906796142592, 139906796150783, ++ERASE, 139906796142592, 139906796150783, ++STORE, 139906796142592, 139906796150783, ++ERASE, 139906796150784, 139906796158975, ++STORE, 139906796150784, 139906796158975, ++STORE, 139906791776256, 139906793897983, ++SNULL, 139906791776256, 139906791792639, ++STORE, 139906791792640, 139906793897983, ++STORE, 139906791776256, 139906791792639, ++SNULL, 139906793889791, 139906793897983, ++STORE, 139906791792640, 139906793889791, ++STORE, 139906793889792, 139906793897983, ++ERASE, 139906793889792, 139906793897983, ++STORE, 139906793889792, 139906793897983, ++STORE, 139906808983552, 139906808991743, ++STORE, 139906789646336, 139906791776255, ++SNULL, 139906789646336, 139906789675007, ++STORE, 139906789675008, 139906791776255, ++STORE, 139906789646336, 139906789675007, ++SNULL, 139906791768063, 139906791776255, ++STORE, 139906789675008, 139906791768063, ++STORE, 139906791768064, 139906791776255, ++ERASE, 139906791768064, 139906791776255, ++STORE, 139906791768064, 139906791776255, ++STORE, 139906787393536, 139906789646335, ++SNULL, 139906787393536, 139906787545087, ++STORE, 139906787545088, 139906789646335, ++STORE, 139906787393536, 139906787545087, ++SNULL, 139906789638143, 139906789646335, ++STORE, 139906787545088, 139906789638143, ++STORE, 139906789638144, 139906789646335, ++ERASE, 139906789638144, 139906789646335, ++STORE, 139906789638144, 139906789646335, ++STORE, 139906785222656, 139906787393535, ++SNULL, 139906785222656, 139906785292287, ++STORE, 139906785292288, 139906787393535, ++STORE, 139906785222656, 139906785292287, ++SNULL, 139906787385343, 139906787393535, ++STORE, 139906785292288, 139906787385343, ++STORE, 139906787385344, 139906787393535, ++ERASE, 139906787385344, 139906787393535, ++STORE, 139906787385344, 139906787393535, ++STORE, 139906782011392, 139906785222655, ++SNULL, 139906782011392, 139906783088639, ++STORE, 139906783088640, 139906785222655, ++STORE, 139906782011392, 139906783088639, ++SNULL, 139906785185791, 139906785222655, ++STORE, 139906783088640, 139906785185791, ++STORE, 139906785185792, 139906785222655, ++ERASE, 139906785185792, 139906785222655, ++STORE, 139906785185792, 139906785222655, ++STORE, 139906779877376, 139906782011391, ++SNULL, 139906779877376, 139906779906047, ++STORE, 139906779906048, 139906782011391, ++STORE, 139906779877376, 139906779906047, ++SNULL, 139906782003199, 139906782011391, ++STORE, 139906779906048, 139906782003199, ++STORE, 139906782003200, 139906782011391, ++ERASE, 139906782003200, 139906782011391, ++STORE, 139906782003200, 139906782011391, ++STORE, 139906777567232, 139906779877375, ++SNULL, 139906777567232, 139906777772031, ++STORE, 139906777772032, 139906779877375, ++STORE, 139906777567232, 139906777772031, ++SNULL, 139906779869183, 139906779877375, ++STORE, 139906777772032, 139906779869183, ++STORE, 139906779869184, 139906779877375, ++ERASE, 139906779869184, 139906779877375, ++STORE, 139906779869184, 139906779877375, ++STORE, 139906808975360, 139906808991743, ++STORE, 139906775183360, 139906777567231, ++SNULL, 139906775183360, 139906775375871, ++STORE, 139906775375872, 139906777567231, ++STORE, 139906775183360, 139906775375871, ++SNULL, 139906777468927, 139906777567231, ++STORE, 139906775375872, 139906777468927, ++STORE, 139906777468928, 139906777567231, ++ERASE, 139906777468928, 139906777567231, ++STORE, 139906777468928, 139906777567231, ++STORE, 139906773065728, 139906775183359, ++SNULL, 139906773065728, 139906773082111, ++STORE, 139906773082112, 139906775183359, ++STORE, 139906773065728, 139906773082111, ++SNULL, 139906775175167, 139906775183359, ++STORE, 139906773082112, 139906775175167, ++STORE, 139906775175168, 139906775183359, ++ERASE, 139906775175168, 139906775183359, ++STORE, 139906775175168, 139906775183359, ++STORE, 139906770497536, 139906773065727, ++SNULL, 139906770497536, 139906770964479, ++STORE, 139906770964480, 139906773065727, ++STORE, 139906770497536, 139906770964479, ++SNULL, 139906773057535, 139906773065727, ++STORE, 139906770964480, 139906773057535, ++STORE, 139906773057536, 139906773065727, ++ERASE, 139906773057536, 139906773065727, ++STORE, 139906773057536, 139906773065727, ++STORE, 139906768384000, 139906770497535, ++SNULL, 139906768384000, 139906768396287, ++STORE, 139906768396288, 139906770497535, ++STORE, 139906768384000, 139906768396287, ++SNULL, 139906770489343, 139906770497535, ++STORE, 139906768396288, 139906770489343, ++STORE, 139906770489344, 139906770497535, ++ERASE, 139906770489344, 139906770497535, ++STORE, 139906770489344, 139906770497535, ++STORE, 139906766204928, 139906768383999, ++SNULL, 139906766204928, 139906766282751, ++STORE, 139906766282752, 139906768383999, ++STORE, 139906766204928, 139906766282751, ++SNULL, 139906768375807, 139906768383999, ++STORE, 139906766282752, 139906768375807, ++STORE, 139906768375808, 139906768383999, ++ERASE, 139906768375808, 139906768383999, ++STORE, 139906768375808, 139906768383999, ++STORE, 139906808967168, 139906808991743, ++STORE, 139906764087296, 139906766204927, ++SNULL, 139906764087296, 139906764103679, ++STORE, 139906764103680, 139906766204927, ++STORE, 139906764087296, 139906764103679, ++SNULL, 139906766196735, 139906766204927, ++STORE, 139906764103680, 139906766196735, ++STORE, 139906766196736, 139906766204927, ++ERASE, 139906766196736, 139906766204927, ++STORE, 139906766196736, 139906766204927, ++STORE, 139906808958976, 139906808991743, ++SNULL, 139906799931391, 139906799939583, ++STORE, 139906799915008, 139906799931391, ++STORE, 139906799931392, 139906799939583, ++SNULL, 139906766200831, 139906766204927, ++STORE, 139906766196736, 139906766200831, ++STORE, 139906766200832, 139906766204927, ++SNULL, 139906768379903, 139906768383999, ++STORE, 139906768375808, 139906768379903, ++STORE, 139906768379904, 139906768383999, ++SNULL, 139906770493439, 139906770497535, ++STORE, 139906770489344, 139906770493439, ++STORE, 139906770493440, 139906770497535, ++SNULL, 139906802151423, 139906802155519, ++STORE, 139906802147328, 139906802151423, ++STORE, 139906802151424, 139906802155519, ++SNULL, 139906773061631, 139906773065727, ++STORE, 139906773057536, 139906773061631, ++STORE, 139906773061632, 139906773065727, ++SNULL, 139906775179263, 139906775183359, ++STORE, 139906775175168, 139906775179263, ++STORE, 139906775179264, 139906775183359, ++SNULL, 139906777563135, 139906777567231, ++STORE, 139906777468928, 139906777563135, ++STORE, 139906777563136, 139906777567231, ++SNULL, 139906779873279, 139906779877375, ++STORE, 139906779869184, 139906779873279, ++STORE, 139906779873280, 139906779877375, ++SNULL, 139906782007295, 139906782011391, ++STORE, 139906782003200, 139906782007295, ++STORE, 139906782007296, 139906782011391, ++SNULL, 139906785193983, 139906785222655, ++STORE, 139906785185792, 139906785193983, ++STORE, 139906785193984, 139906785222655, ++SNULL, 139906787389439, 139906787393535, ++STORE, 139906787385344, 139906787389439, ++STORE, 139906787389440, 139906787393535, ++SNULL, 139906789642239, 139906789646335, ++STORE, 139906789638144, 139906789642239, ++STORE, 139906789642240, 139906789646335, ++SNULL, 139906791772159, 139906791776255, ++STORE, 139906791768064, 139906791772159, ++STORE, 139906791772160, 139906791776255, ++SNULL, 139906793893887, 139906793897983, ++STORE, 139906793889792, 139906793893887, ++STORE, 139906793893888, 139906793897983, ++SNULL, 139906796146687, 139906796150783, ++STORE, 139906796142592, 139906796146687, ++STORE, 139906796146688, 139906796150783, ++SNULL, 139906804547583, 139906804551679, ++STORE, 139906804531200, 139906804547583, ++STORE, 139906804547584, 139906804551679, ++SNULL, 139906806771711, 139906806775807, ++STORE, 139906806206464, 139906806771711, ++STORE, 139906806771712, 139906806775807, ++SNULL, 94728600285183, 94728600289279, ++STORE, 94728600281088, 94728600285183, ++STORE, 94728600285184, 94728600289279, ++SNULL, 139906809024511, 139906809028607, ++STORE, 139906809020416, 139906809024511, ++STORE, 139906809024512, 139906809028607, ++ERASE, 139906808991744, 139906809012223, ++STORE, 94728620138496, 94728620273663, ++STORE, 139906808692736, 139906808991743, ++STORE, 139906808426496, 139906808991743, ++STORE, 139906808160256, 139906808991743, ++STORE, 139906807894016, 139906808991743, ++SNULL, 139906808692735, 139906808991743, ++STORE, 139906807894016, 139906808692735, ++STORE, 139906808692736, 139906808991743, ++SNULL, 139906808692736, 139906808958975, ++STORE, 139906808958976, 139906808991743, ++STORE, 139906808692736, 139906808958975, ++ }; ++ ++ unsigned long set16[] = { ++STORE, 94174808662016, 94174809321471, ++STORE, 94174811414528, 94174811426815, ++STORE, 94174811426816, 94174811430911, ++STORE, 94174811430912, 94174811443199, ++STORE, 94174841700352, 94174841835519, ++STORE, 140173257838592, 140173259497471, ++STORE, 140173259497472, 140173261594623, ++STORE, 140173261594624, 140173261611007, ++STORE, 140173261611008, 140173261619199, ++STORE, 140173261619200, 140173261635583, ++STORE, 140173261635584, 140173261778943, ++STORE, 140173263863808, 140173263871999, ++STORE, 140173263876096, 140173263880191, ++STORE, 140173263880192, 140173263884287, ++STORE, 140173263884288, 140173263888383, ++STORE, 140729801007104, 140729801142271, ++STORE, 140729801617408, 140729801629695, ++STORE, 140729801629696, 140729801633791, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728166858752, 140737488351231, ++SNULL, 140728166862847, 140737488351231, ++STORE, 140728166858752, 140728166862847, ++STORE, 140728166727680, 140728166862847, ++STORE, 93912949866496, 93912950337535, ++SNULL, 93912950288383, 93912950337535, ++STORE, 93912949866496, 93912950288383, ++STORE, 93912950288384, 93912950337535, ++ERASE, 93912950288384, 93912950337535, ++STORE, 93912950292480, 93912950337535, ++STORE, 139921863385088, 139921865637887, ++SNULL, 139921863528447, 139921865637887, ++STORE, 139921863385088, 139921863528447, ++STORE, 139921863528448, 139921865637887, ++ERASE, 139921863528448, 139921865637887, ++STORE, 139921865625600, 139921865633791, ++STORE, 139921865633792, 139921865637887, ++STORE, 140728167899136, 140728167903231, ++STORE, 140728167886848, 140728167899135, ++STORE, 139921865601024, 139921865625599, ++STORE, 139921865592832, 139921865601023, ++STORE, 139921861251072, 139921863385087, ++SNULL, 139921861251072, 139921861279743, ++STORE, 139921861279744, 139921863385087, ++STORE, 139921861251072, 139921861279743, ++SNULL, 139921863376895, 139921863385087, ++STORE, 139921861279744, 139921863376895, ++STORE, 139921863376896, 139921863385087, ++ERASE, 139921863376896, 139921863385087, ++STORE, 139921863376896, 139921863385087, ++STORE, 139921858867200, 139921861251071, ++SNULL, 139921858867200, 139921859133439, ++STORE, 139921859133440, 139921861251071, ++STORE, 139921858867200, 139921859133439, ++SNULL, 139921861226495, 139921861251071, ++STORE, 139921859133440, 139921861226495, ++STORE, 139921861226496, 139921861251071, ++SNULL, 139921861226496, 139921861246975, ++STORE, 139921861246976, 139921861251071, ++STORE, 139921861226496, 139921861246975, ++ERASE, 139921861226496, 139921861246975, ++STORE, 139921861226496, 139921861246975, ++ERASE, 139921861246976, 139921861251071, ++STORE, 139921861246976, 139921861251071, ++STORE, 139921856675840, 139921858867199, ++SNULL, 139921856675840, 139921856765951, ++STORE, 139921856765952, 139921858867199, ++STORE, 139921856675840, 139921856765951, ++SNULL, 139921858859007, 139921858867199, ++STORE, 139921856765952, 139921858859007, ++STORE, 139921858859008, 139921858867199, ++ERASE, 139921858859008, 139921858867199, ++STORE, 139921858859008, 139921858867199, ++STORE, 139921854414848, 139921856675839, ++SNULL, 139921854414848, 139921854566399, ++STORE, 139921854566400, 139921856675839, ++STORE, 139921854414848, 139921854566399, ++SNULL, 139921856659455, 139921856675839, ++STORE, 139921854566400, 139921856659455, ++STORE, 139921856659456, 139921856675839, ++SNULL, 139921856659456, 139921856667647, ++STORE, 139921856667648, 139921856675839, ++STORE, 139921856659456, 139921856667647, ++ERASE, 139921856659456, 139921856667647, ++STORE, 139921856659456, 139921856667647, ++ERASE, 139921856667648, 139921856675839, ++STORE, 139921856667648, 139921856675839, ++STORE, 139921852284928, 139921854414847, ++SNULL, 139921852284928, 139921852313599, ++STORE, 139921852313600, 139921854414847, ++STORE, 139921852284928, 139921852313599, ++SNULL, 139921854406655, 139921854414847, ++STORE, 139921852313600, 139921854406655, ++STORE, 139921854406656, 139921854414847, ++ERASE, 139921854406656, 139921854414847, ++STORE, 139921854406656, 139921854414847, ++STORE, 139921850068992, 139921852284927, ++SNULL, 139921850068992, 139921850167295, ++STORE, 139921850167296, 139921852284927, ++STORE, 139921850068992, 139921850167295, ++SNULL, 139921852260351, 139921852284927, ++STORE, 139921850167296, 139921852260351, ++STORE, 139921852260352, 139921852284927, ++SNULL, 139921852260352, 139921852268543, ++STORE, 139921852268544, 139921852284927, ++STORE, 139921852260352, 139921852268543, ++ERASE, 139921852260352, 139921852268543, ++STORE, 139921852260352, 139921852268543, ++ERASE, 139921852268544, 139921852284927, ++STORE, 139921852268544, 139921852284927, ++STORE, 139921865584640, 139921865601023, ++STORE, 139921846272000, 139921850068991, ++SNULL, 139921846272000, 139921847930879, ++STORE, 139921847930880, 139921850068991, ++STORE, 139921846272000, 139921847930879, ++SNULL, 139921850028031, 139921850068991, ++STORE, 139921847930880, 139921850028031, ++STORE, 139921850028032, 139921850068991, ++SNULL, 139921850028032, 139921850052607, ++STORE, 139921850052608, 139921850068991, ++STORE, 139921850028032, 139921850052607, ++ERASE, 139921850028032, 139921850052607, ++STORE, 139921850028032, 139921850052607, ++ERASE, 139921850052608, 139921850068991, ++STORE, 139921850052608, 139921850068991, ++STORE, 139921844154368, 139921846271999, ++SNULL, 139921844154368, 139921844170751, ++STORE, 139921844170752, 139921846271999, ++STORE, 139921844154368, 139921844170751, ++SNULL, 139921846263807, 139921846271999, ++STORE, 139921844170752, 139921846263807, ++STORE, 139921846263808, 139921846271999, ++ERASE, 139921846263808, 139921846271999, ++STORE, 139921846263808, 139921846271999, ++STORE, 139921842036736, 139921844154367, ++SNULL, 139921842036736, 139921842053119, ++STORE, 139921842053120, 139921844154367, ++STORE, 139921842036736, 139921842053119, ++SNULL, 139921844146175, 139921844154367, ++STORE, 139921842053120, 139921844146175, ++STORE, 139921844146176, 139921844154367, ++ERASE, 139921844146176, 139921844154367, ++STORE, 139921844146176, 139921844154367, ++STORE, 139921839468544, 139921842036735, ++SNULL, 139921839468544, 139921839935487, ++STORE, 139921839935488, 139921842036735, ++STORE, 139921839468544, 139921839935487, ++SNULL, 139921842028543, 139921842036735, ++STORE, 139921839935488, 139921842028543, ++STORE, 139921842028544, 139921842036735, ++ERASE, 139921842028544, 139921842036735, ++STORE, 139921842028544, 139921842036735, ++STORE, 139921837355008, 139921839468543, ++SNULL, 139921837355008, 139921837367295, ++STORE, 139921837367296, 139921839468543, ++STORE, 139921837355008, 139921837367295, ++SNULL, 139921839460351, 139921839468543, ++STORE, 139921837367296, 139921839460351, ++STORE, 139921839460352, 139921839468543, ++ERASE, 139921839460352, 139921839468543, ++STORE, 139921839460352, 139921839468543, ++STORE, 139921865576448, 139921865601023, ++STORE, 139921865564160, 139921865601023, ++SNULL, 139921850044415, 139921850052607, ++STORE, 139921850028032, 139921850044415, ++STORE, 139921850044416, 139921850052607, ++SNULL, 139921839464447, 139921839468543, ++STORE, 139921839460352, 139921839464447, ++STORE, 139921839464448, 139921839468543, ++SNULL, 139921852264447, 139921852268543, ++STORE, 139921852260352, 139921852264447, ++STORE, 139921852264448, 139921852268543, ++SNULL, 139921842032639, 139921842036735, ++STORE, 139921842028544, 139921842032639, ++STORE, 139921842032640, 139921842036735, ++SNULL, 139921844150271, 139921844154367, ++STORE, 139921844146176, 139921844150271, ++STORE, 139921844150272, 139921844154367, ++SNULL, 139921846267903, 139921846271999, ++STORE, 139921846263808, 139921846267903, ++STORE, 139921846267904, 139921846271999, ++SNULL, 139921854410751, 139921854414847, ++STORE, 139921854406656, 139921854410751, ++STORE, 139921854410752, 139921854414847, ++SNULL, 139921856663551, 139921856667647, ++STORE, 139921856659456, 139921856663551, ++STORE, 139921856663552, 139921856667647, ++SNULL, 139921858863103, 139921858867199, ++STORE, 139921858859008, 139921858863103, ++STORE, 139921858863104, 139921858867199, ++SNULL, 139921861242879, 139921861246975, ++STORE, 139921861226496, 139921861242879, ++STORE, 139921861242880, 139921861246975, ++SNULL, 139921863380991, 139921863385087, ++STORE, 139921863376896, 139921863380991, ++STORE, 139921863380992, 139921863385087, ++SNULL, 93912950333439, 93912950337535, ++STORE, 93912950292480, 93912950333439, ++STORE, 93912950333440, 93912950337535, ++SNULL, 139921865629695, 139921865633791, ++STORE, 139921865625600, 139921865629695, ++STORE, 139921865629696, 139921865633791, ++ERASE, 139921865601024, 139921865625599, ++STORE, 93912968110080, 93912968245247, ++STORE, 139921828913152, 139921837355007, ++STORE, 139921865621504, 139921865625599, ++STORE, 139921865617408, 139921865621503, ++STORE, 139921865613312, 139921865617407, ++STORE, 139921865547776, 139921865564159, ++ }; ++ ++ unsigned long set17[] = { ++STORE, 94397057224704, 94397057646591, ++STORE, 94397057650688, 94397057691647, ++STORE, 94397057691648, 94397057695743, ++STORE, 94397075271680, 94397075406847, ++STORE, 139953169051648, 139953169063935, ++STORE, 139953169063936, 139953171156991, ++STORE, 139953171156992, 139953171161087, ++STORE, 139953171161088, 139953171165183, ++STORE, 139953171165184, 139953171632127, ++STORE, 139953171632128, 139953173725183, ++STORE, 139953173725184, 139953173729279, ++STORE, 139953173729280, 139953173733375, ++STORE, 139953173733376, 139953173749759, ++STORE, 139953173749760, 139953175842815, ++STORE, 139953175842816, 139953175846911, ++STORE, 139953175846912, 139953175851007, ++STORE, 139953175851008, 139953175867391, ++STORE, 139953175867392, 139953177960447, ++STORE, 139953177960448, 139953177964543, ++STORE, 139953177964544, 139953177968639, ++STORE, 139953177968640, 139953179627519, ++STORE, 139953179627520, 139953181724671, ++STORE, 139953181724672, 139953181741055, ++STORE, 139953181741056, 139953181749247, ++STORE, 139953181749248, 139953181765631, ++STORE, 139953181765632, 139953181863935, ++STORE, 139953181863936, 139953183956991, ++STORE, 139953183956992, 139953183961087, ++STORE, 139953183961088, 139953183965183, ++STORE, 139953183965184, 139953183981567, ++STORE, 139953183981568, 139953184010239, ++STORE, 139953184010240, 139953186103295, ++STORE, 139953186103296, 139953186107391, ++STORE, 139953186107392, 139953186111487, ++STORE, 139953186111488, 139953186263039, ++STORE, 139953186263040, 139953188356095, ++STORE, 139953188356096, 139953188360191, ++STORE, 139953188360192, 139953188364287, ++STORE, 139953188364288, 139953188372479, ++STORE, 139953188372480, 139953188462591, ++STORE, 139953188462592, 139953190555647, ++STORE, 139953190555648, 139953190559743, ++STORE, 139953190559744, 139953190563839, ++STORE, 139953190563840, 139953190830079, ++STORE, 139953190830080, 139953192923135, ++STORE, 139953192923136, 139953192939519, ++STORE, 139953192939520, 139953192943615, ++STORE, 139953192943616, 139953192947711, ++STORE, 139953192947712, 139953192976383, ++STORE, 139953192976384, 139953195073535, ++STORE, 139953195073536, 139953195077631, ++STORE, 139953195077632, 139953195081727, ++STORE, 139953195081728, 139953195225087, ++STORE, 139953197281280, 139953197318143, ++STORE, 139953197322240, 139953197326335, ++STORE, 139953197326336, 139953197330431, ++STORE, 139953197330432, 139953197334527, ++STORE, 140720477511680, 140720477646847, ++STORE, 140720478302208, 140720478314495, ++STORE, 140720478314496, 140720478318591, ++ }; ++ unsigned long set18[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140724953673728, 140737488351231, ++SNULL, 140724953677823, 140737488351231, ++STORE, 140724953673728, 140724953677823, ++STORE, 140724953542656, 140724953677823, ++STORE, 94675199266816, 94675199311871, ++SNULL, 94675199303679, 94675199311871, ++STORE, 94675199266816, 94675199303679, ++STORE, 94675199303680, 94675199311871, ++ERASE, 94675199303680, 94675199311871, ++STORE, 94675199303680, 94675199311871, ++STORE, 140222970605568, 140222972858367, ++SNULL, 140222970748927, 140222972858367, ++STORE, 140222970605568, 140222970748927, ++STORE, 140222970748928, 140222972858367, ++ERASE, 140222970748928, 140222972858367, ++STORE, 140222972846080, 140222972854271, ++STORE, 140222972854272, 140222972858367, ++STORE, 140724954365952, 140724954370047, ++STORE, 140724954353664, 140724954365951, ++STORE, 140222972841984, 140222972846079, ++STORE, 140222972833792, 140222972841983, ++STORE, 140222968475648, 140222970605567, ++SNULL, 140222968475648, 140222968504319, ++STORE, 140222968504320, 140222970605567, ++STORE, 140222968475648, 140222968504319, ++SNULL, 140222970597375, 140222970605567, ++STORE, 140222968504320, 140222970597375, ++STORE, 140222970597376, 140222970605567, ++ERASE, 140222970597376, 140222970605567, ++STORE, 140222970597376, 140222970605567, ++ }; ++ unsigned long set19[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140725182459904, 140737488351231, ++SNULL, 140725182463999, 140737488351231, ++STORE, 140725182459904, 140725182463999, ++STORE, 140725182328832, 140725182463999, ++STORE, 94730166636544, 94730166763519, ++SNULL, 94730166747135, 94730166763519, ++STORE, 94730166636544, 94730166747135, ++STORE, 94730166747136, 94730166763519, ++ERASE, 94730166747136, 94730166763519, ++STORE, 94730166751232, 94730166763519, ++STORE, 140656834555904, 140656836808703, ++SNULL, 140656834699263, 140656836808703, ++STORE, 140656834555904, 140656834699263, ++STORE, 140656834699264, 140656836808703, ++ERASE, 140656834699264, 140656836808703, ++STORE, 140656836796416, 140656836804607, ++STORE, 140656836804608, 140656836808703, ++STORE, 140725183389696, 140725183393791, ++STORE, 140725183377408, 140725183389695, ++STORE, 140656836788224, 140656836796415, ++STORE, 140656832331776, 140656834555903, ++SNULL, 140656833982463, 140656834555903, ++STORE, 140656832331776, 140656833982463, ++STORE, 140656833982464, 140656834555903, ++SNULL, 140656833982464, 140656834551807, ++STORE, 140656834551808, 140656834555903, ++STORE, 140656833982464, 140656834551807, ++ERASE, 140656833982464, 140656834551807, ++STORE, 140656833982464, 140656834551807, ++ERASE, 140656834551808, 140656834555903, ++STORE, 140656834551808, 140656834555903, ++STORE, 140656836763648, 140656836788223, ++STORE, 140656830070784, 140656832331775, ++SNULL, 140656830070784, 140656830222335, ++STORE, 140656830222336, 140656832331775, ++STORE, 140656830070784, 140656830222335, ++SNULL, 140656832315391, 140656832331775, ++STORE, 140656830222336, 140656832315391, ++STORE, 140656832315392, 140656832331775, ++SNULL, 140656832315392, 140656832323583, ++STORE, 140656832323584, 140656832331775, ++STORE, 140656832315392, 140656832323583, ++ERASE, 140656832315392, 140656832323583, ++STORE, 140656832315392, 140656832323583, ++ERASE, 140656832323584, 140656832331775, ++STORE, 140656832323584, 140656832331775, ++STORE, 140656827940864, 140656830070783, ++SNULL, 140656827940864, 140656827969535, ++STORE, 140656827969536, 140656830070783, ++STORE, 140656827940864, 140656827969535, ++SNULL, 140656830062591, 140656830070783, ++STORE, 140656827969536, 140656830062591, ++STORE, 140656830062592, 140656830070783, ++ERASE, 140656830062592, 140656830070783, ++STORE, 140656830062592, 140656830070783, ++STORE, 140656825724928, 140656827940863, ++SNULL, 140656825724928, 140656825823231, ++STORE, 140656825823232, 140656827940863, ++STORE, 140656825724928, 140656825823231, ++SNULL, 140656827916287, 140656827940863, ++STORE, 140656825823232, 140656827916287, ++STORE, 140656827916288, 140656827940863, ++SNULL, 140656827916288, 140656827924479, ++STORE, 140656827924480, 140656827940863, ++STORE, 140656827916288, 140656827924479, ++ERASE, 140656827916288, 140656827924479, ++STORE, 140656827916288, 140656827924479, ++ERASE, 140656827924480, 140656827940863, ++STORE, 140656827924480, 140656827940863, ++STORE, 140656821927936, 140656825724927, ++SNULL, 140656821927936, 140656823586815, ++STORE, 140656823586816, 140656825724927, ++STORE, 140656821927936, 140656823586815, ++SNULL, 140656825683967, 140656825724927, ++STORE, 140656823586816, 140656825683967, ++STORE, 140656825683968, 140656825724927, ++SNULL, 140656825683968, 140656825708543, ++STORE, 140656825708544, 140656825724927, ++STORE, 140656825683968, 140656825708543, ++ERASE, 140656825683968, 140656825708543, ++STORE, 140656825683968, 140656825708543, ++ERASE, 140656825708544, 140656825724927, ++STORE, 140656825708544, 140656825724927, ++STORE, 140656819806208, 140656821927935, ++SNULL, 140656819806208, 140656819822591, ++STORE, 140656819822592, 140656821927935, ++STORE, 140656819806208, 140656819822591, ++SNULL, 140656821919743, 140656821927935, ++STORE, 140656819822592, 140656821919743, ++STORE, 140656821919744, 140656821927935, ++ERASE, 140656821919744, 140656821927935, ++STORE, 140656821919744, 140656821927935, ++STORE, 140656836755456, 140656836763647, ++STORE, 140656817553408, 140656819806207, ++SNULL, 140656817553408, 140656817704959, ++STORE, 140656817704960, 140656819806207, ++STORE, 140656817553408, 140656817704959, ++SNULL, 140656819798015, 140656819806207, ++STORE, 140656817704960, 140656819798015, ++STORE, 140656819798016, 140656819806207, ++ERASE, 140656819798016, 140656819806207, ++STORE, 140656819798016, 140656819806207, ++STORE, 140656815382528, 140656817553407, ++SNULL, 140656815382528, 140656815452159, ++STORE, 140656815452160, 140656817553407, ++STORE, 140656815382528, 140656815452159, ++SNULL, 140656817545215, 140656817553407, ++STORE, 140656815452160, 140656817545215, ++STORE, 140656817545216, 140656817553407, ++ERASE, 140656817545216, 140656817553407, ++STORE, 140656817545216, 140656817553407, ++STORE, 140656812171264, 140656815382527, ++SNULL, 140656812171264, 140656813248511, ++STORE, 140656813248512, 140656815382527, ++STORE, 140656812171264, 140656813248511, ++SNULL, 140656815345663, 140656815382527, ++STORE, 140656813248512, 140656815345663, ++STORE, 140656815345664, 140656815382527, ++ERASE, 140656815345664, 140656815382527, ++STORE, 140656815345664, 140656815382527, ++STORE, 140656810037248, 140656812171263, ++SNULL, 140656810037248, 140656810065919, ++STORE, 140656810065920, 140656812171263, ++STORE, 140656810037248, 140656810065919, ++SNULL, 140656812163071, 140656812171263, ++STORE, 140656810065920, 140656812163071, ++STORE, 140656812163072, 140656812171263, ++ERASE, 140656812163072, 140656812171263, ++STORE, 140656812163072, 140656812171263, ++STORE, 140656807727104, 140656810037247, ++SNULL, 140656807727104, 140656807931903, ++STORE, 140656807931904, 140656810037247, ++STORE, 140656807727104, 140656807931903, ++SNULL, 140656810029055, 140656810037247, ++STORE, 140656807931904, 140656810029055, ++STORE, 140656810029056, 140656810037247, ++ERASE, 140656810029056, 140656810037247, ++STORE, 140656810029056, 140656810037247, ++STORE, 140656805343232, 140656807727103, ++SNULL, 140656805343232, 140656805535743, ++STORE, 140656805535744, 140656807727103, ++STORE, 140656805343232, 140656805535743, ++SNULL, 140656807628799, 140656807727103, ++STORE, 140656805535744, 140656807628799, ++STORE, 140656807628800, 140656807727103, ++ERASE, 140656807628800, 140656807727103, ++STORE, 140656807628800, 140656807727103, ++STORE, 140656836747264, 140656836763647, ++STORE, 140656802775040, 140656805343231, ++SNULL, 140656802775040, 140656803241983, ++STORE, 140656803241984, 140656805343231, ++STORE, 140656802775040, 140656803241983, ++SNULL, 140656805335039, 140656805343231, ++STORE, 140656803241984, 140656805335039, ++STORE, 140656805335040, 140656805343231, ++ERASE, 140656805335040, 140656805343231, ++STORE, 140656805335040, 140656805343231, ++STORE, 140656800661504, 140656802775039, ++SNULL, 140656800661504, 140656800673791, ++STORE, 140656800673792, 140656802775039, ++STORE, 140656800661504, 140656800673791, ++SNULL, 140656802766847, 140656802775039, ++STORE, 140656800673792, 140656802766847, ++STORE, 140656802766848, 140656802775039, ++ERASE, 140656802766848, 140656802775039, ++STORE, 140656802766848, 140656802775039, ++STORE, 140656798482432, 140656800661503, ++SNULL, 140656798482432, 140656798560255, ++STORE, 140656798560256, 140656800661503, ++STORE, 140656798482432, 140656798560255, ++SNULL, 140656800653311, 140656800661503, ++STORE, 140656798560256, 140656800653311, ++STORE, 140656800653312, 140656800661503, ++ERASE, 140656800653312, 140656800661503, ++STORE, 140656800653312, 140656800661503, ++STORE, 140656796364800, 140656798482431, ++SNULL, 140656796364800, 140656796381183, ++STORE, 140656796381184, 140656798482431, ++STORE, 140656796364800, 140656796381183, ++SNULL, 140656798474239, 140656798482431, ++STORE, 140656796381184, 140656798474239, ++STORE, 140656798474240, 140656798482431, ++ERASE, 140656798474240, 140656798482431, ++STORE, 140656798474240, 140656798482431, ++STORE, 140656836739072, 140656836763647, ++STORE, 140656836726784, 140656836763647, ++SNULL, 140656825700351, 140656825708543, ++STORE, 140656825683968, 140656825700351, ++STORE, 140656825700352, 140656825708543, ++SNULL, 140656798478335, 140656798482431, ++STORE, 140656798474240, 140656798478335, ++STORE, 140656798478336, 140656798482431, ++SNULL, 140656800657407, 140656800661503, ++STORE, 140656800653312, 140656800657407, ++STORE, 140656800657408, 140656800661503, ++SNULL, 140656802770943, 140656802775039, ++STORE, 140656802766848, 140656802770943, ++STORE, 140656802770944, 140656802775039, ++SNULL, 140656827920383, 140656827924479, ++STORE, 140656827916288, 140656827920383, ++STORE, 140656827920384, 140656827924479, ++SNULL, 140656805339135, 140656805343231, ++STORE, 140656805335040, 140656805339135, ++STORE, 140656805339136, 140656805343231, ++SNULL, 140656807723007, 140656807727103, ++STORE, 140656807628800, 140656807723007, ++STORE, 140656807723008, 140656807727103, ++SNULL, 140656810033151, 140656810037247, ++STORE, 140656810029056, 140656810033151, ++STORE, 140656810033152, 140656810037247, ++SNULL, 140656812167167, 140656812171263, ++STORE, 140656812163072, 140656812167167, ++STORE, 140656812167168, 140656812171263, ++SNULL, 140656815353855, 140656815382527, ++STORE, 140656815345664, 140656815353855, ++STORE, 140656815353856, 140656815382527, ++SNULL, 140656817549311, 140656817553407, ++STORE, 140656817545216, 140656817549311, ++STORE, 140656817549312, 140656817553407, ++SNULL, 140656819802111, 140656819806207, ++STORE, 140656819798016, 140656819802111, ++STORE, 140656819802112, 140656819806207, ++SNULL, 140656821923839, 140656821927935, ++STORE, 140656821919744, 140656821923839, ++STORE, 140656821923840, 140656821927935, ++SNULL, 140656830066687, 140656830070783, ++STORE, 140656830062592, 140656830066687, ++STORE, 140656830066688, 140656830070783, ++SNULL, 140656832319487, 140656832323583, ++STORE, 140656832315392, 140656832319487, ++STORE, 140656832319488, 140656832323583, ++SNULL, 140656834547711, 140656834551807, ++STORE, 140656833982464, 140656834547711, ++STORE, 140656834547712, 140656834551807, ++SNULL, 94730166759423, 94730166763519, ++STORE, 94730166751232, 94730166759423, ++STORE, 94730166759424, 94730166763519, ++SNULL, 140656836800511, 140656836804607, ++STORE, 140656836796416, 140656836800511, ++STORE, 140656836800512, 140656836804607, ++ERASE, 140656836763648, 140656836788223, ++STORE, 94730171318272, 94730171453439, ++STORE, 140656836784128, 140656836788223, ++STORE, 140656836780032, 140656836784127, ++STORE, 140656791920640, 140656796364799, ++STORE, 140656836775936, 140656836780031, ++STORE, 140656787476480, 140656791920639, ++STORE, 140656779083776, 140656787476479, ++SNULL, 140656779087871, 140656787476479, ++STORE, 140656779083776, 140656779087871, ++STORE, 140656779087872, 140656787476479, ++STORE, 140656836771840, 140656836775935, ++STORE, 140656774639616, 140656779083775, ++STORE, 140656766246912, 140656774639615, ++SNULL, 140656766251007, 140656774639615, ++STORE, 140656766246912, 140656766251007, ++STORE, 140656766251008, 140656774639615, ++ERASE, 140656791920640, 140656796364799, ++ERASE, 140656836780032, 140656836784127, ++ERASE, 140656787476480, 140656791920639, ++ERASE, 140656836775936, 140656836780031, ++STORE, 140656836780032, 140656836784127, ++STORE, 140656791920640, 140656796364799, ++STORE, 140656836775936, 140656836780031, ++STORE, 140656787476480, 140656791920639, ++ERASE, 140656774639616, 140656779083775, ++ }; ++ unsigned long set20[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735952392192, 140737488351231, ++SNULL, 140735952396287, 140737488351231, ++STORE, 140735952392192, 140735952396287, ++STORE, 140735952261120, 140735952396287, ++STORE, 94849008947200, 94849009414143, ++SNULL, 94849009364991, 94849009414143, ++STORE, 94849008947200, 94849009364991, ++STORE, 94849009364992, 94849009414143, ++ERASE, 94849009364992, 94849009414143, ++STORE, 94849009364992, 94849009414143, ++STORE, 140590397943808, 140590400196607, ++SNULL, 140590398087167, 140590400196607, ++STORE, 140590397943808, 140590398087167, ++STORE, 140590398087168, 140590400196607, ++ERASE, 140590398087168, 140590400196607, ++STORE, 140590400184320, 140590400192511, ++STORE, 140590400192512, 140590400196607, ++STORE, 140735952850944, 140735952855039, ++STORE, 140735952838656, 140735952850943, ++STORE, 140590400180224, 140590400184319, ++STORE, 140590400172032, 140590400180223, ++STORE, 140590395809792, 140590397943807, ++SNULL, 140590395809792, 140590395838463, ++STORE, 140590395838464, 140590397943807, ++STORE, 140590395809792, 140590395838463, ++SNULL, 140590397935615, 140590397943807, ++STORE, 140590395838464, 140590397935615, ++STORE, 140590397935616, 140590397943807, ++ERASE, 140590397935616, 140590397943807, ++STORE, 140590397935616, 140590397943807, ++STORE, 140590393425920, 140590395809791, ++SNULL, 140590393425920, 140590393692159, ++STORE, 140590393692160, 140590395809791, ++STORE, 140590393425920, 140590393692159, ++SNULL, 140590395785215, 140590395809791, ++STORE, 140590393692160, 140590395785215, ++STORE, 140590395785216, 140590395809791, ++SNULL, 140590395785216, 140590395805695, ++STORE, 140590395805696, 140590395809791, ++STORE, 140590395785216, 140590395805695, ++ERASE, 140590395785216, 140590395805695, ++STORE, 140590395785216, 140590395805695, ++ERASE, 140590395805696, 140590395809791, ++STORE, 140590395805696, 140590395809791, ++STORE, 140590391234560, 140590393425919, ++SNULL, 140590391234560, 140590391324671, ++STORE, 140590391324672, 140590393425919, ++STORE, 140590391234560, 140590391324671, ++SNULL, 140590393417727, 140590393425919, ++STORE, 140590391324672, 140590393417727, ++STORE, 140590393417728, 140590393425919, ++ERASE, 140590393417728, 140590393425919, ++STORE, 140590393417728, 140590393425919, ++STORE, 140590388973568, 140590391234559, ++SNULL, 140590388973568, 140590389125119, ++STORE, 140590389125120, 140590391234559, ++STORE, 140590388973568, 140590389125119, ++SNULL, 140590391218175, 140590391234559, ++STORE, 140590389125120, 140590391218175, ++STORE, 140590391218176, 140590391234559, ++SNULL, 140590391218176, 140590391226367, ++STORE, 140590391226368, 140590391234559, ++STORE, 140590391218176, 140590391226367, ++ERASE, 140590391218176, 140590391226367, ++STORE, 140590391218176, 140590391226367, ++ERASE, 140590391226368, 140590391234559, ++STORE, 140590391226368, 140590391234559, ++STORE, 140590386843648, 140590388973567, ++SNULL, 140590386843648, 140590386872319, ++STORE, 140590386872320, 140590388973567, ++STORE, 140590386843648, 140590386872319, ++SNULL, 140590388965375, 140590388973567, ++STORE, 140590386872320, 140590388965375, ++STORE, 140590388965376, 140590388973567, ++ERASE, 140590388965376, 140590388973567, ++STORE, 140590388965376, 140590388973567, ++STORE, 140590384627712, 140590386843647, ++SNULL, 140590384627712, 140590384726015, ++STORE, 140590384726016, 140590386843647, ++STORE, 140590384627712, 140590384726015, ++SNULL, 140590386819071, 140590386843647, ++STORE, 140590384726016, 140590386819071, ++STORE, 140590386819072, 140590386843647, ++SNULL, 140590386819072, 140590386827263, ++STORE, 140590386827264, 140590386843647, ++STORE, 140590386819072, 140590386827263, ++ERASE, 140590386819072, 140590386827263, ++STORE, 140590386819072, 140590386827263, ++ERASE, 140590386827264, 140590386843647, ++STORE, 140590386827264, 140590386843647, ++STORE, 140590400163840, 140590400180223, ++STORE, 140590380830720, 140590384627711, ++SNULL, 140590380830720, 140590382489599, ++STORE, 140590382489600, 140590384627711, ++STORE, 140590380830720, 140590382489599, ++SNULL, 140590384586751, 140590384627711, ++STORE, 140590382489600, 140590384586751, ++STORE, 140590384586752, 140590384627711, ++SNULL, 140590384586752, 140590384611327, ++STORE, 140590384611328, 140590384627711, ++STORE, 140590384586752, 140590384611327, ++ERASE, 140590384586752, 140590384611327, ++STORE, 140590384586752, 140590384611327, ++ERASE, 140590384611328, 140590384627711, ++STORE, 140590384611328, 140590384627711, ++STORE, 140590378713088, 140590380830719, ++SNULL, 140590378713088, 140590378729471, ++STORE, 140590378729472, 140590380830719, ++STORE, 140590378713088, 140590378729471, ++SNULL, 140590380822527, 140590380830719, ++STORE, 140590378729472, 140590380822527, ++STORE, 140590380822528, 140590380830719, ++ERASE, 140590380822528, 140590380830719, ++STORE, 140590380822528, 140590380830719, ++STORE, 140590376595456, 140590378713087, ++SNULL, 140590376595456, 140590376611839, ++STORE, 140590376611840, 140590378713087, ++STORE, 140590376595456, 140590376611839, ++SNULL, 140590378704895, 140590378713087, ++STORE, 140590376611840, 140590378704895, ++STORE, 140590378704896, 140590378713087, ++ERASE, 140590378704896, 140590378713087, ++STORE, 140590378704896, 140590378713087, ++STORE, 140590374027264, 140590376595455, ++SNULL, 140590374027264, 140590374494207, ++STORE, 140590374494208, 140590376595455, ++STORE, 140590374027264, 140590374494207, ++SNULL, 140590376587263, 140590376595455, ++STORE, 140590374494208, 140590376587263, ++STORE, 140590376587264, 140590376595455, ++ERASE, 140590376587264, 140590376595455, ++STORE, 140590376587264, 140590376595455, ++STORE, 140590371913728, 140590374027263, ++SNULL, 140590371913728, 140590371926015, ++STORE, 140590371926016, 140590374027263, ++STORE, 140590371913728, 140590371926015, ++SNULL, 140590374019071, 140590374027263, ++STORE, 140590371926016, 140590374019071, ++STORE, 140590374019072, 140590374027263, ++ERASE, 140590374019072, 140590374027263, ++STORE, 140590374019072, 140590374027263, ++STORE, 140590400155648, 140590400180223, ++STORE, 140590400143360, 140590400180223, ++SNULL, 140590384603135, 140590384611327, ++STORE, 140590384586752, 140590384603135, ++STORE, 140590384603136, 140590384611327, ++SNULL, 140590374023167, 140590374027263, ++STORE, 140590374019072, 140590374023167, ++STORE, 140590374023168, 140590374027263, ++SNULL, 140590386823167, 140590386827263, ++STORE, 140590386819072, 140590386823167, ++STORE, 140590386823168, 140590386827263, ++SNULL, 140590376591359, 140590376595455, ++ }; ++ unsigned long set21[] = { ++STORE, 93874710941696, 93874711363583, ++STORE, 93874711367680, 93874711408639, ++STORE, 93874711408640, 93874711412735, ++STORE, 93874720989184, 93874721124351, ++STORE, 140708365086720, 140708365099007, ++STORE, 140708365099008, 140708367192063, ++STORE, 140708367192064, 140708367196159, ++STORE, 140708367196160, 140708367200255, ++STORE, 140708367200256, 140708367667199, ++STORE, 140708367667200, 140708369760255, ++STORE, 140708369760256, 140708369764351, ++STORE, 140708369764352, 140708369768447, ++STORE, 140708369768448, 140708369784831, ++STORE, 140708369784832, 140708371877887, ++STORE, 140708371877888, 140708371881983, ++STORE, 140708371881984, 140708371886079, ++STORE, 140708371886080, 140708371902463, ++STORE, 140708371902464, 140708373995519, ++STORE, 140708373995520, 140708373999615, ++STORE, 140708373999616, 140708374003711, ++STORE, 140708374003712, 140708375662591, ++STORE, 140708375662592, 140708377759743, ++STORE, 140708377759744, 140708377776127, ++STORE, 140708377776128, 140708377784319, ++STORE, 140708377784320, 140708377800703, ++STORE, 140708377800704, 140708377899007, ++STORE, 140708377899008, 140708379992063, ++STORE, 140708379992064, 140708379996159, ++STORE, 140708379996160, 140708380000255, ++STORE, 140708380000256, 140708380016639, ++STORE, 140708380016640, 140708380045311, ++STORE, 140708380045312, 140708382138367, ++STORE, 140708382138368, 140708382142463, ++STORE, 140708382142464, 140708382146559, ++STORE, 140708382146560, 140708382298111, ++STORE, 140708382298112, 140708384391167, ++STORE, 140708384391168, 140708384395263, ++STORE, 140708384395264, 140708384399359, ++STORE, 140708384399360, 140708384407551, ++STORE, 140708384407552, 140708384497663, ++STORE, 140708384497664, 140708386590719, ++STORE, 140708386590720, 140708386594815, ++STORE, 140708386594816, 140708386598911, ++STORE, 140708386598912, 140708386865151, ++STORE, 140708386865152, 140708388958207, ++STORE, 140708388958208, 140708388974591, ++STORE, 140708388974592, 140708388978687, ++STORE, 140708388978688, 140708388982783, ++STORE, 140708388982784, 140708389011455, ++STORE, 140708389011456, 140708391108607, ++STORE, 140708391108608, 140708391112703, ++STORE, 140708391112704, 140708391116799, ++STORE, 140708391116800, 140708391260159, ++STORE, 140708393291776, 140708393308159, ++STORE, 140708393308160, 140708393312255, ++STORE, 140708393312256, 140708393316351, ++STORE, 140708393316352, 140708393353215, ++STORE, 140708393353216, 140708393357311, ++STORE, 140708393357312, 140708393361407, ++STORE, 140708393361408, 140708393365503, ++STORE, 140708393365504, 140708393369599, ++STORE, 140730557042688, 140730557177855, ++STORE, 140730557235200, 140730557247487, ++STORE, 140730557247488, 140730557251583, ++ERASE, 140708393353216, 140708393357311, ++ERASE, 140708393312256, 140708393316351, ++ERASE, 140708393308160, 140708393312255, ++ERASE, 140708393291776, 140708393308159, ++ }; ++ unsigned long set22[] = { ++STORE, 93951397134336, 93951397183487, ++STORE, 93951397183488, 93951397728255, ++STORE, 93951397728256, 93951397826559, ++STORE, 93951397826560, 93951397842943, ++STORE, 93951397842944, 93951397847039, ++STORE, 93951425974272, 93951426109439, ++STORE, 140685152665600, 140685152677887, ++STORE, 140685152677888, 140685152829439, ++STORE, 140685152829440, 140685154181119, ++STORE, 140685154181120, 140685154484223, ++STORE, 140685154484224, 140685154496511, ++STORE, 140685154496512, 140685154508799, ++STORE, 140685154508800, 140685154525183, ++STORE, 140685154525184, 140685154541567, ++STORE, 140685154541568, 140685154590719, ++STORE, 140685154590720, 140685154603007, ++STORE, 140685154603008, 140685154607103, ++STORE, 140685154607104, 140685154611199, ++STORE, 140685154611200, 140685154615295, ++STORE, 140685154615296, 140685154631679, ++STORE, 140685154639872, 140685154643967, ++STORE, 140685154643968, 140685154766847, ++STORE, 140685154766848, 140685154799615, ++STORE, 140685154803712, 140685154807807, ++STORE, 140685154807808, 140685154811903, ++STORE, 140685154811904, 140685154815999, ++STORE, 140722188902400, 140722189037567, ++STORE, 140722189512704, 140722189524991, ++STORE, 140722189524992, 140722189529087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733429354496, 140737488351231, ++SNULL, 140733429358591, 140737488351231, ++STORE, 140733429354496, 140733429358591, ++STORE, 140733429223424, 140733429358591, ++STORE, 94526683537408, 94526683660287, ++SNULL, 94526683553791, 94526683660287, ++STORE, 94526683537408, 94526683553791, ++STORE, 94526683553792, 94526683660287, ++ERASE, 94526683553792, 94526683660287, ++STORE, 94526683553792, 94526683623423, ++STORE, 94526683623424, 94526683647999, ++STORE, 94526683652096, 94526683660287, ++STORE, 140551363747840, 140551363923967, ++SNULL, 140551363751935, 140551363923967, ++STORE, 140551363747840, 140551363751935, ++STORE, 140551363751936, 140551363923967, ++ERASE, 140551363751936, 140551363923967, ++STORE, 140551363751936, 140551363874815, ++STORE, 140551363874816, 140551363907583, ++STORE, 140551363911680, 140551363919871, ++STORE, 140551363919872, 140551363923967, ++STORE, 140733429690368, 140733429694463, ++STORE, 140733429678080, 140733429690367, ++STORE, 140551363739648, 140551363747839, ++STORE, 140551363731456, 140551363739647, ++STORE, 140551363379200, 140551363731455, ++SNULL, 140551363379200, 140551363420159, ++STORE, 140551363420160, 140551363731455, ++STORE, 140551363379200, 140551363420159, ++SNULL, 140551363706879, 140551363731455, ++STORE, 140551363420160, 140551363706879, ++STORE, 140551363706880, 140551363731455, ++SNULL, 140551363420160, 140551363637247, ++STORE, 140551363637248, 140551363706879, ++STORE, 140551363420160, 140551363637247, ++ERASE, 140551363420160, 140551363637247, ++STORE, 140551363420160, 140551363637247, ++SNULL, 140551363637248, 140551363702783, ++STORE, 140551363702784, 140551363706879, ++STORE, 140551363637248, 140551363702783, ++ERASE, 140551363637248, 140551363702783, ++STORE, 140551363637248, 140551363702783, ++ERASE, 140551363706880, 140551363731455, ++STORE, 140551363706880, 140551363731455, ++STORE, 140551361531904, 140551363379199, ++SNULL, 140551361683455, 140551363379199, ++STORE, 140551361531904, 140551361683455, ++STORE, 140551361683456, 140551363379199, ++SNULL, 140551361683456, 140551363035135, ++STORE, 140551363035136, 140551363379199, ++STORE, 140551361683456, 140551363035135, ++ERASE, 140551361683456, 140551363035135, ++STORE, 140551361683456, 140551363035135, ++SNULL, 140551363035136, 140551363338239, ++STORE, 140551363338240, 140551363379199, ++STORE, 140551363035136, 140551363338239, ++ERASE, 140551363035136, 140551363338239, ++STORE, 140551363035136, 140551363379199, ++SNULL, 140551363338239, 140551363379199, ++STORE, 140551363035136, 140551363338239, ++STORE, 140551363338240, 140551363379199, ++SNULL, 140551363338240, 140551363362815, ++STORE, 140551363362816, 140551363379199, ++STORE, 140551363338240, 140551363362815, ++ERASE, 140551363338240, 140551363362815, ++STORE, 140551363338240, 140551363362815, ++ERASE, 140551363362816, 140551363379199, ++STORE, 140551363362816, 140551363379199, ++STORE, 140551361519616, 140551361531903, ++SNULL, 140551363350527, 140551363362815, ++STORE, 140551363338240, 140551363350527, ++STORE, 140551363350528, 140551363362815, ++SNULL, 140551363727359, 140551363731455, ++STORE, 140551363706880, 140551363727359, ++STORE, 140551363727360, 140551363731455, ++SNULL, 94526683656191, 94526683660287, ++STORE, 94526683652096, 94526683656191, ++STORE, 94526683656192, 94526683660287, ++SNULL, 140551363915775, 140551363919871, ++STORE, 140551363911680, 140551363915775, ++STORE, 140551363915776, 140551363919871, ++ERASE, 140551363739648, 140551363747839, ++STORE, 94526715490304, 94526715625471, ++STORE, 140551361253376, 140551361531903, ++STORE, 140551360987136, 140551361531903, ++STORE, 140551360720896, 140551361531903, ++STORE, 140551360454656, 140551361531903, ++SNULL, 140551361253375, 140551361531903, ++STORE, 140551360454656, 140551361253375, ++STORE, 140551361253376, 140551361531903, ++SNULL, 140551361253376, 140551361519615, ++STORE, 140551361519616, 140551361531903, ++STORE, 140551361253376, 140551361519615, ++ERASE, 140551361253376, 140551361519615, ++ }; ++ ++ unsigned long set23[] = { ++STORE, 94014447943680, 94014448156671, ++STORE, 94014450253824, 94014450257919, ++STORE, 94014450257920, 94014450266111, ++STORE, 94014450266112, 94014450278399, ++STORE, 94014464225280, 94014464630783, ++STORE, 139761764306944, 139761765965823, ++STORE, 139761765965824, 139761768062975, ++STORE, 139761768062976, 139761768079359, ++STORE, 139761768079360, 139761768087551, ++STORE, 139761768087552, 139761768103935, ++STORE, 139761768103936, 139761768116223, ++STORE, 139761768116224, 139761770209279, ++STORE, 139761770209280, 139761770213375, ++STORE, 139761770213376, 139761770217471, ++STORE, 139761770217472, 139761770360831, ++STORE, 139761770729472, 139761772412927, ++STORE, 139761772412928, 139761772429311, ++STORE, 139761772457984, 139761772462079, ++STORE, 139761772462080, 139761772466175, ++STORE, 139761772466176, 139761772470271, ++STORE, 140724336517120, 140724336652287, ++STORE, 140724336955392, 140724336967679, ++STORE, 140724336967680, 140724336971775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140721840295936, 140737488351231, ++SNULL, 140721840300031, 140737488351231, ++STORE, 140721840295936, 140721840300031, ++STORE, 140721840164864, 140721840300031, ++STORE, 93937913667584, 93937915830271, ++SNULL, 93937913729023, 93937915830271, ++STORE, 93937913667584, 93937913729023, ++STORE, 93937913729024, 93937915830271, ++ERASE, 93937913729024, 93937915830271, ++STORE, 93937915822080, 93937915830271, ++STORE, 140598835335168, 140598837587967, ++SNULL, 140598835478527, 140598837587967, ++STORE, 140598835335168, 140598835478527, ++STORE, 140598835478528, 140598837587967, ++ERASE, 140598835478528, 140598837587967, ++STORE, 140598837575680, 140598837583871, ++STORE, 140598837583872, 140598837587967, ++STORE, 140721841086464, 140721841090559, ++STORE, 140721841074176, 140721841086463, ++STORE, 140598837547008, 140598837575679, ++STORE, 140598837538816, 140598837547007, ++STORE, 140598831538176, 140598835335167, ++SNULL, 140598831538176, 140598833197055, ++STORE, 140598833197056, 140598835335167, ++STORE, 140598831538176, 140598833197055, ++SNULL, 140598835294207, 140598835335167, ++STORE, 140598833197056, 140598835294207, ++STORE, 140598835294208, 140598835335167, ++SNULL, 140598835294208, 140598835318783, ++STORE, 140598835318784, 140598835335167, ++STORE, 140598835294208, 140598835318783, ++ERASE, 140598835294208, 140598835318783, ++STORE, 140598835294208, 140598835318783, ++ERASE, 140598835318784, 140598835335167, ++STORE, 140598835318784, 140598835335167, ++SNULL, 140598835310591, 140598835318783, ++STORE, 140598835294208, 140598835310591, ++STORE, 140598835310592, 140598835318783, ++SNULL, 93937915826175, 93937915830271, ++STORE, 93937915822080, 93937915826175, ++STORE, 93937915826176, 93937915830271, ++SNULL, 140598837579775, 140598837583871, ++STORE, 140598837575680, 140598837579775, ++STORE, 140598837579776, 140598837583871, ++ERASE, 140598837547008, 140598837575679, ++STORE, 93937929179136, 93937929314303, ++STORE, 140598835855360, 140598837538815, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728187723776, 140737488351231, ++SNULL, 140728187727871, 140737488351231, ++STORE, 140728187723776, 140728187727871, ++STORE, 140728187592704, 140728187727871, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140583951437824, 140583953690623, ++SNULL, 140583951581183, 140583953690623, ++STORE, 140583951437824, 140583951581183, ++STORE, 140583951581184, 140583953690623, ++ERASE, 140583951581184, 140583953690623, ++STORE, 140583953678336, 140583953686527, ++STORE, 140583953686528, 140583953690623, ++STORE, 140728189116416, 140728189120511, ++STORE, 140728189104128, 140728189116415, ++STORE, 140583953649664, 140583953678335, ++STORE, 140583953641472, 140583953649663, ++STORE, 140583948275712, 140583951437823, ++SNULL, 140583948275712, 140583949336575, ++STORE, 140583949336576, 140583951437823, ++STORE, 140583948275712, 140583949336575, ++SNULL, 140583951429631, 140583951437823, ++STORE, 140583949336576, 140583951429631, ++STORE, 140583951429632, 140583951437823, ++ERASE, 140583951429632, 140583951437823, ++STORE, 140583951429632, 140583951437823, ++STORE, 140583944478720, 140583948275711, ++SNULL, 140583944478720, 140583946137599, ++STORE, 140583946137600, 140583948275711, ++STORE, 140583944478720, 140583946137599, ++SNULL, 140583948234751, 140583948275711, ++STORE, 140583946137600, 140583948234751, ++STORE, 140583948234752, 140583948275711, ++SNULL, 140583948234752, 140583948259327, ++STORE, 140583948259328, 140583948275711, ++STORE, 140583948234752, 140583948259327, ++ERASE, 140583948234752, 140583948259327, ++STORE, 140583948234752, 140583948259327, ++ERASE, 140583948259328, 140583948275711, ++STORE, 140583948259328, 140583948275711, ++STORE, 140583953629184, 140583953649663, ++SNULL, 140583948251135, 140583948259327, ++STORE, 140583948234752, 140583948251135, ++STORE, 140583948251136, 140583948259327, ++SNULL, 140583951433727, 140583951437823, ++STORE, 140583951429632, 140583951433727, ++STORE, 140583951433728, 140583951437823, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140583953682431, 140583953686527, ++STORE, 140583953678336, 140583953682431, ++STORE, 140583953682432, 140583953686527, ++ERASE, 140583953649664, 140583953678335, ++STORE, 17821696, 17956863, ++STORE, 17821696, 18104319, ++STORE, 140583951945728, 140583953629183, ++STORE, 94014447943680, 94014448156671, ++STORE, 94014450253824, 94014450257919, ++STORE, 94014450257920, 94014450266111, ++STORE, 94014450266112, 94014450278399, ++STORE, 94014464225280, 94014465196031, ++STORE, 139761764306944, 139761765965823, ++STORE, 139761765965824, 139761768062975, ++STORE, 139761768062976, 139761768079359, ++STORE, 139761768079360, 139761768087551, ++STORE, 139761768087552, 139761768103935, ++STORE, 139761768103936, 139761768116223, ++STORE, 139761768116224, 139761770209279, ++STORE, 139761770209280, 139761770213375, ++STORE, 139761770213376, 139761770217471, ++STORE, 139761770217472, 139761770360831, ++STORE, 139761770729472, 139761772412927, ++STORE, 139761772412928, 139761772429311, ++STORE, 139761772457984, 139761772462079, ++STORE, 139761772462080, 139761772466175, ++STORE, 139761772466176, 139761772470271, ++STORE, 140724336517120, 140724336652287, ++STORE, 140724336955392, 140724336967679, ++STORE, 140724336967680, 140724336971775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726063296512, 140737488351231, ++SNULL, 140726063300607, 140737488351231, ++STORE, 140726063296512, 140726063300607, ++STORE, 140726063165440, 140726063300607, ++STORE, 94016795934720, 94016798158847, ++SNULL, 94016796045311, 94016798158847, ++STORE, 94016795934720, 94016796045311, ++STORE, 94016796045312, 94016798158847, ++ERASE, 94016796045312, 94016798158847, ++STORE, 94016798138368, 94016798150655, ++STORE, 94016798150656, 94016798158847, ++STORE, 139975915966464, 139975918219263, ++SNULL, 139975916109823, 139975918219263, ++STORE, 139975915966464, 139975916109823, ++STORE, 139975916109824, 139975918219263, ++ERASE, 139975916109824, 139975918219263, ++STORE, 139975918206976, 139975918215167, ++STORE, 139975918215168, 139975918219263, ++STORE, 140726064541696, 140726064545791, ++STORE, 140726064529408, 140726064541695, ++STORE, 139975918178304, 139975918206975, ++STORE, 139975918170112, 139975918178303, ++STORE, 139975912169472, 139975915966463, ++SNULL, 139975912169472, 139975913828351, ++STORE, 139975913828352, 139975915966463, ++STORE, 139975912169472, 139975913828351, ++SNULL, 139975915925503, 139975915966463, ++STORE, 139975913828352, 139975915925503, ++STORE, 139975915925504, 139975915966463, ++SNULL, 139975915925504, 139975915950079, ++STORE, 139975915950080, 139975915966463, ++STORE, 139975915925504, 139975915950079, ++ERASE, 139975915925504, 139975915950079, ++STORE, 139975915925504, 139975915950079, ++ERASE, 139975915950080, 139975915966463, ++STORE, 139975915950080, 139975915966463, ++SNULL, 139975915941887, 139975915950079, ++STORE, 139975915925504, 139975915941887, ++STORE, 139975915941888, 139975915950079, ++SNULL, 94016798146559, 94016798150655, ++STORE, 94016798138368, 94016798146559, ++STORE, 94016798146560, 94016798150655, ++SNULL, 139975918211071, 139975918215167, ++STORE, 139975918206976, 139975918211071, ++STORE, 139975918211072, 139975918215167, ++ERASE, 139975918178304, 139975918206975, ++STORE, 94016804925440, 94016805060607, ++STORE, 94596177661952, 94596177772543, ++STORE, 94596179865600, 94596179873791, ++STORE, 94596179873792, 94596179877887, ++STORE, 94596179877888, 94596179886079, ++STORE, 94596211597312, 94596211863551, ++STORE, 140127351840768, 140127353499647, ++STORE, 140127353499648, 140127355596799, ++STORE, 140127355596800, 140127355613183, ++STORE, 140127355613184, 140127355621375, ++STORE, 140127355621376, 140127355637759, ++STORE, 140127355637760, 140127355781119, ++STORE, 140127357841408, 140127357849599, ++STORE, 140127357878272, 140127357882367, ++STORE, 140127357882368, 140127357886463, ++STORE, 140127357886464, 140127357890559, ++STORE, 140726167252992, 140726167392255, ++STORE, 140726167838720, 140726167851007, ++STORE, 140726167851008, 140726167855103, ++STORE, 140737488347136, 140737488351231, ++STORE, 140731874017280, 140737488351231, ++SNULL, 140731874021375, 140737488351231, ++STORE, 140731874017280, 140731874021375, ++STORE, 140731873886208, 140731874021375, ++STORE, 94178682265600, 94178684489727, ++SNULL, 94178682376191, 94178684489727, ++STORE, 94178682265600, 94178682376191, ++STORE, 94178682376192, 94178684489727, ++ERASE, 94178682376192, 94178684489727, ++STORE, 94178684469248, 94178684481535, ++STORE, 94178684481536, 94178684489727, ++STORE, 140460853403648, 140460855656447, ++SNULL, 140460853547007, 140460855656447, ++STORE, 140460853403648, 140460853547007, ++STORE, 140460853547008, 140460855656447, ++ERASE, 140460853547008, 140460855656447, ++STORE, 140460855644160, 140460855652351, ++STORE, 140460855652352, 140460855656447, ++STORE, 140731874103296, 140731874107391, ++STORE, 140731874091008, 140731874103295, ++STORE, 140460855615488, 140460855644159, ++STORE, 140460855607296, 140460855615487, ++STORE, 140460849606656, 140460853403647, ++SNULL, 140460849606656, 140460851265535, ++STORE, 140460851265536, 140460853403647, ++STORE, 140460849606656, 140460851265535, ++SNULL, 140460853362687, 140460853403647, ++STORE, 140460851265536, 140460853362687, ++STORE, 140460853362688, 140460853403647, ++SNULL, 140460853362688, 140460853387263, ++STORE, 140460853387264, 140460853403647, ++STORE, 140460853362688, 140460853387263, ++ERASE, 140460853362688, 140460853387263, ++STORE, 140460853362688, 140460853387263, ++ERASE, 140460853387264, 140460853403647, ++STORE, 140460853387264, 140460853403647, ++SNULL, 140460853379071, 140460853387263, ++STORE, 140460853362688, 140460853379071, ++STORE, 140460853379072, 140460853387263, ++SNULL, 94178684477439, 94178684481535, ++STORE, 94178684469248, 94178684477439, ++STORE, 94178684477440, 94178684481535, ++SNULL, 140460855648255, 140460855652351, ++STORE, 140460855644160, 140460855648255, ++STORE, 140460855648256, 140460855652351, ++ERASE, 140460855615488, 140460855644159, ++STORE, 94178692063232, 94178692198399, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733096603648, 140737488351231, ++SNULL, 140733096611839, 140737488351231, ++STORE, 140733096603648, 140733096611839, ++STORE, 140733096472576, 140733096611839, ++STORE, 94796716122112, 94796718325759, ++SNULL, 94796716224511, 94796718325759, ++STORE, 94796716122112, 94796716224511, ++STORE, 94796716224512, 94796718325759, ++ERASE, 94796716224512, 94796718325759, ++STORE, 94796718317568, 94796718325759, ++STORE, 139667892793344, 139667895046143, ++SNULL, 139667892936703, 139667895046143, ++STORE, 139667892793344, 139667892936703, ++STORE, 139667892936704, 139667895046143, ++ERASE, 139667892936704, 139667895046143, ++STORE, 139667895033856, 139667895042047, ++STORE, 139667895042048, 139667895046143, ++STORE, 140733096857600, 140733096861695, ++STORE, 140733096845312, 140733096857599, ++STORE, 139667895005184, 139667895033855, ++STORE, 139667894996992, 139667895005183, ++STORE, 139667890532352, 139667892793343, ++SNULL, 139667890532352, 139667890683903, ++STORE, 139667890683904, 139667892793343, ++STORE, 139667890532352, 139667890683903, ++SNULL, 139667892776959, 139667892793343, ++STORE, 139667890683904, 139667892776959, ++STORE, 139667892776960, 139667892793343, ++SNULL, 139667892776960, 139667892785151, ++STORE, 139667892785152, 139667892793343, ++STORE, 139667892776960, 139667892785151, ++ERASE, 139667892776960, 139667892785151, ++STORE, 139667892776960, 139667892785151, ++ERASE, 139667892785152, 139667892793343, ++STORE, 139667892785152, 139667892793343, ++STORE, 139667886735360, 139667890532351, ++SNULL, 139667886735360, 139667888394239, ++STORE, 139667888394240, 139667890532351, ++STORE, 139667886735360, 139667888394239, ++SNULL, 139667890491391, 139667890532351, ++STORE, 139667888394240, 139667890491391, ++STORE, 139667890491392, 139667890532351, ++SNULL, 139667890491392, 139667890515967, ++STORE, 139667890515968, 139667890532351, ++STORE, 139667890491392, 139667890515967, ++ERASE, 139667890491392, 139667890515967, ++STORE, 139667890491392, 139667890515967, ++ERASE, 139667890515968, 139667890532351, ++STORE, 139667890515968, 139667890532351, ++STORE, 139667884167168, 139667886735359, ++SNULL, 139667884167168, 139667884634111, ++STORE, 139667884634112, 139667886735359, ++STORE, 139667884167168, 139667884634111, ++SNULL, 139667886727167, 139667886735359, ++STORE, 139667884634112, 139667886727167, ++STORE, 139667886727168, 139667886735359, ++ERASE, 139667886727168, 139667886735359, ++STORE, 139667886727168, 139667886735359, ++STORE, 139667882053632, 139667884167167, ++SNULL, 139667882053632, 139667882065919, ++STORE, 139667882065920, 139667884167167, ++STORE, 139667882053632, 139667882065919, ++SNULL, 139667884158975, 139667884167167, ++STORE, 139667882065920, 139667884158975, ++STORE, 139667884158976, 139667884167167, ++ERASE, 139667884158976, 139667884167167, ++STORE, 139667884158976, 139667884167167, ++STORE, 139667879837696, 139667882053631, ++SNULL, 139667879837696, 139667879935999, ++STORE, 139667879936000, 139667882053631, ++STORE, 139667879837696, 139667879935999, ++SNULL, 139667882029055, 139667882053631, ++STORE, 139667879936000, 139667882029055, ++STORE, 139667882029056, 139667882053631, ++SNULL, 139667882029056, 139667882037247, ++STORE, 139667882037248, 139667882053631, ++STORE, 139667882029056, 139667882037247, ++ERASE, 139667882029056, 139667882037247, ++STORE, 139667882029056, 139667882037247, ++ERASE, 139667882037248, 139667882053631, ++STORE, 139667882037248, 139667882053631, ++STORE, 139667894988800, 139667895005183, ++SNULL, 139667890507775, 139667890515967, ++STORE, 139667890491392, 139667890507775, ++STORE, 139667890507776, 139667890515967, ++SNULL, 139667882033151, 139667882037247, ++STORE, 139667882029056, 139667882033151, ++STORE, 139667882033152, 139667882037247, ++SNULL, 139667884163071, 139667884167167, ++STORE, 139667884158976, 139667884163071, ++STORE, 139667884163072, 139667884167167, ++SNULL, 139667886731263, 139667886735359, ++STORE, 139667886727168, 139667886731263, ++STORE, 139667886731264, 139667886735359, ++SNULL, 139667892781055, 139667892785151, ++STORE, 139667892776960, 139667892781055, ++STORE, 139667892781056, 139667892785151, ++SNULL, 94796718321663, 94796718325759, ++STORE, 94796718317568, 94796718321663, ++STORE, 94796718321664, 94796718325759, ++SNULL, 139667895037951, 139667895042047, ++STORE, 139667895033856, 139667895037951, ++STORE, 139667895037952, 139667895042047, ++ERASE, 139667895005184, 139667895033855, ++STORE, 94796726063104, 94796726198271, ++STORE, 139667893305344, 139667894988799, ++STORE, 139667895005184, 139667895033855, ++STORE, 94796726063104, 94796726333439, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722489507840, 140737488351231, ++SNULL, 140722489516031, 140737488351231, ++STORE, 140722489507840, 140722489516031, ++STORE, 140722489376768, 140722489516031, ++STORE, 93980993265664, 93980995489791, ++SNULL, 93980993376255, 93980995489791, ++STORE, 93980993265664, 93980993376255, ++STORE, 93980993376256, 93980995489791, ++ERASE, 93980993376256, 93980995489791, ++STORE, 93980995469312, 93980995481599, ++STORE, 93980995481600, 93980995489791, ++STORE, 140261313593344, 140261315846143, ++SNULL, 140261313736703, 140261315846143, ++STORE, 140261313593344, 140261313736703, ++STORE, 140261313736704, 140261315846143, ++ERASE, 140261313736704, 140261315846143, ++STORE, 140261315833856, 140261315842047, ++STORE, 140261315842048, 140261315846143, ++STORE, 140722489675776, 140722489679871, ++STORE, 140722489663488, 140722489675775, ++STORE, 140261315805184, 140261315833855, ++STORE, 140261315796992, 140261315805183, ++STORE, 140261309796352, 140261313593343, ++SNULL, 140261309796352, 140261311455231, ++STORE, 140261311455232, 140261313593343, ++STORE, 140261309796352, 140261311455231, ++SNULL, 140261313552383, 140261313593343, ++STORE, 140261311455232, 140261313552383, ++STORE, 140261313552384, 140261313593343, ++SNULL, 140261313552384, 140261313576959, ++STORE, 140261313576960, 140261313593343, ++STORE, 140261313552384, 140261313576959, ++ERASE, 140261313552384, 140261313576959, ++STORE, 140261313552384, 140261313576959, ++ERASE, 140261313576960, 140261313593343, ++STORE, 140261313576960, 140261313593343, ++SNULL, 140261313568767, 140261313576959, ++STORE, 140261313552384, 140261313568767, ++STORE, 140261313568768, 140261313576959, ++SNULL, 93980995477503, 93980995481599, ++STORE, 93980995469312, 93980995477503, ++STORE, 93980995477504, 93980995481599, ++SNULL, 140261315837951, 140261315842047, ++STORE, 140261315833856, 140261315837951, ++STORE, 140261315837952, 140261315842047, ++ERASE, 140261315805184, 140261315833855, ++STORE, 93980997443584, 93980997578751, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140734059450368, 140737488351231, ++SNULL, 140734059462655, 140737488351231, ++STORE, 140734059450368, 140734059462655, ++STORE, 140734059319296, 140734059462655, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140307554983936, 140307557236735, ++SNULL, 140307555127295, 140307557236735, ++STORE, 140307554983936, 140307555127295, ++STORE, 140307555127296, 140307557236735, ++ERASE, 140307555127296, 140307557236735, ++STORE, 140307557224448, 140307557232639, ++STORE, 140307557232640, 140307557236735, ++STORE, 140734059483136, 140734059487231, ++STORE, 140734059470848, 140734059483135, ++STORE, 140307557195776, 140307557224447, ++STORE, 140307557187584, 140307557195775, ++STORE, 140307551821824, 140307554983935, ++SNULL, 140307551821824, 140307552882687, ++STORE, 140307552882688, 140307554983935, ++STORE, 140307551821824, 140307552882687, ++SNULL, 140307554975743, 140307554983935, ++STORE, 140307552882688, 140307554975743, ++STORE, 140307554975744, 140307554983935, ++ERASE, 140307554975744, 140307554983935, ++STORE, 140307554975744, 140307554983935, ++STORE, 140307548024832, 140307551821823, ++SNULL, 140307548024832, 140307549683711, ++STORE, 140307549683712, 140307551821823, ++STORE, 140307548024832, 140307549683711, ++SNULL, 140307551780863, 140307551821823, ++STORE, 140307549683712, 140307551780863, ++STORE, 140307551780864, 140307551821823, ++SNULL, 140307551780864, 140307551805439, ++STORE, 140307551805440, 140307551821823, ++STORE, 140307551780864, 140307551805439, ++ERASE, 140307551780864, 140307551805439, ++STORE, 140307551780864, 140307551805439, ++ERASE, 140307551805440, 140307551821823, ++STORE, 140307551805440, 140307551821823, ++STORE, 140307557175296, 140307557195775, ++SNULL, 140307551797247, 140307551805439, ++STORE, 140307551780864, 140307551797247, ++STORE, 140307551797248, 140307551805439, ++SNULL, 140307554979839, 140307554983935, ++STORE, 140307554975744, 140307554979839, ++STORE, 140307554979840, 140307554983935, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140307557228543, 140307557232639, ++STORE, 140307557224448, 140307557228543, ++STORE, 140307557228544, 140307557232639, ++ERASE, 140307557195776, 140307557224447, ++STORE, 39698432, 39833599, ++STORE, 39698432, 39981055, ++STORE, 94306485321728, 94306485432319, ++STORE, 94306487525376, 94306487533567, ++STORE, 94306487533568, 94306487537663, ++STORE, 94306487537664, 94306487545855, ++STORE, 94306488868864, 94306489004031, ++STORE, 140497673998336, 140497675657215, ++STORE, 140497675657216, 140497677754367, ++STORE, 140497677754368, 140497677770751, ++STORE, 140497677770752, 140497677778943, ++STORE, 140497677778944, 140497677795327, ++STORE, 140497677795328, 140497677938687, ++STORE, 140497679998976, 140497680007167, ++STORE, 140497680035840, 140497680039935, ++STORE, 140497680039936, 140497680044031, ++STORE, 140497680044032, 140497680048127, ++STORE, 140732780462080, 140732780601343, ++STORE, 140732782239744, 140732782252031, ++STORE, 140732782252032, 140732782256127, ++STORE, 94236915900416, 94236916011007, ++STORE, 94236918104064, 94236918112255, ++STORE, 94236918112256, 94236918116351, ++STORE, 94236918116352, 94236918124543, ++STORE, 94236939489280, 94236939624447, ++STORE, 140046091743232, 140046093402111, ++STORE, 140046093402112, 140046095499263, ++STORE, 140046095499264, 140046095515647, ++STORE, 140046095515648, 140046095523839, ++STORE, 140046095523840, 140046095540223, ++STORE, 140046095540224, 140046095683583, ++STORE, 140046097743872, 140046097752063, ++STORE, 140046097780736, 140046097784831, ++STORE, 140046097784832, 140046097788927, ++STORE, 140046097788928, 140046097793023, ++STORE, 140726694449152, 140726694588415, ++STORE, 140726695313408, 140726695325695, ++STORE, 140726695325696, 140726695329791, ++STORE, 94894582779904, 94894582992895, ++STORE, 94894585090048, 94894585094143, ++STORE, 94894585094144, 94894585102335, ++STORE, 94894585102336, 94894585114623, ++STORE, 94894592868352, 94894594293759, ++STORE, 139733563842560, 139733565501439, ++STORE, 139733565501440, 139733567598591, ++STORE, 139733567598592, 139733567614975, ++STORE, 139733567614976, 139733567623167, ++STORE, 139733567623168, 139733567639551, ++STORE, 139733567639552, 139733567651839, ++STORE, 139733567651840, 139733569744895, ++STORE, 139733569744896, 139733569748991, ++STORE, 139733569748992, 139733569753087, ++STORE, 139733569753088, 139733569896447, ++STORE, 139733570265088, 139733571948543, ++STORE, 139733571948544, 139733571964927, ++STORE, 139733571993600, 139733571997695, ++STORE, 139733571997696, 139733572001791, ++STORE, 139733572001792, 139733572005887, ++STORE, 140726369255424, 140726369394687, ++STORE, 140726370402304, 140726370414591, ++STORE, 140726370414592, 140726370418687, ++STORE, 94899236483072, 94899236696063, ++STORE, 94899238793216, 94899238797311, ++STORE, 94899238797312, 94899238805503, ++STORE, 94899238805504, 94899238817791, ++STORE, 94899263045632, 94899263979519, ++STORE, 140040959893504, 140040961552383, ++STORE, 140040961552384, 140040963649535, ++STORE, 140040963649536, 140040963665919, ++STORE, 140040963665920, 140040963674111, ++STORE, 140040963674112, 140040963690495, ++STORE, 140040963690496, 140040963702783, ++STORE, 140040963702784, 140040965795839, ++STORE, 140040965795840, 140040965799935, ++STORE, 140040965799936, 140040965804031, ++STORE, 140040965804032, 140040965947391, ++STORE, 140040966316032, 140040967999487, ++STORE, 140040967999488, 140040968015871, ++STORE, 140040968044544, 140040968048639, ++STORE, 140040968048640, 140040968052735, ++STORE, 140040968052736, 140040968056831, ++STORE, 140729921359872, 140729921499135, ++STORE, 140729921613824, 140729921626111, ++STORE, 140729921626112, 140729921630207, ++STORE, 94818265190400, 94818265403391, ++STORE, 94818267500544, 94818267504639, ++STORE, 94818267504640, 94818267512831, ++STORE, 94818267512832, 94818267525119, ++STORE, 94818283372544, 94818285858815, ++STORE, 139818425675776, 139818427334655, ++STORE, 139818427334656, 139818429431807, ++STORE, 139818429431808, 139818429448191, ++STORE, 139818429448192, 139818429456383, ++STORE, 139818429456384, 139818429472767, ++STORE, 139818429472768, 139818429485055, ++STORE, 139818429485056, 139818431578111, ++STORE, 139818431578112, 139818431582207, ++STORE, 139818431582208, 139818431586303, ++STORE, 139818431586304, 139818431729663, ++STORE, 139818432098304, 139818433781759, ++STORE, 139818433781760, 139818433798143, ++STORE, 139818433826816, 139818433830911, ++STORE, 139818433830912, 139818433835007, ++STORE, 139818433835008, 139818433839103, ++STORE, 140726170509312, 140726170648575, ++STORE, 140726171824128, 140726171836415, ++STORE, 140726171836416, 140726171840511, ++STORE, 94611513188352, 94611513401343, ++STORE, 94611515498496, 94611515502591, ++STORE, 94611515502592, 94611515510783, ++STORE, 94611515510784, 94611515523071, ++STORE, 94611516502016, 94611516907519, ++STORE, 140596246388736, 140596248047615, ++STORE, 140596248047616, 140596250144767, ++STORE, 140596250144768, 140596250161151, ++STORE, 140596250161152, 140596250169343, ++STORE, 140596250169344, 140596250185727, ++STORE, 140596250185728, 140596250198015, ++STORE, 140596250198016, 140596252291071, ++STORE, 140596252291072, 140596252295167, ++STORE, 140596252295168, 140596252299263, ++STORE, 140596252299264, 140596252442623, ++STORE, 140596252811264, 140596254494719, ++STORE, 140596254494720, 140596254511103, ++STORE, 140596254539776, 140596254543871, ++STORE, 140596254543872, 140596254547967, ++STORE, 140596254547968, 140596254552063, ++STORE, 140731551338496, 140731551477759, ++STORE, 140731551780864, 140731551793151, ++STORE, 140731551793152, 140731551797247, ++STORE, 94313835851776, 94313836064767, ++STORE, 94313838161920, 94313838166015, ++STORE, 94313838166016, 94313838174207, ++STORE, 94313838174208, 94313838186495, ++STORE, 94313858416640, 94313861906431, ++STORE, 140693503918080, 140693505576959, ++STORE, 140693505576960, 140693507674111, ++STORE, 140693507674112, 140693507690495, ++STORE, 140693507690496, 140693507698687, ++STORE, 140693507698688, 140693507715071, ++STORE, 140693507715072, 140693507727359, ++STORE, 140693507727360, 140693509820415, ++STORE, 140693509820416, 140693509824511, ++STORE, 140693509824512, 140693509828607, ++STORE, 140693509828608, 140693509971967, ++STORE, 140693510340608, 140693512024063, ++STORE, 140693512024064, 140693512040447, ++STORE, 140693512069120, 140693512073215, ++STORE, 140693512073216, 140693512077311, ++STORE, 140693512077312, 140693512081407, ++STORE, 140721116065792, 140721116205055, ++STORE, 140721117831168, 140721117843455, ++STORE, 140721117843456, 140721117847551, ++STORE, 94843650150400, 94843650363391, ++STORE, 94843652460544, 94843652464639, ++STORE, 94843652464640, 94843652472831, ++STORE, 94843652472832, 94843652485119, ++STORE, 94843685388288, 94843686281215, ++STORE, 140484193681408, 140484195340287, ++STORE, 140484195340288, 140484197437439, ++STORE, 140484197437440, 140484197453823, ++STORE, 140484197453824, 140484197462015, ++STORE, 140484197462016, 140484197478399, ++STORE, 140484197478400, 140484197490687, ++STORE, 140484197490688, 140484199583743, ++STORE, 140484199583744, 140484199587839, ++STORE, 140484199587840, 140484199591935, ++STORE, 140484199591936, 140484199735295, ++STORE, 140484200103936, 140484201787391, ++STORE, 140484201787392, 140484201803775, ++STORE, 140484201832448, 140484201836543, ++STORE, 140484201836544, 140484201840639, ++STORE, 140484201840640, 140484201844735, ++STORE, 140726294315008, 140726294454271, ++STORE, 140726295646208, 140726295658495, ++STORE, 140726295658496, 140726295662591, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720422371328, 140737488351231, ++SNULL, 140720422379519, 140737488351231, ++STORE, 140720422371328, 140720422379519, ++STORE, 140720422240256, 140720422379519, ++STORE, 94417967845376, 94417970180095, ++SNULL, 94417968058367, 94417970180095, ++STORE, 94417967845376, 94417968058367, ++STORE, 94417968058368, 94417970180095, ++ERASE, 94417968058368, 94417970180095, ++STORE, 94417970155520, 94417970167807, ++STORE, 94417970167808, 94417970180095, ++STORE, 140252450045952, 140252452298751, ++SNULL, 140252450189311, 140252452298751, ++STORE, 140252450045952, 140252450189311, ++STORE, 140252450189312, 140252452298751, ++ERASE, 140252450189312, 140252452298751, ++STORE, 140252452286464, 140252452294655, ++STORE, 140252452294656, 140252452298751, ++STORE, 140720422416384, 140720422420479, ++STORE, 140720422404096, 140720422416383, ++STORE, 140252452257792, 140252452286463, ++STORE, 140252452249600, 140252452257791, ++STORE, 140252447932416, 140252450045951, ++SNULL, 140252447932416, 140252447944703, ++STORE, 140252447944704, 140252450045951, ++STORE, 140252447932416, 140252447944703, ++SNULL, 140252450037759, 140252450045951, ++STORE, 140252447944704, 140252450037759, ++STORE, 140252450037760, 140252450045951, ++ERASE, 140252450037760, 140252450045951, ++STORE, 140252450037760, 140252450045951, ++STORE, 140252444135424, 140252447932415, ++SNULL, 140252444135424, 140252445794303, ++STORE, 140252445794304, 140252447932415, ++STORE, 140252444135424, 140252445794303, ++SNULL, 140252447891455, 140252447932415, ++STORE, 140252445794304, 140252447891455, ++STORE, 140252447891456, 140252447932415, ++SNULL, 140252447891456, 140252447916031, ++STORE, 140252447916032, 140252447932415, ++STORE, 140252447891456, 140252447916031, ++ERASE, 140252447891456, 140252447916031, ++STORE, 140252447891456, 140252447916031, ++ERASE, 140252447916032, 140252447932415, ++STORE, 140252447916032, 140252447932415, ++STORE, 140252452241408, 140252452257791, ++SNULL, 140252447907839, 140252447916031, ++STORE, 140252447891456, 140252447907839, ++STORE, 140252447907840, 140252447916031, ++SNULL, 140252450041855, 140252450045951, ++STORE, 140252450037760, 140252450041855, ++STORE, 140252450041856, 140252450045951, ++SNULL, 94417970159615, 94417970167807, ++STORE, 94417970155520, 94417970159615, ++STORE, 94417970159616, 94417970167807, ++SNULL, 140252452290559, 140252452294655, ++STORE, 140252452286464, 140252452290559, ++STORE, 140252452290560, 140252452294655, ++ERASE, 140252452257792, 140252452286463, ++STORE, 94417996333056, 94417996468223, ++STORE, 140252450557952, 140252452241407, ++STORE, 94417996333056, 94417996603391, ++STORE, 94417996333056, 94417996738559, ++STORE, 94417996333056, 94417996910591, ++SNULL, 94417996881919, 94417996910591, ++STORE, 94417996333056, 94417996881919, ++STORE, 94417996881920, 94417996910591, ++ERASE, 94417996881920, 94417996910591, ++STORE, 94417996333056, 94417997017087, ++STORE, 94417996333056, 94417997152255, ++SNULL, 94417997135871, 94417997152255, ++STORE, 94417996333056, 94417997135871, ++STORE, 94417997135872, 94417997152255, ++ERASE, 94417997135872, 94417997152255, ++STORE, 94417996333056, 94417997291519, ++SNULL, 94417997271039, 94417997291519, ++STORE, 94417996333056, 94417997271039, ++STORE, 94417997271040, 94417997291519, ++ERASE, 94417997271040, 94417997291519, ++STORE, 94417996333056, 94417997406207, ++SNULL, 94417997381631, 94417997406207, ++STORE, 94417996333056, 94417997381631, ++STORE, 94417997381632, 94417997406207, ++ERASE, 94417997381632, 94417997406207, ++STORE, 94417996333056, 94417997516799, ++SNULL, 94417997488127, 94417997516799, ++STORE, 94417996333056, 94417997488127, ++STORE, 94417997488128, 94417997516799, ++ERASE, 94417997488128, 94417997516799, ++STORE, 94417996333056, 94417997643775, ++SNULL, 94417997631487, 94417997643775, ++STORE, 94417996333056, 94417997631487, ++STORE, 94417997631488, 94417997643775, ++ERASE, 94417997631488, 94417997643775, ++SNULL, 94417997590527, 94417997631487, ++STORE, 94417996333056, 94417997590527, ++STORE, 94417997590528, 94417997631487, ++ERASE, 94417997590528, 94417997631487, ++STORE, 94417996333056, 94417997733887, ++STORE, 94417996333056, 94417997869055, ++STORE, 94417996333056, 94417998004223, ++SNULL, 94417998000127, 94417998004223, ++STORE, 94417996333056, 94417998000127, ++STORE, 94417998000128, 94417998004223, ++ERASE, 94417998000128, 94417998004223, ++STORE, 94049170993152, 94049171206143, ++STORE, 94049173303296, 94049173307391, ++STORE, 94049173307392, 94049173315583, ++STORE, 94049173315584, 94049173327871, ++STORE, 94049176236032, 94049183645695, ++STORE, 139807795544064, 139807797202943, ++STORE, 139807797202944, 139807799300095, ++STORE, 139807799300096, 139807799316479, ++STORE, 139807799316480, 139807799324671, ++STORE, 139807799324672, 139807799341055, ++STORE, 139807799341056, 139807799353343, ++STORE, 139807799353344, 139807801446399, ++STORE, 139807801446400, 139807801450495, ++STORE, 139807801450496, 139807801454591, ++STORE, 139807801454592, 139807801597951, ++STORE, 139807801966592, 139807803650047, ++STORE, 139807803650048, 139807803666431, ++STORE, 139807803695104, 139807803699199, ++STORE, 139807803699200, 139807803703295, ++STORE, 139807803703296, 139807803707391, ++STORE, 140727555538944, 140727555678207, ++STORE, 140727555940352, 140727555952639, ++STORE, 140727555952640, 140727555956735, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722483441664, 140737488351231, ++SNULL, 140722483449855, 140737488351231, ++STORE, 140722483441664, 140722483449855, ++STORE, 140722483310592, 140722483449855, ++STORE, 94416704921600, 94416707145727, ++SNULL, 94416705032191, 94416707145727, ++STORE, 94416704921600, 94416705032191, ++STORE, 94416705032192, 94416707145727, ++ERASE, 94416705032192, 94416707145727, ++STORE, 94416707125248, 94416707137535, ++STORE, 94416707137536, 94416707145727, ++STORE, 140555439296512, 140555441549311, ++SNULL, 140555439439871, 140555441549311, ++STORE, 140555439296512, 140555439439871, ++STORE, 140555439439872, 140555441549311, ++ERASE, 140555439439872, 140555441549311, ++STORE, 140555441537024, 140555441545215, ++STORE, 140555441545216, 140555441549311, ++STORE, 140722484781056, 140722484785151, ++STORE, 140722484768768, 140722484781055, ++STORE, 140555441508352, 140555441537023, ++STORE, 140555441500160, 140555441508351, ++STORE, 140555435499520, 140555439296511, ++SNULL, 140555435499520, 140555437158399, ++STORE, 140555437158400, 140555439296511, ++STORE, 140555435499520, 140555437158399, ++SNULL, 140555439255551, 140555439296511, ++STORE, 140555437158400, 140555439255551, ++STORE, 140555439255552, 140555439296511, ++SNULL, 140555439255552, 140555439280127, ++STORE, 140555439280128, 140555439296511, ++STORE, 140555439255552, 140555439280127, ++ERASE, 140555439255552, 140555439280127, ++STORE, 140555439255552, 140555439280127, ++ERASE, 140555439280128, 140555439296511, ++STORE, 140555439280128, 140555439296511, ++SNULL, 140555439271935, 140555439280127, ++STORE, 140555439255552, 140555439271935, ++STORE, 140555439271936, 140555439280127, ++SNULL, 94416707133439, 94416707137535, ++STORE, 94416707125248, 94416707133439, ++STORE, 94416707133440, 94416707137535, ++SNULL, 140555441541119, 140555441545215, ++STORE, 140555441537024, 140555441541119, ++STORE, 140555441541120, 140555441545215, ++ERASE, 140555441508352, 140555441537023, ++STORE, 94416724672512, 94416724807679, ++STORE, 94686636953600, 94686637166591, ++STORE, 94686639263744, 94686639267839, ++STORE, 94686639267840, 94686639276031, ++STORE, 94686639276032, 94686639288319, ++STORE, 94686662193152, 94686663163903, ++STORE, 140312944431104, 140312946089983, ++STORE, 140312946089984, 140312948187135, ++STORE, 140312948187136, 140312948203519, ++STORE, 140312948203520, 140312948211711, ++STORE, 140312948211712, 140312948228095, ++STORE, 140312948228096, 140312948240383, ++STORE, 140312948240384, 140312950333439, ++STORE, 140312950333440, 140312950337535, ++STORE, 140312950337536, 140312950341631, ++STORE, 140312950341632, 140312950484991, ++STORE, 140312950853632, 140312952537087, ++STORE, 140312952537088, 140312952553471, ++STORE, 140312952582144, 140312952586239, ++STORE, 140312952586240, 140312952590335, ++STORE, 140312952590336, 140312952594431, ++STORE, 140730598920192, 140730599059455, ++STORE, 140730599108608, 140730599120895, ++STORE, 140730599120896, 140730599124991, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726234079232, 140737488351231, ++SNULL, 140726234087423, 140737488351231, ++STORE, 140726234079232, 140726234087423, ++STORE, 140726233948160, 140726234087423, ++STORE, 94589467578368, 94589469802495, ++SNULL, 94589467688959, 94589469802495, ++STORE, 94589467578368, 94589467688959, ++STORE, 94589467688960, 94589469802495, ++ERASE, 94589467688960, 94589469802495, ++STORE, 94589469782016, 94589469794303, ++STORE, 94589469794304, 94589469802495, ++STORE, 140587082842112, 140587085094911, ++SNULL, 140587082985471, 140587085094911, ++STORE, 140587082842112, 140587082985471, ++STORE, 140587082985472, 140587085094911, ++ERASE, 140587082985472, 140587085094911, ++STORE, 140587085082624, 140587085090815, ++STORE, 140587085090816, 140587085094911, ++STORE, 140726234103808, 140726234107903, ++STORE, 140726234091520, 140726234103807, ++STORE, 140587085053952, 140587085082623, ++STORE, 140587085045760, 140587085053951, ++STORE, 140587079045120, 140587082842111, ++SNULL, 140587079045120, 140587080703999, ++STORE, 140587080704000, 140587082842111, ++STORE, 140587079045120, 140587080703999, ++SNULL, 140587082801151, 140587082842111, ++STORE, 140587080704000, 140587082801151, ++STORE, 140587082801152, 140587082842111, ++SNULL, 140587082801152, 140587082825727, ++STORE, 140587082825728, 140587082842111, ++STORE, 140587082801152, 140587082825727, ++ERASE, 140587082801152, 140587082825727, ++STORE, 140587082801152, 140587082825727, ++ERASE, 140587082825728, 140587082842111, ++STORE, 140587082825728, 140587082842111, ++SNULL, 140587082817535, 140587082825727, ++STORE, 140587082801152, 140587082817535, ++STORE, 140587082817536, 140587082825727, ++SNULL, 94589469790207, 94589469794303, ++STORE, 94589469782016, 94589469790207, ++STORE, 94589469790208, 94589469794303, ++SNULL, 140587085086719, 140587085090815, ++STORE, 140587085082624, 140587085086719, ++STORE, 140587085086720, 140587085090815, ++ERASE, 140587085053952, 140587085082623, ++STORE, 94589477507072, 94589477642239, ++STORE, 94225448325120, 94225448538111, ++STORE, 94225450635264, 94225450639359, ++STORE, 94225450639360, 94225450647551, ++STORE, 94225450647552, 94225450659839, ++STORE, 94225470246912, 94225473548287, ++STORE, 140199245496320, 140199247155199, ++STORE, 140199247155200, 140199249252351, ++STORE, 140199249252352, 140199249268735, ++STORE, 140199249268736, 140199249276927, ++STORE, 140199249276928, 140199249293311, ++STORE, 140199249293312, 140199249305599, ++STORE, 140199249305600, 140199251398655, ++STORE, 140199251398656, 140199251402751, ++STORE, 140199251402752, 140199251406847, ++STORE, 140199251406848, 140199251550207, ++STORE, 140199251918848, 140199253602303, ++STORE, 140199253602304, 140199253618687, ++STORE, 140199253647360, 140199253651455, ++STORE, 140199253651456, 140199253655551, ++STORE, 140199253655552, 140199253659647, ++STORE, 140726264414208, 140726264553471, ++STORE, 140726265843712, 140726265855999, ++STORE, 140726265856000, 140726265860095, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733508358144, 140737488351231, ++SNULL, 140733508366335, 140737488351231, ++STORE, 140733508358144, 140733508366335, ++STORE, 140733508227072, 140733508366335, ++STORE, 94766263947264, 94766266171391, ++SNULL, 94766264057855, 94766266171391, ++STORE, 94766263947264, 94766264057855, ++STORE, 94766264057856, 94766266171391, ++ERASE, 94766264057856, 94766266171391, ++STORE, 94766266150912, 94766266163199, ++STORE, 94766266163200, 94766266171391, ++STORE, 140693985132544, 140693987385343, ++SNULL, 140693985275903, 140693987385343, ++STORE, 140693985132544, 140693985275903, ++STORE, 140693985275904, 140693987385343, ++ERASE, 140693985275904, 140693987385343, ++STORE, 140693987373056, 140693987381247, ++STORE, 140693987381248, 140693987385343, ++STORE, 140733509939200, 140733509943295, ++STORE, 140733509926912, 140733509939199, ++STORE, 140693987344384, 140693987373055, ++STORE, 140693987336192, 140693987344383, ++STORE, 140693981335552, 140693985132543, ++SNULL, 140693981335552, 140693982994431, ++STORE, 140693982994432, 140693985132543, ++STORE, 140693981335552, 140693982994431, ++SNULL, 140693985091583, 140693985132543, ++STORE, 140693982994432, 140693985091583, ++STORE, 140693985091584, 140693985132543, ++SNULL, 140693985091584, 140693985116159, ++STORE, 140693985116160, 140693985132543, ++STORE, 140693985091584, 140693985116159, ++ERASE, 140693985091584, 140693985116159, ++STORE, 140693985091584, 140693985116159, ++ERASE, 140693985116160, 140693985132543, ++STORE, 140693985116160, 140693985132543, ++SNULL, 140693985107967, 140693985116159, ++STORE, 140693985091584, 140693985107967, ++STORE, 140693985107968, 140693985116159, ++SNULL, 94766266159103, 94766266163199, ++STORE, 94766266150912, 94766266159103, ++STORE, 94766266159104, 94766266163199, ++SNULL, 140693987377151, 140693987381247, ++STORE, 140693987373056, 140693987377151, ++STORE, 140693987377152, 140693987381247, ++ERASE, 140693987344384, 140693987373055, ++STORE, 94766282035200, 94766282170367, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724769353728, 140737488351231, ++SNULL, 140724769361919, 140737488351231, ++STORE, 140724769353728, 140724769361919, ++STORE, 140724769222656, 140724769361919, ++STORE, 94710460526592, 94710462750719, ++SNULL, 94710460637183, 94710462750719, ++STORE, 94710460526592, 94710460637183, ++STORE, 94710460637184, 94710462750719, ++ERASE, 94710460637184, 94710462750719, ++STORE, 94710462730240, 94710462742527, ++STORE, 94710462742528, 94710462750719, ++STORE, 140469764395008, 140469766647807, ++SNULL, 140469764538367, 140469766647807, ++STORE, 140469764395008, 140469764538367, ++STORE, 140469764538368, 140469766647807, ++ERASE, 140469764538368, 140469766647807, ++STORE, 140469766635520, 140469766643711, ++STORE, 140469766643712, 140469766647807, ++STORE, 140724770877440, 140724770881535, ++STORE, 140724770865152, 140724770877439, ++STORE, 140469766606848, 140469766635519, ++STORE, 140469766598656, 140469766606847, ++STORE, 140469760598016, 140469764395007, ++SNULL, 140469760598016, 140469762256895, ++STORE, 140469762256896, 140469764395007, ++STORE, 140469760598016, 140469762256895, ++SNULL, 140469764354047, 140469764395007, ++STORE, 140469762256896, 140469764354047, ++STORE, 140469764354048, 140469764395007, ++SNULL, 140469764354048, 140469764378623, ++STORE, 140469764378624, 140469764395007, ++STORE, 140469764354048, 140469764378623, ++ERASE, 140469764354048, 140469764378623, ++STORE, 140469764354048, 140469764378623, ++ERASE, 140469764378624, 140469764395007, ++STORE, 140469764378624, 140469764395007, ++SNULL, 140469764370431, 140469764378623, ++STORE, 140469764354048, 140469764370431, ++STORE, 140469764370432, 140469764378623, ++SNULL, 94710462738431, 94710462742527, ++STORE, 94710462730240, 94710462738431, ++STORE, 94710462738432, 94710462742527, ++SNULL, 140469766639615, 140469766643711, ++STORE, 140469766635520, 140469766639615, ++STORE, 140469766639616, 140469766643711, ++ERASE, 140469766606848, 140469766635519, ++STORE, 94710485581824, 94710485716991, ++STORE, 94105755795456, 94105756008447, ++STORE, 94105758105600, 94105758109695, ++STORE, 94105758109696, 94105758117887, ++STORE, 94105758117888, 94105758130175, ++STORE, 94105788981248, 94105794871295, ++STORE, 140641190031360, 140641191690239, ++STORE, 140641191690240, 140641193787391, ++STORE, 140641193787392, 140641193803775, ++STORE, 140641193803776, 140641193811967, ++STORE, 140641193811968, 140641193828351, ++STORE, 140641193828352, 140641193840639, ++STORE, 140641193840640, 140641195933695, ++STORE, 140641195933696, 140641195937791, ++STORE, 140641195937792, 140641195941887, ++STORE, 140641195941888, 140641196085247, ++STORE, 140641196453888, 140641198137343, ++STORE, 140641198137344, 140641198153727, ++STORE, 140641198182400, 140641198186495, ++STORE, 140641198186496, 140641198190591, ++STORE, 140641198190592, 140641198194687, ++STORE, 140731980034048, 140731980173311, ++STORE, 140731981078528, 140731981090815, ++STORE, 140731981090816, 140731981094911, ++STORE, 93828086431744, 93828086644735, ++STORE, 93828088741888, 93828088745983, ++STORE, 93828088745984, 93828088754175, ++STORE, 93828088754176, 93828088766463, ++STORE, 93828094193664, 93828096831487, ++STORE, 139844717334528, 139844718993407, ++STORE, 139844718993408, 139844721090559, ++STORE, 139844721090560, 139844721106943, ++STORE, 139844721106944, 139844721115135, ++STORE, 139844721115136, 139844721131519, ++STORE, 139844721131520, 139844721143807, ++STORE, 139844721143808, 139844723236863, ++STORE, 139844723236864, 139844723240959, ++STORE, 139844723240960, 139844723245055, ++STORE, 139844723245056, 139844723388415, ++STORE, 139844723757056, 139844725440511, ++STORE, 139844725440512, 139844725456895, ++STORE, 139844725485568, 139844725489663, ++STORE, 139844725489664, 139844725493759, ++STORE, 139844725493760, 139844725497855, ++STORE, 140729996185600, 140729996324863, ++STORE, 140729996828672, 140729996840959, ++STORE, 140729996840960, 140729996845055, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722494771200, 140737488351231, ++SNULL, 140722494775295, 140737488351231, ++STORE, 140722494771200, 140722494775295, ++STORE, 140722494640128, 140722494775295, ++STORE, 94324011311104, 94324013535231, ++SNULL, 94324011421695, 94324013535231, ++STORE, 94324011311104, 94324011421695, ++STORE, 94324011421696, 94324013535231, ++ERASE, 94324011421696, 94324013535231, ++STORE, 94324013514752, 94324013527039, ++STORE, 94324013527040, 94324013535231, ++STORE, 140151462309888, 140151464562687, ++SNULL, 140151462453247, 140151464562687, ++STORE, 140151462309888, 140151462453247, ++STORE, 140151462453248, 140151464562687, ++ERASE, 140151462453248, 140151464562687, ++STORE, 140151464550400, 140151464558591, ++STORE, 140151464558592, 140151464562687, ++STORE, 140722495467520, 140722495471615, ++STORE, 140722495455232, 140722495467519, ++STORE, 140151464521728, 140151464550399, ++STORE, 140151464513536, 140151464521727, ++STORE, 140151458512896, 140151462309887, ++SNULL, 140151458512896, 140151460171775, ++STORE, 140151460171776, 140151462309887, ++STORE, 140151458512896, 140151460171775, ++SNULL, 140151462268927, 140151462309887, ++STORE, 140151460171776, 140151462268927, ++STORE, 140151462268928, 140151462309887, ++SNULL, 140151462268928, 140151462293503, ++STORE, 140151462293504, 140151462309887, ++STORE, 140151462268928, 140151462293503, ++ERASE, 140151462268928, 140151462293503, ++STORE, 140151462268928, 140151462293503, ++ERASE, 140151462293504, 140151462309887, ++STORE, 140151462293504, 140151462309887, ++SNULL, 140151462285311, 140151462293503, ++STORE, 140151462268928, 140151462285311, ++STORE, 140151462285312, 140151462293503, ++SNULL, 94324013522943, 94324013527039, ++STORE, 94324013514752, 94324013522943, ++STORE, 94324013522944, 94324013527039, ++SNULL, 140151464554495, 140151464558591, ++STORE, 140151464550400, 140151464554495, ++STORE, 140151464554496, 140151464558591, ++ERASE, 140151464521728, 140151464550399, ++STORE, 94324024778752, 94324024913919, ++STORE, 94899262967808, 94899263180799, ++STORE, 94899265277952, 94899265282047, ++STORE, 94899265282048, 94899265290239, ++STORE, 94899265290240, 94899265302527, ++STORE, 94899295469568, 94899298689023, ++STORE, 140434388418560, 140434390077439, ++STORE, 140434390077440, 140434392174591, ++STORE, 140434392174592, 140434392190975, ++STORE, 140434392190976, 140434392199167, ++STORE, 140434392199168, 140434392215551, ++STORE, 140434392215552, 140434392227839, ++STORE, 140434392227840, 140434394320895, ++STORE, 140434394320896, 140434394324991, ++STORE, 140434394324992, 140434394329087, ++STORE, 140434394329088, 140434394472447, ++STORE, 140434394841088, 140434396524543, ++STORE, 140434396524544, 140434396540927, ++STORE, 140434396569600, 140434396573695, ++STORE, 140434396573696, 140434396577791, ++STORE, 140434396577792, 140434396581887, ++STORE, 140720618135552, 140720618274815, ++STORE, 140720618418176, 140720618430463, ++STORE, 140720618430464, 140720618434559, ++STORE, 94425529798656, 94425530011647, ++STORE, 94425532108800, 94425532112895, ++STORE, 94425532112896, 94425532121087, ++STORE, 94425532121088, 94425532133375, ++STORE, 94425557753856, 94425566576639, ++STORE, 140600528470016, 140600530128895, ++STORE, 140600530128896, 140600532226047, ++STORE, 140600532226048, 140600532242431, ++STORE, 140600532242432, 140600532250623, ++STORE, 140600532250624, 140600532267007, ++STORE, 140600532267008, 140600532279295, ++STORE, 140600532279296, 140600534372351, ++STORE, 140600534372352, 140600534376447, ++STORE, 140600534376448, 140600534380543, ++STORE, 140600534380544, 140600534523903, ++STORE, 140600534892544, 140600536575999, ++STORE, 140600536576000, 140600536592383, ++STORE, 140600536621056, 140600536625151, ++STORE, 140600536625152, 140600536629247, ++STORE, 140600536629248, 140600536633343, ++STORE, 140721857785856, 140721857925119, ++STORE, 140721858068480, 140721858080767, ++STORE, 140721858080768, 140721858084863, ++STORE, 94425529798656, 94425530011647, ++STORE, 94425532108800, 94425532112895, ++STORE, 94425532112896, 94425532121087, ++STORE, 94425532121088, 94425532133375, ++STORE, 94425557753856, 94425568772095, ++STORE, 140600528470016, 140600530128895, ++STORE, 140600530128896, 140600532226047, ++STORE, 140600532226048, 140600532242431, ++STORE, 140600532242432, 140600532250623, ++STORE, 140600532250624, 140600532267007, ++STORE, 140600532267008, 140600532279295, ++STORE, 140600532279296, 140600534372351, ++STORE, 140600534372352, 140600534376447, ++STORE, 140600534376448, 140600534380543, ++STORE, 140600534380544, 140600534523903, ++STORE, 140600534892544, 140600536575999, ++STORE, 140600536576000, 140600536592383, ++STORE, 140600536621056, 140600536625151, ++STORE, 140600536625152, 140600536629247, ++STORE, 140600536629248, 140600536633343, ++STORE, 140721857785856, 140721857925119, ++STORE, 140721858068480, 140721858080767, ++STORE, 140721858080768, 140721858084863, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735611645952, 140737488351231, ++SNULL, 140735611654143, 140737488351231, ++STORE, 140735611645952, 140735611654143, ++STORE, 140735611514880, 140735611654143, ++STORE, 94592137641984, 94592139866111, ++SNULL, 94592137752575, 94592139866111, ++STORE, 94592137641984, 94592137752575, ++STORE, 94592137752576, 94592139866111, ++ERASE, 94592137752576, 94592139866111, ++STORE, 94592139845632, 94592139857919, ++STORE, 94592139857920, 94592139866111, ++STORE, 140350425030656, 140350427283455, ++SNULL, 140350425174015, 140350427283455, ++STORE, 140350425030656, 140350425174015, ++STORE, 140350425174016, 140350427283455, ++ERASE, 140350425174016, 140350427283455, ++STORE, 140350427271168, 140350427279359, ++STORE, 140350427279360, 140350427283455, ++STORE, 140735612043264, 140735612047359, ++STORE, 140735612030976, 140735612043263, ++STORE, 140350427242496, 140350427271167, ++STORE, 140350427234304, 140350427242495, ++STORE, 140350421233664, 140350425030655, ++SNULL, 140350421233664, 140350422892543, ++STORE, 140350422892544, 140350425030655, ++STORE, 140350421233664, 140350422892543, ++SNULL, 140350424989695, 140350425030655, ++STORE, 140350422892544, 140350424989695, ++STORE, 140350424989696, 140350425030655, ++SNULL, 140350424989696, 140350425014271, ++STORE, 140350425014272, 140350425030655, ++STORE, 140350424989696, 140350425014271, ++ERASE, 140350424989696, 140350425014271, ++STORE, 140350424989696, 140350425014271, ++ERASE, 140350425014272, 140350425030655, ++STORE, 140350425014272, 140350425030655, ++SNULL, 140350425006079, 140350425014271, ++STORE, 140350424989696, 140350425006079, ++STORE, 140350425006080, 140350425014271, ++SNULL, 94592139853823, 94592139857919, ++STORE, 94592139845632, 94592139853823, ++STORE, 94592139853824, 94592139857919, ++SNULL, 140350427275263, 140350427279359, ++STORE, 140350427271168, 140350427275263, ++STORE, 140350427275264, 140350427279359, ++ERASE, 140350427242496, 140350427271167, ++STORE, 94592164823040, 94592164958207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723500535808, 140737488351231, ++SNULL, 140723500543999, 140737488351231, ++STORE, 140723500535808, 140723500543999, ++STORE, 140723500404736, 140723500543999, ++STORE, 94458379010048, 94458381234175, ++SNULL, 94458379120639, 94458381234175, ++STORE, 94458379010048, 94458379120639, ++STORE, 94458379120640, 94458381234175, ++ERASE, 94458379120640, 94458381234175, ++STORE, 94458381213696, 94458381225983, ++STORE, 94458381225984, 94458381234175, ++STORE, 139771674230784, 139771676483583, ++SNULL, 139771674374143, 139771676483583, ++STORE, 139771674230784, 139771674374143, ++STORE, 139771674374144, 139771676483583, ++ERASE, 139771674374144, 139771676483583, ++STORE, 139771676471296, 139771676479487, ++STORE, 139771676479488, 139771676483583, ++STORE, 140723500769280, 140723500773375, ++STORE, 140723500756992, 140723500769279, ++STORE, 139771676442624, 139771676471295, ++STORE, 139771676434432, 139771676442623, ++STORE, 139771670433792, 139771674230783, ++SNULL, 139771670433792, 139771672092671, ++STORE, 139771672092672, 139771674230783, ++STORE, 139771670433792, 139771672092671, ++SNULL, 139771674189823, 139771674230783, ++STORE, 139771672092672, 139771674189823, ++STORE, 139771674189824, 139771674230783, ++SNULL, 139771674189824, 139771674214399, ++STORE, 139771674214400, 139771674230783, ++STORE, 139771674189824, 139771674214399, ++ERASE, 139771674189824, 139771674214399, ++STORE, 139771674189824, 139771674214399, ++ERASE, 139771674214400, 139771674230783, ++STORE, 139771674214400, 139771674230783, ++SNULL, 139771674206207, 139771674214399, ++STORE, 139771674189824, 139771674206207, ++STORE, 139771674206208, 139771674214399, ++SNULL, 94458381221887, 94458381225983, ++STORE, 94458381213696, 94458381221887, ++STORE, 94458381221888, 94458381225983, ++SNULL, 139771676475391, 139771676479487, ++STORE, 139771676471296, 139771676475391, ++STORE, 139771676475392, 139771676479487, ++ERASE, 139771676442624, 139771676471295, ++STORE, 94458401873920, 94458402009087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731316264960, 140737488351231, ++SNULL, 140731316273151, 140737488351231, ++STORE, 140731316264960, 140731316273151, ++STORE, 140731316133888, 140731316273151, ++STORE, 94437830881280, 94437833215999, ++SNULL, 94437831094271, 94437833215999, ++STORE, 94437830881280, 94437831094271, ++STORE, 94437831094272, 94437833215999, ++ERASE, 94437831094272, 94437833215999, ++STORE, 94437833191424, 94437833203711, ++STORE, 94437833203712, 94437833215999, ++STORE, 140265986031616, 140265988284415, ++SNULL, 140265986174975, 140265988284415, ++STORE, 140265986031616, 140265986174975, ++STORE, 140265986174976, 140265988284415, ++ERASE, 140265986174976, 140265988284415, ++STORE, 140265988272128, 140265988280319, ++STORE, 140265988280320, 140265988284415, ++STORE, 140731316318208, 140731316322303, ++STORE, 140731316305920, 140731316318207, ++STORE, 140265988243456, 140265988272127, ++STORE, 140265988235264, 140265988243455, ++STORE, 140265983918080, 140265986031615, ++SNULL, 140265983918080, 140265983930367, ++STORE, 140265983930368, 140265986031615, ++STORE, 140265983918080, 140265983930367, ++SNULL, 140265986023423, 140265986031615, ++STORE, 140265983930368, 140265986023423, ++STORE, 140265986023424, 140265986031615, ++ERASE, 140265986023424, 140265986031615, ++STORE, 140265986023424, 140265986031615, ++STORE, 140265980121088, 140265983918079, ++SNULL, 140265980121088, 140265981779967, ++STORE, 140265981779968, 140265983918079, ++STORE, 140265980121088, 140265981779967, ++SNULL, 140265983877119, 140265983918079, ++STORE, 140265981779968, 140265983877119, ++STORE, 140265983877120, 140265983918079, ++SNULL, 140265983877120, 140265983901695, ++STORE, 140265983901696, 140265983918079, ++STORE, 140265983877120, 140265983901695, ++ERASE, 140265983877120, 140265983901695, ++STORE, 140265983877120, 140265983901695, ++ERASE, 140265983901696, 140265983918079, ++STORE, 140265983901696, 140265983918079, ++STORE, 140265988227072, 140265988243455, ++SNULL, 140265983893503, 140265983901695, ++STORE, 140265983877120, 140265983893503, ++STORE, 140265983893504, 140265983901695, ++SNULL, 140265986027519, 140265986031615, ++STORE, 140265986023424, 140265986027519, ++STORE, 140265986027520, 140265986031615, ++SNULL, 94437833195519, 94437833203711, ++STORE, 94437833191424, 94437833195519, ++STORE, 94437833195520, 94437833203711, ++SNULL, 140265988276223, 140265988280319, ++STORE, 140265988272128, 140265988276223, ++STORE, 140265988276224, 140265988280319, ++ERASE, 140265988243456, 140265988272127, ++STORE, 94437847638016, 94437847773183, ++STORE, 140265986543616, 140265988227071, ++STORE, 94437847638016, 94437847908351, ++STORE, 94437847638016, 94437848043519, ++STORE, 94437847638016, 94437848190975, ++SNULL, 94437848178687, 94437848190975, ++STORE, 94437847638016, 94437848178687, ++STORE, 94437848178688, 94437848190975, ++ERASE, 94437848178688, 94437848190975, ++STORE, 94437847638016, 94437848330239, ++STORE, 94437847638016, 94437848465407, ++SNULL, 94437848444927, 94437848465407, ++STORE, 94437847638016, 94437848444927, ++STORE, 94437848444928, 94437848465407, ++ERASE, 94437848444928, 94437848465407, ++STORE, 94437847638016, 94437848584191, ++STORE, 94437847638016, 94437848719359, ++SNULL, 94437848678399, 94437848719359, ++STORE, 94437847638016, 94437848678399, ++STORE, 94437848678400, 94437848719359, ++ERASE, 94437848678400, 94437848719359, ++STORE, 94437847638016, 94437848842239, ++SNULL, 94437848825855, 94437848842239, ++STORE, 94437847638016, 94437848825855, ++STORE, 94437848825856, 94437848842239, ++ERASE, 94437848825856, 94437848842239, ++STORE, 94437847638016, 94437848961023, ++STORE, 94437847638016, 94437849096191, ++STORE, 94661814710272, 94661814923263, ++STORE, 94661817020416, 94661817024511, ++STORE, 94661817024512, 94661817032703, ++STORE, 94661817032704, 94661817044991, ++STORE, 94661840424960, 94661841240063, ++STORE, 140582259814400, 140582261473279, ++STORE, 140582261473280, 140582263570431, ++STORE, 140582263570432, 140582263586815, ++STORE, 140582263586816, 140582263595007, ++STORE, 140582263595008, 140582263611391, ++STORE, 140582263611392, 140582263623679, ++STORE, 140582263623680, 140582265716735, ++STORE, 140582265716736, 140582265720831, ++STORE, 140582265720832, 140582265724927, ++STORE, 140582265724928, 140582265868287, ++STORE, 140582266236928, 140582267920383, ++STORE, 140582267920384, 140582267936767, ++STORE, 140582267965440, 140582267969535, ++STORE, 140582267969536, 140582267973631, ++STORE, 140582267973632, 140582267977727, ++STORE, 140735472508928, 140735472648191, ++STORE, 140735472672768, 140735472685055, ++STORE, 140735472685056, 140735472689151, ++STORE, 94440069140480, 94440069353471, ++STORE, 94440071450624, 94440071454719, ++STORE, 94440071454720, 94440071462911, ++STORE, 94440071462912, 94440071475199, ++STORE, 94440072122368, 94440079048703, ++STORE, 140112218095616, 140112219754495, ++STORE, 140112219754496, 140112221851647, ++STORE, 140112221851648, 140112221868031, ++STORE, 140112221868032, 140112221876223, ++STORE, 140112221876224, 140112221892607, ++STORE, 140112221892608, 140112221904895, ++STORE, 140112221904896, 140112223997951, ++STORE, 140112223997952, 140112224002047, ++STORE, 140112224002048, 140112224006143, ++STORE, 140112224006144, 140112224149503, ++STORE, 140112224518144, 140112226201599, ++STORE, 140112226201600, 140112226217983, ++STORE, 140112226246656, 140112226250751, ++STORE, 140112226250752, 140112226254847, ++STORE, 140112226254848, 140112226258943, ++STORE, 140737460969472, 140737461108735, ++STORE, 140737462083584, 140737462095871, ++STORE, 140737462095872, 140737462099967, ++STORE, 94257654345728, 94257654390783, ++STORE, 94257656483840, 94257656487935, ++STORE, 94257656487936, 94257656492031, ++STORE, 94257656492032, 94257656496127, ++STORE, 94257665859584, 94257665994751, ++STORE, 140507070345216, 140507070386175, ++STORE, 140507070386176, 140507072483327, ++STORE, 140507072483328, 140507072487423, ++STORE, 140507072487424, 140507072491519, ++STORE, 140507072491520, 140507072516095, ++STORE, 140507072516096, 140507072561151, ++STORE, 140507072561152, 140507074654207, ++STORE, 140507074654208, 140507074658303, ++STORE, 140507074658304, 140507074662399, ++STORE, 140507074662400, 140507074744319, ++STORE, 140507074744320, 140507076841471, ++STORE, 140507076841472, 140507076845567, ++STORE, 140507076845568, 140507076849663, ++STORE, 140507076849664, 140507076857855, ++STORE, 140507076857856, 140507076886527, ++STORE, 140507076886528, 140507078979583, ++STORE, 140507078979584, 140507078983679, ++STORE, 140507078983680, 140507078987775, ++STORE, 140507078987776, 140507079086079, ++STORE, 140507079086080, 140507081179135, ++STORE, 140507081179136, 140507081183231, ++STORE, 140507081183232, 140507081187327, ++STORE, 140507081187328, 140507081203711, ++STORE, 140507081203712, 140507081220095, ++STORE, 140507081220096, 140507083317247, ++STORE, 140507083317248, 140507083321343, ++STORE, 140507083321344, 140507083325439, ++STORE, 140507083325440, 140507083792383, ++STORE, 140507083792384, 140507085885439, ++STORE, 140507085885440, 140507085889535, ++STORE, 140507085889536, 140507085893631, ++STORE, 140507085893632, 140507085905919, ++STORE, 140507085905920, 140507087998975, ++STORE, 140507087998976, 140507088003071, ++STORE, 140507088003072, 140507088007167, ++STORE, 140507088007168, 140507088125951, ++STORE, 140507088125952, 140507090219007, ++STORE, 140507090219008, 140507090223103, ++STORE, 140507090223104, 140507090227199, ++STORE, 140507090227200, 140507090268159, ++STORE, 140507090268160, 140507091927039, ++STORE, 140507091927040, 140507094024191, ++STORE, 140507094024192, 140507094040575, ++STORE, 140507094040576, 140507094048767, ++STORE, 140507094048768, 140507094065151, ++STORE, 140507094065152, 140507094216703, ++STORE, 140507094216704, 140507096309759, ++STORE, 140507096309760, 140507096313855, ++STORE, 140507096313856, 140507096317951, ++STORE, 140507096317952, 140507096326143, ++STORE, 140507096326144, 140507096379391, ++STORE, 140507096379392, 140507098472447, ++STORE, 140507098472448, 140507098476543, ++STORE, 140507098476544, 140507098480639, ++STORE, 140507098480640, 140507098623999, ++STORE, 140507098980352, 140507100663807, ++STORE, 140507100663808, 140507100692479, ++STORE, 140507100721152, 140507100725247, ++STORE, 140507100725248, 140507100729343, ++STORE, 140507100729344, 140507100733439, ++STORE, 140728152780800, 140728152915967, ++STORE, 140728153698304, 140728153710591, ++STORE, 140728153710592, 140728153714687, ++STORE, 140507068137472, 140507070345215, ++SNULL, 140507068137472, 140507068190719, ++STORE, 140507068190720, 140507070345215, ++STORE, 140507068137472, 140507068190719, ++SNULL, 140507070287871, 140507070345215, ++STORE, 140507068190720, 140507070287871, ++STORE, 140507070287872, 140507070345215, ++SNULL, 140507070287872, 140507070296063, ++STORE, 140507070296064, 140507070345215, ++STORE, 140507070287872, 140507070296063, ++ERASE, 140507070287872, 140507070296063, ++STORE, 140507070287872, 140507070296063, ++ERASE, 140507070296064, 140507070345215, ++STORE, 140507070296064, 140507070345215, ++STORE, 140507100692480, 140507100721151, ++STORE, 140507065810944, 140507068137471, ++SNULL, 140507065810944, 140507065843711, ++STORE, 140507065843712, 140507068137471, ++STORE, 140507065810944, 140507065843711, ++SNULL, 140507067940863, 140507068137471, ++STORE, 140507065843712, 140507067940863, ++STORE, 140507067940864, 140507068137471, ++SNULL, 140507067940864, 140507067949055, ++STORE, 140507067949056, 140507068137471, ++STORE, 140507067940864, 140507067949055, ++ERASE, 140507067940864, 140507067949055, ++STORE, 140507067940864, 140507067949055, ++ERASE, 140507067949056, 140507068137471, ++STORE, 140507067949056, 140507068137471, ++SNULL, 140507067944959, 140507067949055, ++STORE, 140507067940864, 140507067944959, ++STORE, 140507067944960, 140507067949055, ++SNULL, 140507070291967, 140507070296063, ++STORE, 140507070287872, 140507070291967, ++STORE, 140507070291968, 140507070296063, ++ERASE, 140507100692480, 140507100721151, ++STORE, 140507063705600, 140507065810943, ++SNULL, 140507063705600, 140507063709695, ++STORE, 140507063709696, 140507065810943, ++STORE, 140507063705600, 140507063709695, ++SNULL, 140507065802751, 140507065810943, ++STORE, 140507063709696, 140507065802751, ++STORE, 140507065802752, 140507065810943, ++ERASE, 140507065802752, 140507065810943, ++STORE, 140507065802752, 140507065810943, ++SNULL, 140507065806847, 140507065810943, ++STORE, 140507065802752, 140507065806847, ++STORE, 140507065806848, 140507065810943, ++STORE, 140507061600256, 140507063705599, ++SNULL, 140507061600256, 140507061604351, ++STORE, 140507061604352, 140507063705599, ++STORE, 140507061600256, 140507061604351, ++SNULL, 140507063697407, 140507063705599, ++STORE, 140507061604352, 140507063697407, ++STORE, 140507063697408, 140507063705599, ++ERASE, 140507063697408, 140507063705599, ++STORE, 140507063697408, 140507063705599, ++SNULL, 140507063701503, 140507063705599, ++STORE, 140507063697408, 140507063701503, ++STORE, 140507063701504, 140507063705599, ++STORE, 140507059490816, 140507061600255, ++SNULL, 140507059490816, 140507059499007, ++STORE, 140507059499008, 140507061600255, ++STORE, 140507059490816, 140507059499007, ++SNULL, 140507061592063, 140507061600255, ++STORE, 140507059499008, 140507061592063, ++STORE, 140507061592064, 140507061600255, ++ERASE, 140507061592064, 140507061600255, ++STORE, 140507061592064, 140507061600255, ++SNULL, 140507061596159, 140507061600255, ++STORE, 140507061592064, 140507061596159, ++STORE, 140507061596160, 140507061600255, ++STORE, 140507057377280, 140507059490815, ++SNULL, 140507057377280, 140507057389567, ++STORE, 140507057389568, 140507059490815, ++STORE, 140507057377280, 140507057389567, ++SNULL, 140507059482623, 140507059490815, ++STORE, 140507057389568, 140507059482623, ++STORE, 140507059482624, 140507059490815, ++ERASE, 140507059482624, 140507059490815, ++STORE, 140507059482624, 140507059490815, ++SNULL, 140507059486719, 140507059490815, ++STORE, 140507059482624, 140507059486719, ++STORE, 140507059486720, 140507059490815, ++STORE, 140507055255552, 140507057377279, ++SNULL, 140507055255552, 140507055276031, ++STORE, 140507055276032, 140507057377279, ++STORE, 140507055255552, 140507055276031, ++SNULL, 140507057369087, 140507057377279, ++STORE, 140507055276032, 140507057369087, ++STORE, 140507057369088, 140507057377279, ++ERASE, 140507057369088, 140507057377279, ++STORE, 140507057369088, 140507057377279, ++SNULL, 140507057373183, 140507057377279, ++STORE, 140507057369088, 140507057373183, ++STORE, 140507057373184, 140507057377279, ++STORE, 140507098693632, 140507098980351, ++SNULL, 140507098959871, 140507098980351, ++STORE, 140507098693632, 140507098959871, ++STORE, 140507098959872, 140507098980351, ++SNULL, 140507098959872, 140507098976255, ++STORE, 140507098976256, 140507098980351, ++STORE, 140507098959872, 140507098976255, ++ERASE, 140507098959872, 140507098976255, ++STORE, 140507098959872, 140507098976255, ++ERASE, 140507098976256, 140507098980351, ++STORE, 140507098976256, 140507098980351, ++STORE, 140507100692480, 140507100721151, ++STORE, 140507053125632, 140507055255551, ++SNULL, 140507053125632, 140507053154303, ++STORE, 140507053154304, 140507055255551, ++STORE, 140507053125632, 140507053154303, ++SNULL, 140507055247359, 140507055255551, ++STORE, 140507053154304, 140507055247359, ++STORE, 140507055247360, 140507055255551, ++ERASE, 140507055247360, 140507055255551, ++STORE, 140507055247360, 140507055255551, ++STORE, 140507051012096, 140507053125631, ++SNULL, 140507051012096, 140507051024383, ++STORE, 140507051024384, 140507053125631, ++STORE, 140507051012096, 140507051024383, ++SNULL, 140507053117439, 140507053125631, ++STORE, 140507051024384, 140507053117439, ++STORE, 140507053117440, 140507053125631, ++ERASE, 140507053117440, 140507053125631, ++STORE, 140507053117440, 140507053125631, ++SNULL, 140507053121535, 140507053125631, ++STORE, 140507053117440, 140507053121535, ++STORE, 140507053121536, 140507053125631, ++SNULL, 140507055251455, 140507055255551, ++STORE, 140507055247360, 140507055251455, ++STORE, 140507055251456, 140507055255551, ++SNULL, 140507098972159, 140507098976255, ++STORE, 140507098959872, 140507098972159, ++STORE, 140507098972160, 140507098976255, ++ERASE, 140507100692480, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100692480, 140507100721151, ++ERASE, 140507068137472, 140507068190719, ++ERASE, 140507068190720, 140507070287871, ++ERASE, 140507070287872, 140507070291967, ++ERASE, 140507070291968, 140507070296063, ++ERASE, 140507070296064, 140507070345215, ++ERASE, 140507065810944, 140507065843711, ++ERASE, 140507065843712, 140507067940863, ++ERASE, 140507067940864, 140507067944959, ++ERASE, 140507067944960, 140507067949055, ++ERASE, 140507067949056, 140507068137471, ++ERASE, 140507063705600, 140507063709695, ++ERASE, 140507063709696, 140507065802751, ++ERASE, 140507065802752, 140507065806847, ++ERASE, 140507065806848, 140507065810943, ++ERASE, 140507061600256, 140507061604351, ++ERASE, 140507061604352, 140507063697407, ++ERASE, 140507063697408, 140507063701503, ++ERASE, 140507063701504, 140507063705599, ++ERASE, 140507059490816, 140507059499007, ++ERASE, 140507059499008, 140507061592063, ++ERASE, 140507061592064, 140507061596159, ++ERASE, 140507061596160, 140507061600255, ++ERASE, 140507057377280, 140507057389567, ++ERASE, 140507057389568, 140507059482623, ++ERASE, 140507059482624, 140507059486719, ++ERASE, 140507059486720, 140507059490815, ++ERASE, 140507055255552, 140507055276031, ++ERASE, 140507055276032, 140507057369087, ++ERASE, 140507057369088, 140507057373183, ++ERASE, 140507057373184, 140507057377279, ++ERASE, 140507098693632, 140507098959871, ++ERASE, 140507098959872, 140507098972159, ++ERASE, 140507098972160, 140507098976255, ++ERASE, 140507098976256, 140507098980351, ++ERASE, 140507051012096, 140507051024383, ++ERASE, 140507051024384, 140507053117439, ++ERASE, 140507053117440, 140507053121535, ++ERASE, 140507053121536, 140507053125631, ++STORE, 94036448296960, 94036448509951, ++STORE, 94036450607104, 94036450611199, ++STORE, 94036450611200, 94036450619391, ++STORE, 94036450619392, 94036450631679, ++STORE, 94036482445312, 94036502376447, ++STORE, 140469487013888, 140469488672767, ++STORE, 140469488672768, 140469490769919, ++STORE, 140469490769920, 140469490786303, ++STORE, 140469490786304, 140469490794495, ++STORE, 140469490794496, 140469490810879, ++STORE, 140469490810880, 140469490823167, ++STORE, 140469490823168, 140469492916223, ++STORE, 140469492916224, 140469492920319, ++STORE, 140469492920320, 140469492924415, ++STORE, 140469492924416, 140469493067775, ++STORE, 140469493436416, 140469495119871, ++STORE, 140469495119872, 140469495136255, ++STORE, 140469495164928, 140469495169023, ++STORE, 140469495169024, 140469495173119, ++STORE, 140469495173120, 140469495177215, ++STORE, 140732281446400, 140732281585663, ++STORE, 140732282736640, 140732282748927, ++STORE, 140732282748928, 140732282753023, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723411931136, 140737488351231, ++SNULL, 140723411939327, 140737488351231, ++STORE, 140723411931136, 140723411939327, ++STORE, 140723411800064, 140723411939327, ++STORE, 93993768685568, 93993770909695, ++SNULL, 93993768796159, 93993770909695, ++STORE, 93993768685568, 93993768796159, ++STORE, 93993768796160, 93993770909695, ++ERASE, 93993768796160, 93993770909695, ++STORE, 93993770889216, 93993770901503, ++STORE, 93993770901504, 93993770909695, ++STORE, 140508681740288, 140508683993087, ++SNULL, 140508681883647, 140508683993087, ++STORE, 140508681740288, 140508681883647, ++STORE, 140508681883648, 140508683993087, ++ERASE, 140508681883648, 140508683993087, ++STORE, 140508683980800, 140508683988991, ++STORE, 140508683988992, 140508683993087, ++STORE, 140723412070400, 140723412074495, ++STORE, 140723412058112, 140723412070399, ++STORE, 140508683952128, 140508683980799, ++STORE, 140508683943936, 140508683952127, ++STORE, 140508677943296, 140508681740287, ++SNULL, 140508677943296, 140508679602175, ++STORE, 140508679602176, 140508681740287, ++STORE, 140508677943296, 140508679602175, ++SNULL, 140508681699327, 140508681740287, ++STORE, 140508679602176, 140508681699327, ++STORE, 140508681699328, 140508681740287, ++SNULL, 140508681699328, 140508681723903, ++STORE, 140508681723904, 140508681740287, ++STORE, 140508681699328, 140508681723903, ++ERASE, 140508681699328, 140508681723903, ++STORE, 140508681699328, 140508681723903, ++ERASE, 140508681723904, 140508681740287, ++STORE, 140508681723904, 140508681740287, ++SNULL, 140508681715711, 140508681723903, ++STORE, 140508681699328, 140508681715711, ++STORE, 140508681715712, 140508681723903, ++SNULL, 93993770897407, 93993770901503, ++STORE, 93993770889216, 93993770897407, ++STORE, 93993770897408, 93993770901503, ++SNULL, 140508683984895, 140508683988991, ++STORE, 140508683980800, 140508683984895, ++STORE, 140508683984896, 140508683988991, ++ERASE, 140508683952128, 140508683980799, ++STORE, 93993791582208, 93993791717375, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140734685458432, 140737488351231, ++SNULL, 140734685466623, 140737488351231, ++STORE, 140734685458432, 140734685466623, ++STORE, 140734685327360, 140734685466623, ++STORE, 93832321548288, 93832323772415, ++SNULL, 93832321658879, 93832323772415, ++STORE, 93832321548288, 93832321658879, ++STORE, 93832321658880, 93832323772415, ++ERASE, 93832321658880, 93832323772415, ++STORE, 93832323751936, 93832323764223, ++STORE, 93832323764224, 93832323772415, ++STORE, 140650945118208, 140650947371007, ++SNULL, 140650945261567, 140650947371007, ++STORE, 140650945118208, 140650945261567, ++STORE, 140650945261568, 140650947371007, ++ERASE, 140650945261568, 140650947371007, ++STORE, 140650947358720, 140650947366911, ++STORE, 140650947366912, 140650947371007, ++STORE, 140734686081024, 140734686085119, ++STORE, 140734686068736, 140734686081023, ++STORE, 140650947330048, 140650947358719, ++STORE, 140650947321856, 140650947330047, ++STORE, 140650941321216, 140650945118207, ++SNULL, 140650941321216, 140650942980095, ++STORE, 140650942980096, 140650945118207, ++STORE, 140650941321216, 140650942980095, ++SNULL, 140650945077247, 140650945118207, ++STORE, 140650942980096, 140650945077247, ++STORE, 140650945077248, 140650945118207, ++SNULL, 140650945077248, 140650945101823, ++STORE, 140650945101824, 140650945118207, ++STORE, 140650945077248, 140650945101823, ++ERASE, 140650945077248, 140650945101823, ++STORE, 140650945077248, 140650945101823, ++ERASE, 140650945101824, 140650945118207, ++STORE, 140650945101824, 140650945118207, ++SNULL, 140650945093631, 140650945101823, ++STORE, 140650945077248, 140650945093631, ++STORE, 140650945093632, 140650945101823, ++SNULL, 93832323760127, 93832323764223, ++STORE, 93832323751936, 93832323760127, ++STORE, 93832323760128, 93832323764223, ++SNULL, 140650947362815, 140650947366911, ++STORE, 140650947358720, 140650947362815, ++STORE, 140650947362816, 140650947366911, ++ERASE, 140650947330048, 140650947358719, ++STORE, 93832331890688, 93832332025855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728333520896, 140737488351231, ++SNULL, 140728333529087, 140737488351231, ++STORE, 140728333520896, 140728333529087, ++STORE, 140728333389824, 140728333529087, ++STORE, 94872734732288, 94872736956415, ++SNULL, 94872734842879, 94872736956415, ++STORE, 94872734732288, 94872734842879, ++STORE, 94872734842880, 94872736956415, ++ERASE, 94872734842880, 94872736956415, ++STORE, 94872736935936, 94872736948223, ++STORE, 94872736948224, 94872736956415, ++STORE, 139755193257984, 139755195510783, ++SNULL, 139755193401343, 139755195510783, ++STORE, 139755193257984, 139755193401343, ++STORE, 139755193401344, 139755195510783, ++ERASE, 139755193401344, 139755195510783, ++STORE, 139755195498496, 139755195506687, ++STORE, 139755195506688, 139755195510783, ++STORE, 140728333926400, 140728333930495, ++STORE, 140728333914112, 140728333926399, ++STORE, 139755195469824, 139755195498495, ++STORE, 139755195461632, 139755195469823, ++STORE, 139755189460992, 139755193257983, ++SNULL, 139755189460992, 139755191119871, ++STORE, 139755191119872, 139755193257983, ++STORE, 139755189460992, 139755191119871, ++SNULL, 139755193217023, 139755193257983, ++STORE, 139755191119872, 139755193217023, ++STORE, 139755193217024, 139755193257983, ++SNULL, 139755193217024, 139755193241599, ++STORE, 139755193241600, 139755193257983, ++STORE, 139755193217024, 139755193241599, ++ERASE, 139755193217024, 139755193241599, ++STORE, 139755193217024, 139755193241599, ++ERASE, 139755193241600, 139755193257983, ++STORE, 139755193241600, 139755193257983, ++SNULL, 139755193233407, 139755193241599, ++STORE, 139755193217024, 139755193233407, ++STORE, 139755193233408, 139755193241599, ++SNULL, 94872736944127, 94872736948223, ++STORE, 94872736935936, 94872736944127, ++STORE, 94872736944128, 94872736948223, ++SNULL, 139755195502591, 139755195506687, ++STORE, 139755195498496, 139755195502591, ++STORE, 139755195502592, 139755195506687, ++ERASE, 139755195469824, 139755195498495, ++STORE, 94872749744128, 94872749879295, ++STORE, 94720243642368, 94720243855359, ++STORE, 94720245952512, 94720245956607, ++STORE, 94720245956608, 94720245964799, ++STORE, 94720245964800, 94720245977087, ++STORE, 94720277745664, 94720278151167, ++STORE, 140453174497280, 140453176156159, ++STORE, 140453176156160, 140453178253311, ++STORE, 140453178253312, 140453178269695, ++STORE, 140453178269696, 140453178277887, ++STORE, 140453178277888, 140453178294271, ++STORE, 140453178294272, 140453178306559, ++STORE, 140453178306560, 140453180399615, ++STORE, 140453180399616, 140453180403711, ++STORE, 140453180403712, 140453180407807, ++STORE, 140453180407808, 140453180551167, ++STORE, 140453180919808, 140453182603263, ++STORE, 140453182603264, 140453182619647, ++STORE, 140453182648320, 140453182652415, ++STORE, 140453182652416, 140453182656511, ++STORE, 140453182656512, 140453182660607, ++STORE, 140733223923712, 140733224062975, ++STORE, 140733224808448, 140733224820735, ++STORE, 140733224820736, 140733224824831, ++STORE, 94321091141632, 94321091354623, ++STORE, 94321093451776, 94321093455871, ++STORE, 94321093455872, 94321093464063, ++STORE, 94321093464064, 94321093476351, ++STORE, 94321115873280, 94321117229055, ++STORE, 139695978840064, 139695980498943, ++STORE, 139695980498944, 139695982596095, ++STORE, 139695982596096, 139695982612479, ++STORE, 139695982612480, 139695982620671, ++STORE, 139695982620672, 139695982637055, ++STORE, 139695982637056, 139695982649343, ++STORE, 139695982649344, 139695984742399, ++STORE, 139695984742400, 139695984746495, ++STORE, 139695984746496, 139695984750591, ++STORE, 139695984750592, 139695984893951, ++STORE, 139695985262592, 139695986946047, ++STORE, 139695986946048, 139695986962431, ++STORE, 139695986991104, 139695986995199, ++STORE, 139695986995200, 139695986999295, ++STORE, 139695986999296, 139695987003391, ++STORE, 140734650564608, 140734650703871, ++STORE, 140734650785792, 140734650798079, ++STORE, 140734650798080, 140734650802175, ++STORE, 94523438456832, 94523438669823, ++STORE, 94523440766976, 94523440771071, ++STORE, 94523440771072, 94523440779263, ++STORE, 94523440779264, 94523440791551, ++STORE, 94523464544256, 94523465842687, ++STORE, 140453231493120, 140453233151999, ++STORE, 140453233152000, 140453235249151, ++STORE, 140453235249152, 140453235265535, ++STORE, 140453235265536, 140453235273727, ++STORE, 140453235273728, 140453235290111, ++STORE, 140453235290112, 140453235302399, ++STORE, 140453235302400, 140453237395455, ++STORE, 140453237395456, 140453237399551, ++STORE, 140453237399552, 140453237403647, ++STORE, 140453237403648, 140453237547007, ++STORE, 140453237915648, 140453239599103, ++STORE, 140453239599104, 140453239615487, ++STORE, 140453239644160, 140453239648255, ++STORE, 140453239648256, 140453239652351, ++STORE, 140453239652352, 140453239656447, ++STORE, 140734679445504, 140734679584767, ++STORE, 140734680018944, 140734680031231, ++STORE, 140734680031232, 140734680035327, ++STORE, 94614776987648, 94614777200639, ++STORE, 94614779297792, 94614779301887, ++STORE, 94614779301888, 94614779310079, ++STORE, 94614779310080, 94614779322367, ++STORE, 94614798467072, 94614800699391, ++STORE, 139677037182976, 139677038841855, ++STORE, 139677038841856, 139677040939007, ++STORE, 139677040939008, 139677040955391, ++STORE, 139677040955392, 139677040963583, ++STORE, 139677040963584, 139677040979967, ++STORE, 139677040979968, 139677040992255, ++STORE, 139677040992256, 139677043085311, ++STORE, 139677043085312, 139677043089407, ++STORE, 139677043089408, 139677043093503, ++STORE, 139677043093504, 139677043236863, ++STORE, 139677043605504, 139677045288959, ++STORE, 139677045288960, 139677045305343, ++STORE, 139677045334016, 139677045338111, ++STORE, 139677045338112, 139677045342207, ++STORE, 139677045342208, 139677045346303, ++STORE, 140721604411392, 140721604550655, ++STORE, 140721606135808, 140721606148095, ++STORE, 140721606148096, 140721606152191, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140729280544768, 140737488351231, ++SNULL, 140729280552959, 140737488351231, ++STORE, 140729280544768, 140729280552959, ++STORE, 140729280413696, 140729280552959, ++STORE, 94863939334144, 94863941558271, ++SNULL, 94863939444735, 94863941558271, ++STORE, 94863939334144, 94863939444735, ++STORE, 94863939444736, 94863941558271, ++ERASE, 94863939444736, 94863941558271, ++STORE, 94863941537792, 94863941550079, ++STORE, 94863941550080, 94863941558271, ++STORE, 139691047276544, 139691049529343, ++SNULL, 139691047419903, 139691049529343, ++STORE, 139691047276544, 139691047419903, ++STORE, 139691047419904, 139691049529343, ++ERASE, 139691047419904, 139691049529343, ++STORE, 139691049517056, 139691049525247, ++STORE, 139691049525248, 139691049529343, ++STORE, 140729281679360, 140729281683455, ++STORE, 140729281667072, 140729281679359, ++STORE, 139691049488384, 139691049517055, ++STORE, 139691049480192, 139691049488383, ++STORE, 139691043479552, 139691047276543, ++SNULL, 139691043479552, 139691045138431, ++STORE, 139691045138432, 139691047276543, ++STORE, 139691043479552, 139691045138431, ++SNULL, 139691047235583, 139691047276543, ++STORE, 139691045138432, 139691047235583, ++STORE, 139691047235584, 139691047276543, ++SNULL, 139691047235584, 139691047260159, ++STORE, 139691047260160, 139691047276543, ++STORE, 139691047235584, 139691047260159, ++ERASE, 139691047235584, 139691047260159, ++STORE, 139691047235584, 139691047260159, ++ERASE, 139691047260160, 139691047276543, ++STORE, 139691047260160, 139691047276543, ++SNULL, 139691047251967, 139691047260159, ++STORE, 139691047235584, 139691047251967, ++STORE, 139691047251968, 139691047260159, ++SNULL, 94863941545983, 94863941550079, ++STORE, 94863941537792, 94863941545983, ++STORE, 94863941545984, 94863941550079, ++SNULL, 139691049521151, 139691049525247, ++STORE, 139691049517056, 139691049521151, ++STORE, 139691049521152, 139691049525247, ++ERASE, 139691049488384, 139691049517055, ++STORE, 94863951294464, 94863951429631, ++STORE, 93998209294336, 93998209507327, ++STORE, 93998211604480, 93998211608575, ++STORE, 93998211608576, 93998211616767, ++STORE, 93998211616768, 93998211629055, ++STORE, 93998227210240, 93998227615743, ++STORE, 140243029913600, 140243031572479, ++STORE, 140243031572480, 140243033669631, ++STORE, 140243033669632, 140243033686015, ++STORE, 140243033686016, 140243033694207, ++STORE, 140243033694208, 140243033710591, ++STORE, 140243033710592, 140243033722879, ++STORE, 140243033722880, 140243035815935, ++STORE, 140243035815936, 140243035820031, ++STORE, 140243035820032, 140243035824127, ++STORE, 140243035824128, 140243035967487, ++STORE, 140243036336128, 140243038019583, ++STORE, 140243038019584, 140243038035967, ++STORE, 140243038064640, 140243038068735, ++STORE, 140243038068736, 140243038072831, ++STORE, 140243038072832, 140243038076927, ++STORE, 140734976479232, 140734976618495, ++STORE, 140734977978368, 140734977990655, ++STORE, 140734977990656, 140734977994751, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722742775808, 140737488351231, ++SNULL, 140722742783999, 140737488351231, ++STORE, 140722742775808, 140722742783999, ++STORE, 140722742644736, 140722742783999, ++STORE, 93857673662464, 93857675997183, ++SNULL, 93857673875455, 93857675997183, ++STORE, 93857673662464, 93857673875455, ++STORE, 93857673875456, 93857675997183, ++ERASE, 93857673875456, 93857675997183, ++STORE, 93857675972608, 93857675984895, ++STORE, 93857675984896, 93857675997183, ++STORE, 140629677498368, 140629679751167, ++SNULL, 140629677641727, 140629679751167, ++STORE, 140629677498368, 140629677641727, ++STORE, 140629677641728, 140629679751167, ++ERASE, 140629677641728, 140629679751167, ++STORE, 140629679738880, 140629679747071, ++STORE, 140629679747072, 140629679751167, ++STORE, 140722743222272, 140722743226367, ++STORE, 140722743209984, 140722743222271, ++STORE, 140629679710208, 140629679738879, ++STORE, 140629679702016, 140629679710207, ++STORE, 140629675384832, 140629677498367, ++SNULL, 140629675384832, 140629675397119, ++STORE, 140629675397120, 140629677498367, ++STORE, 140629675384832, 140629675397119, ++SNULL, 140629677490175, 140629677498367, ++STORE, 140629675397120, 140629677490175, ++STORE, 140629677490176, 140629677498367, ++ERASE, 140629677490176, 140629677498367, ++STORE, 140629677490176, 140629677498367, ++STORE, 140629671587840, 140629675384831, ++SNULL, 140629671587840, 140629673246719, ++STORE, 140629673246720, 140629675384831, ++STORE, 140629671587840, 140629673246719, ++SNULL, 140629675343871, 140629675384831, ++STORE, 140629673246720, 140629675343871, ++STORE, 140629675343872, 140629675384831, ++SNULL, 140629675343872, 140629675368447, ++STORE, 140629675368448, 140629675384831, ++STORE, 140629675343872, 140629675368447, ++ERASE, 140629675343872, 140629675368447, ++STORE, 140629675343872, 140629675368447, ++ERASE, 140629675368448, 140629675384831, ++STORE, 140629675368448, 140629675384831, ++STORE, 140629679693824, 140629679710207, ++SNULL, 140629675360255, 140629675368447, ++STORE, 140629675343872, 140629675360255, ++STORE, 140629675360256, 140629675368447, ++SNULL, 140629677494271, 140629677498367, ++STORE, 140629677490176, 140629677494271, ++STORE, 140629677494272, 140629677498367, ++SNULL, 93857675976703, 93857675984895, ++STORE, 93857675972608, 93857675976703, ++STORE, 93857675976704, 93857675984895, ++SNULL, 140629679742975, 140629679747071, ++STORE, 140629679738880, 140629679742975, ++STORE, 140629679742976, 140629679747071, ++ERASE, 140629679710208, 140629679738879, ++STORE, 93857705832448, 93857705967615, ++STORE, 140629678010368, 140629679693823, ++STORE, 93857705832448, 93857706102783, ++STORE, 93857705832448, 93857706237951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735922421760, 140737488351231, ++SNULL, 140735922429951, 140737488351231, ++STORE, 140735922421760, 140735922429951, ++STORE, 140735922290688, 140735922429951, ++STORE, 94651136139264, 94651138363391, ++SNULL, 94651136249855, 94651138363391, ++STORE, 94651136139264, 94651136249855, ++STORE, 94651136249856, 94651138363391, ++ERASE, 94651136249856, 94651138363391, ++STORE, 94651138342912, 94651138355199, ++STORE, 94651138355200, 94651138363391, ++STORE, 140325788266496, 140325790519295, ++SNULL, 140325788409855, 140325790519295, ++STORE, 140325788266496, 140325788409855, ++STORE, 140325788409856, 140325790519295, ++ERASE, 140325788409856, 140325790519295, ++STORE, 140325790507008, 140325790515199, ++STORE, 140325790515200, 140325790519295, ++STORE, 140735923572736, 140735923576831, ++STORE, 140735923560448, 140735923572735, ++STORE, 140325790478336, 140325790507007, ++STORE, 140325790470144, 140325790478335, ++STORE, 140325784469504, 140325788266495, ++SNULL, 140325784469504, 140325786128383, ++STORE, 140325786128384, 140325788266495, ++STORE, 140325784469504, 140325786128383, ++SNULL, 140325788225535, 140325788266495, ++STORE, 140325786128384, 140325788225535, ++STORE, 140325788225536, 140325788266495, ++SNULL, 140325788225536, 140325788250111, ++STORE, 140325788250112, 140325788266495, ++STORE, 140325788225536, 140325788250111, ++ERASE, 140325788225536, 140325788250111, ++STORE, 140325788225536, 140325788250111, ++ERASE, 140325788250112, 140325788266495, ++STORE, 140325788250112, 140325788266495, ++SNULL, 140325788241919, 140325788250111, ++STORE, 140325788225536, 140325788241919, ++STORE, 140325788241920, 140325788250111, ++SNULL, 94651138351103, 94651138355199, ++STORE, 94651138342912, 94651138351103, ++STORE, 94651138351104, 94651138355199, ++SNULL, 140325790511103, 140325790515199, ++STORE, 140325790507008, 140325790511103, ++STORE, 140325790511104, 140325790515199, ++ERASE, 140325790478336, 140325790507007, ++STORE, 94651146297344, 94651146432511, ++STORE, 94212330168320, 94212330381311, ++STORE, 94212332478464, 94212332482559, ++STORE, 94212332482560, 94212332490751, ++STORE, 94212332490752, 94212332503039, ++STORE, 94212348891136, 94212349825023, ++STORE, 140611630604288, 140611632263167, ++STORE, 140611632263168, 140611634360319, ++STORE, 140611634360320, 140611634376703, ++STORE, 140611634376704, 140611634384895, ++STORE, 140611634384896, 140611634401279, ++STORE, 140611634401280, 140611634413567, ++STORE, 140611634413568, 140611636506623, ++STORE, 140611636506624, 140611636510719, ++STORE, 140611636510720, 140611636514815, ++STORE, 140611636514816, 140611636658175, ++STORE, 140611637026816, 140611638710271, ++STORE, 140611638710272, 140611638726655, ++STORE, 140611638755328, 140611638759423, ++STORE, 140611638759424, 140611638763519, ++STORE, 140611638763520, 140611638767615, ++STORE, 140726974533632, 140726974672895, ++STORE, 140726974943232, 140726974955519, ++STORE, 140726974955520, 140726974959615, ++STORE, 94572463521792, 94572463734783, ++STORE, 94572465831936, 94572465836031, ++STORE, 94572465836032, 94572465844223, ++STORE, 94572465844224, 94572465856511, ++STORE, 94572491534336, 94572492865535, ++STORE, 140644351492096, 140644353150975, ++STORE, 140644353150976, 140644355248127, ++STORE, 140644355248128, 140644355264511, ++STORE, 140644355264512, 140644355272703, ++STORE, 140644355272704, 140644355289087, ++STORE, 140644355289088, 140644355301375, ++STORE, 140644355301376, 140644357394431, ++STORE, 140644357394432, 140644357398527, ++STORE, 140644357398528, 140644357402623, ++STORE, 140644357402624, 140644357545983, ++STORE, 140644357914624, 140644359598079, ++STORE, 140644359598080, 140644359614463, ++STORE, 140644359643136, 140644359647231, ++STORE, 140644359647232, 140644359651327, ++STORE, 140644359651328, 140644359655423, ++STORE, 140727841824768, 140727841964031, ++STORE, 140727843188736, 140727843201023, ++STORE, 140727843201024, 140727843205119, ++STORE, 94144315457536, 94144315670527, ++STORE, 94144317767680, 94144317771775, ++STORE, 94144317771776, 94144317779967, ++STORE, 94144317779968, 94144317792255, ++STORE, 94144318369792, 94144320815103, ++STORE, 140316717645824, 140316719304703, ++STORE, 140316719304704, 140316721401855, ++STORE, 140316721401856, 140316721418239, ++STORE, 140316721418240, 140316721426431, ++STORE, 140316721426432, 140316721442815, ++STORE, 140316721442816, 140316721455103, ++STORE, 140316721455104, 140316723548159, ++STORE, 140316723548160, 140316723552255, ++STORE, 140316723552256, 140316723556351, ++STORE, 140316723556352, 140316723699711, ++STORE, 140316724068352, 140316725751807, ++STORE, 140316725751808, 140316725768191, ++STORE, 140316725796864, 140316725800959, ++STORE, 140316725800960, 140316725805055, ++STORE, 140316725805056, 140316725809151, ++STORE, 140725744283648, 140725744422911, ++STORE, 140725745852416, 140725745864703, ++STORE, 140725745864704, 140725745868799, ++STORE, 94646858846208, 94646859059199, ++STORE, 94646861156352, 94646861160447, ++STORE, 94646861160448, 94646861168639, ++STORE, 94646861168640, 94646861180927, ++STORE, 94646879805440, 94646881894399, ++STORE, 140435449745408, 140435451404287, ++STORE, 140435451404288, 140435453501439, ++STORE, 140435453501440, 140435453517823, ++STORE, 140435453517824, 140435453526015, ++STORE, 140435453526016, 140435453542399, ++STORE, 140435453542400, 140435453554687, ++STORE, 140435453554688, 140435455647743, ++STORE, 140435455647744, 140435455651839, ++STORE, 140435455651840, 140435455655935, ++STORE, 140435455655936, 140435455799295, ++STORE, 140435456167936, 140435457851391, ++STORE, 140435457851392, 140435457867775, ++STORE, 140435457896448, 140435457900543, ++STORE, 140435457900544, 140435457904639, ++STORE, 140435457904640, 140435457908735, ++STORE, 140721033818112, 140721033957375, ++STORE, 140721034018816, 140721034031103, ++STORE, 140721034031104, 140721034035199, ++STORE, 94872903438336, 94872903651327, ++STORE, 94872905748480, 94872905752575, ++STORE, 94872905752576, 94872905760767, ++STORE, 94872905760768, 94872905773055, ++STORE, 94872931246080, 94872931651583, ++STORE, 139771607810048, 139771609468927, ++STORE, 139771609468928, 139771611566079, ++STORE, 139771611566080, 139771611582463, ++STORE, 139771611582464, 139771611590655, ++STORE, 139771611590656, 139771611607039, ++STORE, 139771611607040, 139771611619327, ++STORE, 139771611619328, 139771613712383, ++STORE, 139771613712384, 139771613716479, ++STORE, 139771613716480, 139771613720575, ++STORE, 139771613720576, 139771613863935, ++STORE, 139771614232576, 139771615916031, ++STORE, 139771615916032, 139771615932415, ++STORE, 139771615961088, 139771615965183, ++STORE, 139771615965184, 139771615969279, ++STORE, 139771615969280, 139771615973375, ++STORE, 140725402931200, 140725403070463, ++STORE, 140725403852800, 140725403865087, ++STORE, 140725403865088, 140725403869183, ++STORE, 94740737736704, 94740737949695, ++STORE, 94740740046848, 94740740050943, ++STORE, 94740740050944, 94740740059135, ++STORE, 94740740059136, 94740740071423, ++STORE, 94740743249920, 94740744724479, ++STORE, 140640287010816, 140640288669695, ++STORE, 140640288669696, 140640290766847, ++STORE, 140640290766848, 140640290783231, ++STORE, 140640290783232, 140640290791423, ++STORE, 140640290791424, 140640290807807, ++STORE, 140640290807808, 140640290820095, ++STORE, 140640290820096, 140640292913151, ++STORE, 140640292913152, 140640292917247, ++STORE, 140640292917248, 140640292921343, ++STORE, 140640292921344, 140640293064703, ++STORE, 140640293433344, 140640295116799, ++STORE, 140640295116800, 140640295133183, ++STORE, 140640295161856, 140640295165951, ++STORE, 140640295165952, 140640295170047, ++STORE, 140640295170048, 140640295174143, ++STORE, 140725133303808, 140725133443071, ++STORE, 140725133684736, 140725133697023, ++STORE, 140725133697024, 140725133701119, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722826371072, 140737488351231, ++SNULL, 140722826375167, 140737488351231, ++STORE, 140722826371072, 140722826375167, ++STORE, 140722826240000, 140722826375167, ++STORE, 94113818611712, 94113820835839, ++SNULL, 94113818722303, 94113820835839, ++STORE, 94113818611712, 94113818722303, ++STORE, 94113818722304, 94113820835839, ++ERASE, 94113818722304, 94113820835839, ++STORE, 94113820815360, 94113820827647, ++STORE, 94113820827648, 94113820835839, ++STORE, 139628194508800, 139628196761599, ++SNULL, 139628194652159, 139628196761599, ++STORE, 139628194508800, 139628194652159, ++STORE, 139628194652160, 139628196761599, ++ERASE, 139628194652160, 139628196761599, ++STORE, 139628196749312, 139628196757503, ++STORE, 139628196757504, 139628196761599, ++STORE, 140722826727424, 140722826731519, ++STORE, 140722826715136, 140722826727423, ++STORE, 139628196720640, 139628196749311, ++STORE, 139628196712448, 139628196720639, ++STORE, 139628190711808, 139628194508799, ++SNULL, 139628190711808, 139628192370687, ++STORE, 139628192370688, 139628194508799, ++STORE, 139628190711808, 139628192370687, ++SNULL, 139628194467839, 139628194508799, ++STORE, 139628192370688, 139628194467839, ++STORE, 139628194467840, 139628194508799, ++SNULL, 139628194467840, 139628194492415, ++STORE, 139628194492416, 139628194508799, ++STORE, 139628194467840, 139628194492415, ++ERASE, 139628194467840, 139628194492415, ++STORE, 139628194467840, 139628194492415, ++ERASE, 139628194492416, 139628194508799, ++STORE, 139628194492416, 139628194508799, ++SNULL, 139628194484223, 139628194492415, ++STORE, 139628194467840, 139628194484223, ++STORE, 139628194484224, 139628194492415, ++SNULL, 94113820823551, 94113820827647, ++STORE, 94113820815360, 94113820823551, ++STORE, 94113820823552, 94113820827647, ++SNULL, 139628196753407, 139628196757503, ++STORE, 139628196749312, 139628196753407, ++STORE, 139628196753408, 139628196757503, ++ERASE, 139628196720640, 139628196749311, ++STORE, 94113830850560, 94113830985727, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731865833472, 140737488351231, ++SNULL, 140731865841663, 140737488351231, ++STORE, 140731865833472, 140731865841663, ++STORE, 140731865702400, 140731865841663, ++STORE, 94763339386880, 94763341611007, ++SNULL, 94763339497471, 94763341611007, ++STORE, 94763339386880, 94763339497471, ++STORE, 94763339497472, 94763341611007, ++ERASE, 94763339497472, 94763341611007, ++STORE, 94763341590528, 94763341602815, ++STORE, 94763341602816, 94763341611007, ++STORE, 139778398486528, 139778400739327, ++SNULL, 139778398629887, 139778400739327, ++STORE, 139778398486528, 139778398629887, ++STORE, 139778398629888, 139778400739327, ++ERASE, 139778398629888, 139778400739327, ++STORE, 139778400727040, 139778400735231, ++STORE, 139778400735232, 139778400739327, ++STORE, 140731865858048, 140731865862143, ++STORE, 140731865845760, 140731865858047, ++STORE, 139778400698368, 139778400727039, ++STORE, 139778400690176, 139778400698367, ++STORE, 139778394689536, 139778398486527, ++SNULL, 139778394689536, 139778396348415, ++STORE, 139778396348416, 139778398486527, ++STORE, 139778394689536, 139778396348415, ++SNULL, 139778398445567, 139778398486527, ++STORE, 139778396348416, 139778398445567, ++STORE, 139778398445568, 139778398486527, ++SNULL, 139778398445568, 139778398470143, ++STORE, 139778398470144, 139778398486527, ++STORE, 139778398445568, 139778398470143, ++ERASE, 139778398445568, 139778398470143, ++STORE, 139778398445568, 139778398470143, ++ERASE, 139778398470144, 139778398486527, ++STORE, 139778398470144, 139778398486527, ++SNULL, 139778398461951, 139778398470143, ++STORE, 139778398445568, 139778398461951, ++STORE, 139778398461952, 139778398470143, ++SNULL, 94763341598719, 94763341602815, ++STORE, 94763341590528, 94763341598719, ++STORE, 94763341598720, 94763341602815, ++SNULL, 139778400731135, 139778400735231, ++STORE, 139778400727040, 139778400731135, ++STORE, 139778400731136, 139778400735231, ++ERASE, 139778400698368, 139778400727039, ++STORE, 94763362197504, 94763362332671, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140732053192704, 140737488351231, ++SNULL, 140732053204991, 140737488351231, ++STORE, 140732053192704, 140732053204991, ++STORE, 140732053061632, 140732053204991, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 140176018599936, 140176020852735, ++SNULL, 140176018743295, 140176020852735, ++STORE, 140176018599936, 140176018743295, ++STORE, 140176018743296, 140176020852735, ++ERASE, 140176018743296, 140176020852735, ++STORE, 140176020840448, 140176020848639, ++STORE, 140176020848640, 140176020852735, ++STORE, 140732053381120, 140732053385215, ++STORE, 140732053368832, 140732053381119, ++STORE, 140176020811776, 140176020840447, ++STORE, 140176020803584, 140176020811775, ++STORE, 140176014766080, 140176018599935, ++SNULL, 140176014766080, 140176016474111, ++STORE, 140176016474112, 140176018599935, ++STORE, 140176014766080, 140176016474111, ++SNULL, 140176018567167, 140176018599935, ++STORE, 140176016474112, 140176018567167, ++STORE, 140176018567168, 140176018599935, ++ERASE, 140176018567168, 140176018599935, ++STORE, 140176018567168, 140176018599935, ++STORE, 140176012570624, 140176014766079, ++SNULL, 140176012570624, 140176012664831, ++STORE, 140176012664832, 140176014766079, ++STORE, 140176012570624, 140176012664831, ++SNULL, 140176014757887, 140176014766079, ++STORE, 140176012664832, 140176014757887, ++STORE, 140176014757888, 140176014766079, ++ERASE, 140176014757888, 140176014766079, ++STORE, 140176014757888, 140176014766079, ++STORE, 140176010051584, 140176012570623, ++SNULL, 140176010051584, 140176010465279, ++STORE, 140176010465280, 140176012570623, ++STORE, 140176010051584, 140176010465279, ++SNULL, 140176012558335, 140176012570623, ++STORE, 140176010465280, 140176012558335, ++STORE, 140176012558336, 140176012570623, ++ERASE, 140176012558336, 140176012570623, ++STORE, 140176012558336, 140176012570623, ++STORE, 140176007417856, 140176010051583, ++SNULL, 140176007417856, 140176007946239, ++STORE, 140176007946240, 140176010051583, ++STORE, 140176007417856, 140176007946239, ++SNULL, 140176010043391, 140176010051583, ++STORE, 140176007946240, 140176010043391, ++STORE, 140176010043392, 140176010051583, ++ERASE, 140176010043392, 140176010051583, ++STORE, 140176010043392, 140176010051583, ++STORE, 140176005304320, 140176007417855, ++SNULL, 140176005304320, 140176005316607, ++STORE, 140176005316608, 140176007417855, ++STORE, 140176005304320, 140176005316607, ++SNULL, 140176007409663, 140176007417855, ++STORE, 140176005316608, 140176007409663, ++STORE, 140176007409664, 140176007417855, ++ERASE, 140176007409664, 140176007417855, ++STORE, 140176007409664, 140176007417855, ++STORE, 140176003100672, 140176005304319, ++SNULL, 140176003100672, 140176003203071, ++STORE, 140176003203072, 140176005304319, ++STORE, 140176003100672, 140176003203071, ++SNULL, 140176005296127, 140176005304319, ++STORE, 140176003203072, 140176005296127, ++STORE, 140176005296128, 140176005304319, ++ERASE, 140176005296128, 140176005304319, ++STORE, 140176005296128, 140176005304319, ++STORE, 140176020795392, 140176020811775, ++STORE, 140175999938560, 140176003100671, ++SNULL, 140175999938560, 140176000999423, ++STORE, 140176000999424, 140176003100671, ++STORE, 140175999938560, 140176000999423, ++SNULL, 140176003092479, 140176003100671, ++STORE, 140176000999424, 140176003092479, ++STORE, 140176003092480, 140176003100671, ++ERASE, 140176003092480, 140176003100671, ++STORE, 140176003092480, 140176003100671, ++STORE, 140175996141568, 140175999938559, ++SNULL, 140175996141568, 140175997800447, ++STORE, 140175997800448, 140175999938559, ++STORE, 140175996141568, 140175997800447, ++SNULL, 140175999897599, 140175999938559, ++STORE, 140175997800448, 140175999897599, ++STORE, 140175999897600, 140175999938559, ++SNULL, 140175999897600, 140175999922175, ++STORE, 140175999922176, 140175999938559, ++STORE, 140175999897600, 140175999922175, ++ERASE, 140175999897600, 140175999922175, ++STORE, 140175999897600, 140175999922175, ++ERASE, 140175999922176, 140175999938559, ++STORE, 140175999922176, 140175999938559, ++STORE, 140176020783104, 140176020811775, ++SNULL, 140175999913983, 140175999922175, ++STORE, 140175999897600, 140175999913983, ++STORE, 140175999913984, 140175999922175, ++SNULL, 140176003096575, 140176003100671, ++STORE, 140176003092480, 140176003096575, ++STORE, 140176003096576, 140176003100671, ++SNULL, 140176005300223, 140176005304319, ++STORE, 140176005296128, 140176005300223, ++STORE, 140176005300224, 140176005304319, ++SNULL, 140176007413759, 140176007417855, ++STORE, 140176007409664, 140176007413759, ++STORE, 140176007413760, 140176007417855, ++SNULL, 140176010047487, 140176010051583, ++STORE, 140176010043392, 140176010047487, ++STORE, 140176010047488, 140176010051583, ++SNULL, 140176012566527, 140176012570623, ++STORE, 140176012558336, 140176012566527, ++STORE, 140176012566528, 140176012570623, ++SNULL, 140176014761983, 140176014766079, ++STORE, 140176014757888, 140176014761983, ++STORE, 140176014761984, 140176014766079, ++SNULL, 140176018571263, 140176018599935, ++STORE, 140176018567168, 140176018571263, ++STORE, 140176018571264, 140176018599935, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 140176020844543, 140176020848639, ++STORE, 140176020840448, 140176020844543, ++STORE, 140176020844544, 140176020848639, ++ERASE, 140176020811776, 140176020840447, ++STORE, 53080064, 53215231, ++STORE, 140176019099648, 140176020783103, ++STORE, 140176020836352, 140176020840447, ++STORE, 140176018964480, 140176019099647, ++STORE, 53080064, 53358591, ++STORE, 140175994044416, 140175996141567, ++STORE, 140176020828160, 140176020840447, ++STORE, 140176020819968, 140176020840447, ++STORE, 140176020783104, 140176020819967, ++STORE, 140176018948096, 140176019099647, ++STORE, 53080064, 53493759, ++STORE, 53080064, 53649407, ++STORE, 140176018939904, 140176019099647, ++STORE, 140176018931712, 140176019099647, ++STORE, 53080064, 53784575, ++STORE, 53080064, 53919743, ++STORE, 140176018915328, 140176019099647, ++STORE, 140176018907136, 140176019099647, ++STORE, 53080064, 54059007, ++STORE, 140175993769984, 140175996141567, ++STORE, 140176018747392, 140176019099647, ++STORE, 53080064, 54198271, ++SNULL, 54190079, 54198271, ++STORE, 53080064, 54190079, ++STORE, 54190080, 54198271, ++ERASE, 54190080, 54198271, ++SNULL, 54181887, 54190079, ++STORE, 53080064, 54181887, ++STORE, 54181888, 54190079, ++ERASE, 54181888, 54190079, ++SNULL, 54173695, 54181887, ++STORE, 53080064, 54173695, ++STORE, 54173696, 54181887, ++ERASE, 54173696, 54181887, ++SNULL, 54165503, 54173695, ++STORE, 53080064, 54165503, ++STORE, 54165504, 54173695, ++ERASE, 54165504, 54173695, ++STORE, 140175993753600, 140175996141567, ++STORE, 140175993688064, 140175996141567, ++STORE, 140175993655296, 140175996141567, ++STORE, 140175991558144, 140175996141567, ++STORE, 140175991492608, 140175996141567, ++STORE, 53080064, 54312959, ++STORE, 140175991361536, 140175996141567, ++STORE, 140175991099392, 140175996141567, ++STORE, 140175991091200, 140175996141567, ++STORE, 140175991074816, 140175996141567, ++STORE, 140175991066624, 140175996141567, ++STORE, 140175991058432, 140175996141567, ++STORE, 53080064, 54448127, ++SNULL, 54439935, 54448127, ++STORE, 53080064, 54439935, ++STORE, 54439936, 54448127, ++ERASE, 54439936, 54448127, ++SNULL, 54431743, 54439935, ++STORE, 53080064, 54431743, ++STORE, 54431744, 54439935, ++ERASE, 54431744, 54439935, ++SNULL, 54419455, 54431743, ++STORE, 53080064, 54419455, ++STORE, 54419456, 54431743, ++ERASE, 54419456, 54431743, ++SNULL, 54403071, 54419455, ++STORE, 53080064, 54403071, ++STORE, 54403072, 54419455, ++ERASE, 54403072, 54419455, ++STORE, 140175991042048, 140175996141567, ++STORE, 53080064, 54538239, ++SNULL, 54534143, 54538239, ++STORE, 53080064, 54534143, ++STORE, 54534144, 54538239, ++ERASE, 54534144, 54538239, ++SNULL, 54530047, 54534143, ++STORE, 53080064, 54530047, ++STORE, 54530048, 54534143, ++ERASE, 54530048, 54534143, ++SNULL, 54525951, 54530047, ++STORE, 53080064, 54525951, ++STORE, 54525952, 54530047, ++ERASE, 54525952, 54530047, ++SNULL, 54521855, 54525951, ++STORE, 53080064, 54521855, ++STORE, 54521856, 54525951, ++ERASE, 54521856, 54525951, ++SNULL, 54517759, 54521855, ++STORE, 53080064, 54517759, ++STORE, 54517760, 54521855, ++ERASE, 54517760, 54521855, ++SNULL, 54513663, 54517759, ++STORE, 53080064, 54513663, ++STORE, 54513664, 54517759, ++ERASE, 54513664, 54517759, ++SNULL, 54509567, 54513663, ++STORE, 53080064, 54509567, ++STORE, 54509568, 54513663, ++ERASE, 54509568, 54513663, ++STORE, 140175991025664, 140175996141567, ++STORE, 140175990992896, 140175996141567, ++STORE, 53080064, 54644735, ++SNULL, 54628351, 54644735, ++STORE, 53080064, 54628351, ++STORE, 54628352, 54644735, ++ERASE, 54628352, 54644735, ++SNULL, 54616063, 54628351, ++STORE, 53080064, 54616063, ++STORE, 54616064, 54628351, ++ERASE, 54616064, 54628351, ++STORE, 140175988895744, 140175996141567, ++STORE, 53080064, 54767615, ++STORE, 140175988879360, 140175996141567, ++STORE, 140175988617216, 140175996141567, ++STORE, 140175988609024, 140175996141567, ++STORE, 140175988600832, 140175996141567, ++STORE, 53080064, 54906879, ++SNULL, 54898687, 54906879, ++STORE, 53080064, 54898687, ++STORE, 54898688, 54906879, ++ERASE, 54898688, 54906879, ++SNULL, 54853631, 54898687, ++STORE, 53080064, 54853631, ++STORE, 54853632, 54898687, ++ERASE, 54853632, 54898687, ++STORE, 140175986503680, 140175996141567, ++STORE, 53080064, 54996991, ++STORE, 140175986495488, 140175996141567, ++STORE, 140175986487296, 140175996141567, ++STORE, 140175985438720, 140175996141567, ++STORE, 53080064, 55136255, ++STORE, 140175985405952, 140175996141567, ++STORE, 140175985139712, 140175996141567, ++SNULL, 140176018964479, 140176019099647, ++STORE, 140176018747392, 140176018964479, ++STORE, 140176018964480, 140176019099647, ++ERASE, 140176018964480, 140176019099647, ++STORE, 140175983042560, 140175996141567, ++STORE, 140175982518272, 140175996141567, ++STORE, 140175980421120, 140175996141567, ++STORE, 53080064, 55287807, ++STORE, 53080064, 55427071, ++STORE, 140176019091456, 140176019099647, ++STORE, 140176019083264, 140176019099647, ++STORE, 140176019075072, 140176019099647, ++STORE, 140176019066880, 140176019099647, ++STORE, 140176019058688, 140176019099647, ++STORE, 140175980158976, 140175996141567, ++STORE, 140176019050496, 140176019099647, ++STORE, 140176019042304, 140176019099647, ++STORE, 140176019034112, 140176019099647, ++STORE, 140176019025920, 140176019099647, ++STORE, 140176019017728, 140176019099647, ++STORE, 140176019009536, 140176019099647, ++STORE, 140176019001344, 140176019099647, ++STORE, 140176018993152, 140176019099647, ++STORE, 140176018984960, 140176019099647, ++STORE, 140176018976768, 140176019099647, ++STORE, 140176018968576, 140176019099647, ++STORE, 140175978061824, 140175996141567, ++STORE, 53080064, 55603199, ++STORE, 140175978029056, 140175996141567, ++STORE, 140175977996288, 140175996141567, ++STORE, 53080064, 55738367, ++STORE, 53080064, 55881727, ++STORE, 140175977963520, 140175996141567, ++STORE, 140175977930752, 140175996141567, ++STORE, 53080064, 56041471, ++STORE, 140175977897984, 140175996141567, ++STORE, 140175977865216, 140175996141567, ++SNULL, 55881727, 56041471, ++STORE, 53080064, 55881727, ++STORE, 55881728, 56041471, ++ERASE, 55881728, 56041471, ++SNULL, 55721983, 55881727, ++STORE, 53080064, 55721983, ++STORE, 55721984, 55881727, ++ERASE, 55721984, 55881727, ++SNULL, 55570431, 55721983, ++STORE, 53080064, 55570431, ++STORE, 55570432, 55721983, ++ERASE, 55570432, 55721983, ++STORE, 140175977857024, 140175996141567, ++STORE, 140175975759872, 140175996141567, ++STORE, 53080064, 55754751, ++STORE, 53080064, 55943167, ++STORE, 140175975751680, 140175996141567, ++STORE, 140175975743488, 140175996141567, ++STORE, 140175975735296, 140175996141567, ++STORE, 140175975727104, 140175996141567, ++STORE, 140175975718912, 140175996141567, ++STORE, 140175975710720, 140175996141567, ++STORE, 140175975702528, 140175996141567, ++STORE, 140175975694336, 140175996141567, ++STORE, 140175975686144, 140175996141567, ++STORE, 140175975677952, 140175996141567, ++STORE, 140175975669760, 140175996141567, ++STORE, 140175974621184, 140175996141567, ++STORE, 140175974612992, 140175996141567, ++STORE, 53080064, 56139775, ++STORE, 140175972515840, 140175996141567, ++STORE, 53080064, 56401919, ++STORE, 140175970418688, 140175996141567, ++STORE, 140175970410496, 140175996141567, ++STORE, 140175970402304, 140175996141567, ++STORE, 140175970394112, 140175996141567, ++STORE, 53080064, 56569855, ++STORE, 140175969865728, 140175996141567, ++SNULL, 140175985139711, 140175996141567, ++STORE, 140175969865728, 140175985139711, ++STORE, 140175985139712, 140175996141567, ++SNULL, 140175985139712, 140175985405951, ++STORE, 140175985405952, 140175996141567, ++STORE, 140175985139712, 140175985405951, ++ERASE, 140175985139712, 140175985405951, ++STORE, 140175965671424, 140175985139711, ++STORE, 140175985397760, 140175996141567, ++STORE, 140175985389568, 140175996141567, ++STORE, 140175985381376, 140175996141567, ++STORE, 140175985373184, 140175996141567, ++STORE, 140175985364992, 140175996141567, ++STORE, 140175985356800, 140175996141567, ++STORE, 140175985348608, 140175996141567, ++STORE, 140175985340416, 140175996141567, ++STORE, 140175985332224, 140175996141567, ++STORE, 140175985324032, 140175996141567, ++STORE, 140175985315840, 140175996141567, ++STORE, 140175985307648, 140175996141567, ++STORE, 140175985299456, 140175996141567, ++STORE, 140175985291264, 140175996141567, ++STORE, 140175985283072, 140175996141567, ++STORE, 140175985274880, 140175996141567, ++STORE, 140175963574272, 140175985139711, ++STORE, 140175985266688, 140175996141567, ++STORE, 140175961477120, 140175985139711, ++STORE, 53080064, 56831999, ++STORE, 140175959379968, 140175985139711, ++STORE, 140175985258496, 140175996141567, ++STORE, 140175957282816, 140175985139711, ++STORE, 140175985250304, 140175996141567, ++STORE, 140175985242112, 140175996141567, ++STORE, 140175985233920, 140175996141567, ++STORE, 140175985225728, 140175996141567, ++STORE, 140175985217536, 140175996141567, ++STORE, 140175957151744, 140175985139711, ++STORE, 140175956627456, 140175985139711, ++SNULL, 140175980158975, 140175985139711, ++STORE, 140175956627456, 140175980158975, ++STORE, 140175980158976, 140175985139711, ++SNULL, 140175980158976, 140175980421119, ++STORE, 140175980421120, 140175985139711, ++STORE, 140175980158976, 140175980421119, ++ERASE, 140175980158976, 140175980421119, ++STORE, 140175954530304, 140175980158975, ++STORE, 140175985209344, 140175996141567, ++STORE, 53080064, 57094143, ++STORE, 140175952433152, 140175980158975, ++STORE, 140175985192960, 140175996141567, ++STORE, 140175985184768, 140175996141567, ++STORE, 140175985176576, 140175996141567, ++STORE, 140175985168384, 140175996141567, ++STORE, 140175985160192, 140175996141567, ++STORE, 140175985152000, 140175996141567, ++STORE, 140175985143808, 140175996141567, ++STORE, 140175980412928, 140175985139711, ++STORE, 140175980404736, 140175985139711, ++STORE, 140175980396544, 140175985139711, ++STORE, 140175980388352, 140175985139711, ++STORE, 140175980380160, 140175985139711, ++STORE, 140175980371968, 140175985139711, ++STORE, 140175980363776, 140175985139711, ++STORE, 140175980355584, 140175985139711, ++STORE, 140175980347392, 140175985139711, ++STORE, 140175980339200, 140175985139711, ++STORE, 53080064, 57356287, ++SNULL, 140176018747392, 140176018907135, ++STORE, 140176018907136, 140176018964479, ++STORE, 140176018747392, 140176018907135, ++ERASE, 140176018747392, 140176018907135, ++STORE, 140175952146432, 140175980158975, ++STORE, 140175950049280, 140175980158975, ++SNULL, 140175952146431, 140175980158975, ++STORE, 140175950049280, 140175952146431, ++STORE, 140175952146432, 140175980158975, ++SNULL, 140175952146432, 140175952433151, ++STORE, 140175952433152, 140175980158975, ++STORE, 140175952146432, 140175952433151, ++ERASE, 140175952146432, 140175952433151, ++STORE, 140176018898944, 140176018964479, ++STORE, 53080064, 57749503, ++STORE, 140175949520896, 140175952146431, ++STORE, 140175947423744, 140175952146431, ++SNULL, 140175993769983, 140175996141567, ++STORE, 140175985143808, 140175993769983, ++STORE, 140175993769984, 140175996141567, ++SNULL, 140175993769984, 140175994044415, ++STORE, 140175994044416, 140175996141567, ++STORE, 140175993769984, 140175994044415, ++ERASE, 140175993769984, 140175994044415, ++STORE, 140176018890752, 140176018964479, ++STORE, 140176018882560, 140176018964479, ++STORE, 140176018874368, 140176018964479, ++STORE, 140176018866176, 140176018964479, ++STORE, 140176018849792, 140176018964479, ++STORE, 140176018841600, 140176018964479, ++STORE, 140176018825216, 140176018964479, ++STORE, 140176018817024, 140176018964479, ++STORE, 140176018800640, 140176018964479, ++STORE, 140176018792448, 140176018964479, ++STORE, 140176018759680, 140176018964479, ++STORE, 140176018751488, 140176018964479, ++STORE, 140175994028032, 140175996141567, ++STORE, 140176018743296, 140176018964479, ++STORE, 140175994011648, 140175996141567, ++STORE, 140175994003456, 140175996141567, ++STORE, 140175993987072, 140175996141567, ++STORE, 140175993978880, 140175996141567, ++STORE, 140175993946112, 140175996141567, ++STORE, 140175993937920, 140175996141567, ++STORE, 140175993921536, 140175996141567, ++STORE, 140175993913344, 140175996141567, ++STORE, 140175993896960, 140175996141567, ++STORE, 140175993888768, 140175996141567, ++STORE, 140175993872384, 140175996141567, ++STORE, 140175993864192, 140175996141567, ++STORE, 140175993831424, 140175996141567, ++STORE, 140175993823232, 140175996141567, ++STORE, 140175993806848, 140175996141567, ++STORE, 140175993798656, 140175996141567, ++STORE, 140175993782272, 140175996141567, ++STORE, 140175993774080, 140175996141567, ++STORE, 140175980322816, 140175985139711, ++STORE, 140175980314624, 140175985139711, ++STORE, 140175980281856, 140175985139711, ++STORE, 140175980273664, 140175985139711, ++STORE, 140175980257280, 140175985139711, ++STORE, 140175945326592, 140175952146431, ++STORE, 140175980249088, 140175985139711, ++STORE, 140175980232704, 140175985139711, ++STORE, 140175980224512, 140175985139711, ++STORE, 140175980208128, 140175985139711, ++STORE, 140175980199936, 140175985139711, ++STORE, 140175980167168, 140175985139711, ++STORE, 140175952433152, 140175985139711, ++STORE, 140175952416768, 140175985139711, ++STORE, 140175952408576, 140175985139711, ++STORE, 140175952392192, 140175985139711, ++STORE, 140175952384000, 140175985139711, ++STORE, 140175952367616, 140175985139711, ++STORE, 140175943229440, 140175952146431, ++STORE, 140175952359424, 140175985139711, ++STORE, 140175952326656, 140175985139711, ++STORE, 140175952318464, 140175985139711, ++STORE, 140175952302080, 140175985139711, ++STORE, 140175952293888, 140175985139711, ++STORE, 140175952277504, 140175985139711, ++STORE, 140175952269312, 140175985139711, ++STORE, 140175952252928, 140175985139711, ++STORE, 140175952244736, 140175985139711, ++STORE, 140175952211968, 140175985139711, ++STORE, 140175952203776, 140175985139711, ++STORE, 140175952187392, 140175985139711, ++STORE, 140175952179200, 140175985139711, ++STORE, 140175952162816, 140175985139711, ++STORE, 140175952154624, 140175985139711, ++STORE, 140175943213056, 140175952146431, ++STORE, 140175943213056, 140175985139711, ++STORE, 140175943180288, 140175985139711, ++STORE, 140175943172096, 140175985139711, ++STORE, 140175943155712, 140175985139711, ++STORE, 140175943147520, 140175985139711, ++STORE, 140175943131136, 140175985139711, ++STORE, 140175943122944, 140175985139711, ++STORE, 140175943106560, 140175985139711, ++STORE, 140175943098368, 140175985139711, ++STORE, 140175943065600, 140175985139711, ++STORE, 140175943057408, 140175985139711, ++STORE, 140175943041024, 140175985139711, ++STORE, 140175943032832, 140175985139711, ++STORE, 140175943016448, 140175985139711, ++STORE, 140175943008256, 140175985139711, ++STORE, 140175942991872, 140175985139711, ++STORE, 140175942983680, 140175985139711, ++STORE, 140175942950912, 140175985139711, ++STORE, 140175942942720, 140175985139711, ++STORE, 140175942926336, 140175985139711, ++STORE, 140175942918144, 140175985139711, ++STORE, 140175942901760, 140175985139711, ++STORE, 140175942893568, 140175985139711, ++STORE, 140175942877184, 140175985139711, ++STORE, 140175942868992, 140175985139711, ++STORE, 140175942836224, 140175985139711, ++STORE, 140175942828032, 140175985139711, ++STORE, 140175942811648, 140175985139711, ++STORE, 140175942803456, 140175985139711, ++STORE, 140175942787072, 140175985139711, ++STORE, 140175942778880, 140175985139711, ++STORE, 140175942762496, 140175985139711, ++STORE, 140175942754304, 140175985139711, ++STORE, 140175942721536, 140175985139711, ++STORE, 140175942713344, 140175985139711, ++STORE, 140175942696960, 140175985139711, ++STORE, 140175942688768, 140175985139711, ++STORE, 140175942672384, 140175985139711, ++STORE, 140175942664192, 140175985139711, ++STORE, 140175942647808, 140175985139711, ++STORE, 140175942639616, 140175985139711, ++STORE, 140175942606848, 140175985139711, ++STORE, 140175942598656, 140175985139711, ++STORE, 140175942582272, 140175985139711, ++STORE, 140175942574080, 140175985139711, ++STORE, 140175942557696, 140175985139711, ++STORE, 140175942549504, 140175985139711, ++STORE, 140175942533120, 140175985139711, ++STORE, 140175942524928, 140175985139711, ++STORE, 140175942492160, 140175985139711, ++STORE, 140175942483968, 140175985139711, ++STORE, 140175942467584, 140175985139711, ++STORE, 140175942459392, 140175985139711, ++STORE, 140175942443008, 140175985139711, ++STORE, 140175942434816, 140175985139711, ++STORE, 140175942418432, 140175985139711, ++STORE, 140175942410240, 140175985139711, ++STORE, 140175942377472, 140175985139711, ++STORE, 140175942369280, 140175985139711, ++STORE, 140175942352896, 140175985139711, ++STORE, 140175942344704, 140175985139711, ++STORE, 140175942328320, 140175985139711, ++STORE, 140175942320128, 140175985139711, ++STORE, 140175942303744, 140175985139711, ++STORE, 140175942295552, 140175985139711, ++STORE, 140175942262784, 140175985139711, ++STORE, 140175942254592, 140175985139711, ++STORE, 140175942238208, 140175985139711, ++STORE, 140175942230016, 140175985139711, ++STORE, 140175942213632, 140175985139711, ++STORE, 140175942205440, 140175985139711, ++STORE, 140175942189056, 140175985139711, ++STORE, 140175942180864, 140175985139711, ++STORE, 140175942148096, 140175985139711, ++STORE, 140175942139904, 140175985139711, ++STORE, 140175942123520, 140175985139711, ++STORE, 140175942115328, 140175985139711, ++STORE, 140175942098944, 140175985139711, ++STORE, 140175942090752, 140175985139711, ++STORE, 140175942074368, 140175985139711, ++STORE, 140175942066176, 140175985139711, ++STORE, 140175942033408, 140175985139711, ++STORE, 140175942025216, 140175985139711, ++STORE, 140175942008832, 140175985139711, ++STORE, 140175942000640, 140175985139711, ++STORE, 140175941984256, 140175985139711, ++STORE, 140175941976064, 140175985139711, ++STORE, 140175941959680, 140175985139711, ++STORE, 140175939862528, 140175985139711, ++STORE, 140175939854336, 140175985139711, ++STORE, 140175939821568, 140175985139711, ++STORE, 140175939813376, 140175985139711, ++STORE, 140175939796992, 140175985139711, ++STORE, 140175939788800, 140175985139711, ++STORE, 140175939772416, 140175985139711, ++STORE, 140175939764224, 140175985139711, ++STORE, 140175939747840, 140175985139711, ++STORE, 140175939739648, 140175985139711, ++STORE, 140175939706880, 140175985139711, ++STORE, 140175939698688, 140175985139711, ++STORE, 140175939682304, 140175985139711, ++STORE, 140175939674112, 140175985139711, ++STORE, 140175939657728, 140175985139711, ++STORE, 140175939649536, 140175985139711, ++STORE, 140175939633152, 140175985139711, ++STORE, 140175939624960, 140175985139711, ++STORE, 140175939592192, 140175985139711, ++STORE, 140175939584000, 140175985139711, ++STORE, 140175939567616, 140175985139711, ++STORE, 140175939559424, 140175985139711, ++STORE, 140175939543040, 140175985139711, ++STORE, 140175939534848, 140175985139711, ++STORE, 140175939518464, 140175985139711, ++STORE, 140175939510272, 140175985139711, ++STORE, 140175939477504, 140175985139711, ++STORE, 140175939469312, 140175985139711, ++STORE, 140175939452928, 140175985139711, ++STORE, 140175939444736, 140175985139711, ++STORE, 140175939428352, 140175985139711, ++STORE, 140175939420160, 140175985139711, ++STORE, 140175939403776, 140175985139711, ++STORE, 140175939395584, 140175985139711, ++STORE, 140175939362816, 140175985139711, ++STORE, 140175939354624, 140175985139711, ++STORE, 140175939338240, 140175985139711, ++STORE, 140175939330048, 140175985139711, ++STORE, 140175939313664, 140175985139711, ++STORE, 140175939305472, 140175985139711, ++STORE, 140175939289088, 140175985139711, ++STORE, 140175939280896, 140175985139711, ++STORE, 140175939248128, 140175985139711, ++STORE, 140175939239936, 140175985139711, ++STORE, 140175939223552, 140175985139711, ++STORE, 140175939215360, 140175985139711, ++STORE, 140175939198976, 140175985139711, ++STORE, 140175939190784, 140175985139711, ++STORE, 140175939174400, 140175985139711, ++STORE, 140175939166208, 140175985139711, ++STORE, 140175939133440, 140175985139711, ++STORE, 140175939125248, 140175985139711, ++STORE, 140175939108864, 140175985139711, ++STORE, 140175939100672, 140175985139711, ++STORE, 140175939084288, 140175985139711, ++STORE, 140175939076096, 140175985139711, ++STORE, 140175939059712, 140175985139711, ++STORE, 140175939051520, 140175985139711, ++STORE, 140175939018752, 140175985139711, ++STORE, 140175939010560, 140175985139711, ++STORE, 140175938994176, 140175985139711, ++STORE, 140175938985984, 140175985139711, ++STORE, 140175938969600, 140175985139711, ++STORE, 140175938961408, 140175985139711, ++STORE, 140175938945024, 140175985139711, ++STORE, 140175938936832, 140175985139711, ++STORE, 140175938904064, 140175985139711, ++STORE, 140175938895872, 140175985139711, ++STORE, 140175938879488, 140175985139711, ++STORE, 140175938871296, 140175985139711, ++STORE, 140175938854912, 140175985139711, ++STORE, 140175938846720, 140175985139711, ++STORE, 140175938830336, 140175985139711, ++STORE, 140175938822144, 140175985139711, ++STORE, 140175938789376, 140175985139711, ++STORE, 140175938781184, 140175985139711, ++STORE, 140175938764800, 140175985139711, ++STORE, 140175938756608, 140175985139711, ++STORE, 140175938740224, 140175985139711, ++STORE, 140175938732032, 140175985139711, ++STORE, 140175938715648, 140175985139711, ++STORE, 140175938707456, 140175985139711, ++STORE, 140175938674688, 140175985139711, ++STORE, 140175938666496, 140175985139711, ++STORE, 140175938650112, 140175985139711, ++STORE, 140175938641920, 140175985139711, ++STORE, 140175938625536, 140175985139711, ++STORE, 140175938617344, 140175985139711, ++STORE, 140175938600960, 140175985139711, ++STORE, 140175938592768, 140175985139711, ++STORE, 140175938560000, 140175985139711, ++STORE, 140175938551808, 140175985139711, ++STORE, 140175938535424, 140175985139711, ++STORE, 140175938527232, 140175985139711, ++STORE, 140175938510848, 140175985139711, ++STORE, 140175938502656, 140175985139711, ++STORE, 140175938486272, 140175985139711, ++STORE, 140175938478080, 140175985139711, ++STORE, 140175938445312, 140175985139711, ++STORE, 140175938437120, 140175985139711, ++STORE, 140175938420736, 140175985139711, ++STORE, 140175938412544, 140175985139711, ++STORE, 140175938396160, 140175985139711, ++STORE, 140175938387968, 140175985139711, ++STORE, 140175938371584, 140175985139711, ++STORE, 140175938363392, 140175985139711, ++STORE, 140175938330624, 140175985139711, ++STORE, 140175938322432, 140175985139711, ++STORE, 140175938306048, 140175985139711, ++STORE, 140175938297856, 140175985139711, ++STORE, 140175938281472, 140175985139711, ++STORE, 140175938273280, 140175985139711, ++STORE, 140175938256896, 140175985139711, ++STORE, 140175938248704, 140175985139711, ++STORE, 140175938215936, 140175985139711, ++STORE, 140175938207744, 140175985139711, ++STORE, 140175938191360, 140175985139711, ++STORE, 140175938183168, 140175985139711, ++STORE, 140175938166784, 140175985139711, ++STORE, 140175938158592, 140175985139711, ++STORE, 140175938142208, 140175985139711, ++STORE, 140175936045056, 140175985139711, ++STORE, 140175936036864, 140175985139711, ++STORE, 140175936004096, 140175985139711, ++STORE, 140175935995904, 140175985139711, ++STORE, 140175935979520, 140175985139711, ++STORE, 140175935971328, 140175985139711, ++STORE, 140175935954944, 140175985139711, ++STORE, 140175935946752, 140175985139711, ++STORE, 140175935930368, 140175985139711, ++STORE, 140175935922176, 140175985139711, ++STORE, 140175935889408, 140175985139711, ++STORE, 140175935881216, 140175985139711, ++STORE, 140175935864832, 140175985139711, ++STORE, 140175935856640, 140175985139711, ++STORE, 140175935840256, 140175985139711, ++STORE, 140175935832064, 140175985139711, ++STORE, 140175935815680, 140175985139711, ++STORE, 140175935807488, 140175985139711, ++STORE, 140175935774720, 140175985139711, ++STORE, 140175935766528, 140175985139711, ++STORE, 140175935750144, 140175985139711, ++STORE, 140175935741952, 140175985139711, ++STORE, 140175935725568, 140175985139711, ++STORE, 140175935717376, 140175985139711, ++STORE, 140175935700992, 140175985139711, ++STORE, 140175935692800, 140175985139711, ++STORE, 140175935660032, 140175985139711, ++STORE, 140175935651840, 140175985139711, ++STORE, 140175935635456, 140175985139711, ++STORE, 140175935627264, 140175985139711, ++STORE, 140175935610880, 140175985139711, ++STORE, 140175935602688, 140175985139711, ++STORE, 140175935586304, 140175985139711, ++STORE, 140175935578112, 140175985139711, ++STORE, 140175935545344, 140175985139711, ++STORE, 140175935537152, 140175985139711, ++STORE, 140175935520768, 140175985139711, ++STORE, 140175935512576, 140175985139711, ++STORE, 140175935496192, 140175985139711, ++STORE, 140175935488000, 140175985139711, ++STORE, 140175935471616, 140175985139711, ++STORE, 140175935463424, 140175985139711, ++STORE, 140175935430656, 140175985139711, ++STORE, 140175935422464, 140175985139711, ++STORE, 140175935406080, 140175985139711, ++STORE, 140175935397888, 140175985139711, ++STORE, 140175935381504, 140175985139711, ++STORE, 140175935373312, 140175985139711, ++STORE, 140175935356928, 140175985139711, ++STORE, 140175935348736, 140175985139711, ++STORE, 140175935315968, 140175985139711, ++STORE, 140175935307776, 140175985139711, ++STORE, 140175935291392, 140175985139711, ++STORE, 140175935283200, 140175985139711, ++STORE, 140175935266816, 140175985139711, ++STORE, 140175935258624, 140175985139711, ++STORE, 140175935242240, 140175985139711, ++STORE, 140175935234048, 140175985139711, ++STORE, 140175935201280, 140175985139711, ++STORE, 140175935193088, 140175985139711, ++STORE, 140175935176704, 140175985139711, ++STORE, 140175935168512, 140175985139711, ++STORE, 140175935152128, 140175985139711, ++STORE, 140175935143936, 140175985139711, ++STORE, 140175935127552, 140175985139711, ++STORE, 140175935119360, 140175985139711, ++STORE, 140175935086592, 140175985139711, ++STORE, 140175935078400, 140175985139711, ++STORE, 140175935062016, 140175985139711, ++STORE, 140175935053824, 140175985139711, ++STORE, 140175935037440, 140175985139711, ++STORE, 140175935029248, 140175985139711, ++STORE, 140175935012864, 140175985139711, ++STORE, 140175935004672, 140175985139711, ++STORE, 140175934971904, 140175985139711, ++STORE, 140175934963712, 140175985139711, ++STORE, 140175934947328, 140175985139711, ++STORE, 140175934939136, 140175985139711, ++STORE, 140175934922752, 140175985139711, ++STORE, 140175934914560, 140175985139711, ++STORE, 140175934898176, 140175985139711, ++STORE, 140175934889984, 140175985139711, ++STORE, 140175934857216, 140175985139711, ++STORE, 140175934849024, 140175985139711, ++STORE, 140175934832640, 140175985139711, ++STORE, 140175934824448, 140175985139711, ++STORE, 140175934808064, 140175985139711, ++STORE, 140175934799872, 140175985139711, ++STORE, 140175934783488, 140175985139711, ++STORE, 140175934775296, 140175985139711, ++STORE, 140175934742528, 140175985139711, ++STORE, 140175934734336, 140175985139711, ++STORE, 140175934717952, 140175985139711, ++STORE, 140175934709760, 140175985139711, ++STORE, 140175934693376, 140175985139711, ++STORE, 140175934685184, 140175985139711, ++STORE, 140175934668800, 140175985139711, ++STORE, 140175934660608, 140175985139711, ++STORE, 140175934627840, 140175985139711, ++STORE, 140175934619648, 140175985139711, ++STORE, 140175934603264, 140175985139711, ++STORE, 140175934595072, 140175985139711, ++STORE, 140175934578688, 140175985139711, ++STORE, 140175934570496, 140175985139711, ++STORE, 140175934554112, 140175985139711, ++STORE, 140175934545920, 140175985139711, ++STORE, 140175934513152, 140175985139711, ++STORE, 140175934504960, 140175985139711, ++STORE, 140175934488576, 140175985139711, ++STORE, 140175934480384, 140175985139711, ++STORE, 140175934464000, 140175985139711, ++STORE, 140175934455808, 140175985139711, ++STORE, 140175934439424, 140175985139711, ++STORE, 140175934431232, 140175985139711, ++STORE, 140175934398464, 140175985139711, ++STORE, 140175934390272, 140175985139711, ++STORE, 140175934373888, 140175985139711, ++STORE, 140175934365696, 140175985139711, ++STORE, 140175934349312, 140175985139711, ++STORE, 140175934341120, 140175985139711, ++STORE, 140175934324736, 140175985139711, ++STORE, 140175932227584, 140175985139711, ++STORE, 140175932219392, 140175985139711, ++STORE, 140175932186624, 140175985139711, ++STORE, 140175932178432, 140175985139711, ++STORE, 140175932162048, 140175985139711, ++STORE, 140175932153856, 140175985139711, ++STORE, 140175932137472, 140175985139711, ++STORE, 53080064, 57884671, ++STORE, 140175932129280, 140175985139711, ++STORE, 140175932112896, 140175985139711, ++STORE, 140175932104704, 140175985139711, ++STORE, 140175932071936, 140175985139711, ++STORE, 140175932063744, 140175985139711, ++STORE, 140175932047360, 140175985139711, ++STORE, 140175932039168, 140175985139711, ++STORE, 140175932022784, 140175985139711, ++STORE, 140175932014592, 140175985139711, ++STORE, 140175931998208, 140175985139711, ++STORE, 140175931990016, 140175985139711, ++STORE, 140175931957248, 140175985139711, ++STORE, 140175931949056, 140175985139711, ++STORE, 140175931932672, 140175985139711, ++STORE, 140175931924480, 140175985139711, ++STORE, 140175931908096, 140175985139711, ++STORE, 140175931899904, 140175985139711, ++STORE, 140175931883520, 140175985139711, ++STORE, 140175931875328, 140175985139711, ++STORE, 140175931842560, 140175985139711, ++STORE, 140175931834368, 140175985139711, ++STORE, 140175931817984, 140175985139711, ++STORE, 140175931809792, 140175985139711, ++STORE, 140175931793408, 140175985139711, ++STORE, 140175931785216, 140175985139711, ++STORE, 140175931768832, 140175985139711, ++STORE, 140175931760640, 140175985139711, ++STORE, 140175931727872, 140175985139711, ++STORE, 140175931719680, 140175985139711, ++STORE, 140175931703296, 140175985139711, ++STORE, 140175931695104, 140175985139711, ++STORE, 140175931678720, 140175985139711, ++STORE, 140175931670528, 140175985139711, ++STORE, 140175931654144, 140175985139711, ++STORE, 140175931645952, 140175985139711, ++STORE, 140175931613184, 140175985139711, ++STORE, 140175931604992, 140175985139711, ++STORE, 140175931588608, 140175985139711, ++STORE, 140175931580416, 140175985139711, ++STORE, 140175931564032, 140175985139711, ++STORE, 140175931555840, 140175985139711, ++STORE, 140175931539456, 140175985139711, ++STORE, 140175931531264, 140175985139711, ++STORE, 140175931498496, 140175985139711, ++STORE, 140175931490304, 140175985139711, ++STORE, 140175931473920, 140175985139711, ++STORE, 140175931465728, 140175985139711, ++STORE, 140175931449344, 140175985139711, ++STORE, 140175931441152, 140175985139711, ++STORE, 140175931424768, 140175985139711, ++STORE, 140175931416576, 140175985139711, ++STORE, 140175931383808, 140175985139711, ++STORE, 140175931375616, 140175985139711, ++STORE, 140175931359232, 140175985139711, ++STORE, 140175931351040, 140175985139711, ++STORE, 140175931334656, 140175985139711, ++STORE, 140175931326464, 140175985139711, ++STORE, 140175931310080, 140175985139711, ++STORE, 140175931301888, 140175985139711, ++STORE, 140175931269120, 140175985139711, ++STORE, 140175931260928, 140175985139711, ++STORE, 140175931244544, 140175985139711, ++STORE, 140175931236352, 140175985139711, ++STORE, 140175931219968, 140175985139711, ++STORE, 140175931211776, 140175985139711, ++STORE, 140175931195392, 140175985139711, ++STORE, 140175931187200, 140175985139711, ++STORE, 140175931154432, 140175985139711, ++STORE, 140175931146240, 140175985139711, ++STORE, 140175931129856, 140175985139711, ++STORE, 140175931121664, 140175985139711, ++STORE, 140175931105280, 140175985139711, ++STORE, 140175931097088, 140175985139711, ++STORE, 140175931080704, 140175985139711, ++STORE, 140175931072512, 140175985139711, ++STORE, 140175931039744, 140175985139711, ++STORE, 140175931031552, 140175985139711, ++STORE, 140175931015168, 140175985139711, ++STORE, 140175931006976, 140175985139711, ++STORE, 140175930990592, 140175985139711, ++STORE, 140175930982400, 140175985139711, ++STORE, 140175930966016, 140175985139711, ++STORE, 140175930957824, 140175985139711, ++STORE, 140175930925056, 140175985139711, ++STORE, 140175930916864, 140175985139711, ++STORE, 140175930900480, 140175985139711, ++STORE, 140175930892288, 140175985139711, ++STORE, 140175930875904, 140175985139711, ++STORE, 140175930867712, 140175985139711, ++STORE, 140175930851328, 140175985139711, ++STORE, 140175930843136, 140175985139711, ++STORE, 140175930810368, 140175985139711, ++STORE, 140175930802176, 140175985139711, ++STORE, 140175930785792, 140175985139711, ++STORE, 140175930777600, 140175985139711, ++STORE, 140175930761216, 140175985139711, ++STORE, 140175930753024, 140175985139711, ++STORE, 140175930736640, 140175985139711, ++STORE, 140175930728448, 140175985139711, ++STORE, 140175930695680, 140175985139711, ++STORE, 140175930687488, 140175985139711, ++STORE, 140175930671104, 140175985139711, ++STORE, 140175930662912, 140175985139711, ++STORE, 140175930646528, 140175985139711, ++STORE, 140175930638336, 140175985139711, ++STORE, 140175930621952, 140175985139711, ++STORE, 140175930613760, 140175985139711, ++STORE, 140175930580992, 140175985139711, ++STORE, 140175930572800, 140175985139711, ++STORE, 140175930556416, 140175985139711, ++STORE, 140175930548224, 140175985139711, ++STORE, 140175930531840, 140175985139711, ++STORE, 140175930523648, 140175985139711, ++STORE, 140175930507264, 140175985139711, ++STORE, 140175928410112, 140175985139711, ++STORE, 140175928401920, 140175985139711, ++STORE, 140175928369152, 140175985139711, ++STORE, 140175928360960, 140175985139711, ++STORE, 140175928344576, 140175985139711, ++STORE, 140175928336384, 140175985139711, ++STORE, 140175928320000, 140175985139711, ++STORE, 140175928311808, 140175985139711, ++STORE, 140175928295424, 140175985139711, ++STORE, 140175927242752, 140175985139711, ++SNULL, 140175956627455, 140175985139711, ++STORE, 140175927242752, 140175956627455, ++STORE, 140175956627456, 140175985139711, ++ }; ++ unsigned long set24[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735281639424, 140737488351231, ++SNULL, 140735281643519, 140737488351231, ++STORE, 140735281639424, 140735281643519, ++STORE, 140735281508352, 140735281643519, ++STORE, 94717834911744, 94717834928127, ++SNULL, 94717834915839, 94717834928127, ++STORE, 94717834911744, 94717834915839, ++STORE, 94717834915840, 94717834928127, ++ERASE, 94717834915840, 94717834928127, ++STORE, 94717834919936, 94717834928127, ++STORE, 140428246065152, 140428248317951, ++SNULL, 140428246208511, 140428248317951, ++STORE, 140428246065152, 140428246208511, ++STORE, 140428246208512, 140428248317951, ++ERASE, 140428246208512, 140428248317951, ++STORE, 140428248305664, 140428248313855, ++STORE, 140428248313856, 140428248317951, ++STORE, 140735281811456, 140735281815551, ++STORE, 140735281799168, 140735281811455, ++STORE, 140428248297472, 140428248305663, ++STORE, 140428243841024, 140428246065151, ++SNULL, 140428245491711, 140428246065151, ++STORE, 140428243841024, 140428245491711, ++STORE, 140428245491712, 140428246065151, ++SNULL, 140428245491712, 140428246061055, ++STORE, 140428246061056, 140428246065151, ++STORE, 140428245491712, 140428246061055, ++ERASE, 140428245491712, 140428246061055, ++STORE, 140428245491712, 140428246061055, ++ERASE, 140428246061056, 140428246065151, ++STORE, 140428246061056, 140428246065151, ++STORE, 140428248268800, 140428248297471, ++STORE, 140428241625088, 140428243841023, ++SNULL, 140428241625088, 140428241723391, ++STORE, 140428241723392, 140428243841023, ++STORE, 140428241625088, 140428241723391, ++SNULL, 140428243816447, 140428243841023, ++STORE, 140428241723392, 140428243816447, ++STORE, 140428243816448, 140428243841023, ++SNULL, 140428243816448, 140428243824639, ++STORE, 140428243824640, 140428243841023, ++STORE, 140428243816448, 140428243824639, ++ERASE, 140428243816448, 140428243824639, ++STORE, 140428243816448, 140428243824639, ++ERASE, 140428243824640, 140428243841023, ++STORE, 140428243824640, 140428243841023, ++STORE, 140428237828096, 140428241625087, ++SNULL, 140428237828096, 140428239486975, ++STORE, 140428239486976, 140428241625087, ++STORE, 140428237828096, 140428239486975, ++SNULL, 140428241584127, 140428241625087, ++STORE, 140428239486976, 140428241584127, ++STORE, 140428241584128, 140428241625087, ++SNULL, 140428241584128, 140428241608703, ++STORE, 140428241608704, 140428241625087, ++STORE, 140428241584128, 140428241608703, ++ERASE, 140428241584128, 140428241608703, ++STORE, 140428241584128, 140428241608703, ++ERASE, 140428241608704, 140428241625087, ++STORE, 140428241608704, 140428241625087, ++STORE, 140428235567104, 140428237828095, ++SNULL, 140428235567104, 140428235718655, ++STORE, 140428235718656, 140428237828095, ++STORE, 140428235567104, 140428235718655, ++SNULL, 140428237811711, 140428237828095, ++STORE, 140428235718656, 140428237811711, ++STORE, 140428237811712, 140428237828095, ++SNULL, 140428237811712, 140428237819903, ++STORE, 140428237819904, 140428237828095, ++STORE, 140428237811712, 140428237819903, ++ERASE, 140428237811712, 140428237819903, ++STORE, 140428237811712, 140428237819903, ++ERASE, 140428237819904, 140428237828095, ++STORE, 140428237819904, 140428237828095, ++STORE, 140428233445376, 140428235567103, ++SNULL, 140428233445376, 140428233461759, ++STORE, 140428233461760, 140428235567103, ++STORE, 140428233445376, 140428233461759, ++SNULL, 140428235558911, 140428235567103, ++STORE, 140428233461760, 140428235558911, ++STORE, 140428235558912, 140428235567103, ++ERASE, 140428235558912, 140428235567103, ++STORE, 140428235558912, 140428235567103, ++STORE, 140428231315456, 140428233445375, ++SNULL, 140428231315456, 140428231344127, ++STORE, 140428231344128, 140428233445375, ++STORE, 140428231315456, 140428231344127, ++SNULL, 140428233437183, 140428233445375, ++STORE, 140428231344128, 140428233437183, ++STORE, 140428233437184, 140428233445375, ++ERASE, 140428233437184, 140428233445375, ++STORE, 140428233437184, 140428233445375, ++STORE, 140428248260608, 140428248268799, ++STORE, 140428229062656, 140428231315455, ++SNULL, 140428229062656, 140428229214207, ++STORE, 140428229214208, 140428231315455, ++STORE, 140428229062656, 140428229214207, ++SNULL, 140428231307263, 140428231315455, ++STORE, 140428229214208, 140428231307263, ++STORE, 140428231307264, 140428231315455, ++ERASE, 140428231307264, 140428231315455, ++STORE, 140428231307264, 140428231315455, ++STORE, 140428226891776, 140428229062655, ++SNULL, 140428226891776, 140428226961407, ++STORE, 140428226961408, 140428229062655, ++STORE, 140428226891776, 140428226961407, ++SNULL, 140428229054463, 140428229062655, ++STORE, 140428226961408, 140428229054463, ++STORE, 140428229054464, 140428229062655, ++ERASE, 140428229054464, 140428229062655, ++STORE, 140428229054464, 140428229062655, ++STORE, 140428223680512, 140428226891775, ++SNULL, 140428223680512, 140428224757759, ++STORE, 140428224757760, 140428226891775, ++STORE, 140428223680512, 140428224757759, ++SNULL, 140428226854911, 140428226891775, ++STORE, 140428224757760, 140428226854911, ++STORE, 140428226854912, 140428226891775, ++ERASE, 140428226854912, 140428226891775, ++STORE, 140428226854912, 140428226891775, ++STORE, 140428221546496, 140428223680511, ++SNULL, 140428221546496, 140428221575167, ++STORE, 140428221575168, 140428223680511, ++STORE, 140428221546496, 140428221575167, ++SNULL, 140428223672319, 140428223680511, ++STORE, 140428221575168, 140428223672319, ++STORE, 140428223672320, 140428223680511, ++ERASE, 140428223672320, 140428223680511, ++STORE, 140428223672320, 140428223680511, ++STORE, 140428219236352, 140428221546495, ++SNULL, 140428219236352, 140428219441151, ++STORE, 140428219441152, 140428221546495, ++STORE, 140428219236352, 140428219441151, ++SNULL, 140428221538303, 140428221546495, ++STORE, 140428219441152, 140428221538303, ++STORE, 140428221538304, 140428221546495, ++ERASE, 140428221538304, 140428221546495, ++STORE, 140428221538304, 140428221546495, ++STORE, 140428216852480, 140428219236351, ++SNULL, 140428216852480, 140428217044991, ++STORE, 140428217044992, 140428219236351, ++STORE, 140428216852480, 140428217044991, ++SNULL, 140428219138047, 140428219236351, ++STORE, 140428217044992, 140428219138047, ++STORE, 140428219138048, 140428219236351, ++ERASE, 140428219138048, 140428219236351, ++STORE, 140428219138048, 140428219236351, ++STORE, 140428248252416, 140428248268799, ++STORE, 140428214284288, 140428216852479, ++SNULL, 140428214284288, 140428214751231, ++STORE, 140428214751232, 140428216852479, ++STORE, 140428214284288, 140428214751231, ++SNULL, 140428216844287, 140428216852479, ++STORE, 140428214751232, 140428216844287, ++STORE, 140428216844288, 140428216852479, ++ERASE, 140428216844288, 140428216852479, ++STORE, 140428216844288, 140428216852479, ++STORE, 140428212170752, 140428214284287, ++SNULL, 140428212170752, 140428212183039, ++STORE, 140428212183040, 140428214284287, ++STORE, 140428212170752, 140428212183039, ++SNULL, 140428214276095, 140428214284287, ++STORE, 140428212183040, 140428214276095, ++STORE, 140428214276096, 140428214284287, ++ERASE, 140428214276096, 140428214284287, ++STORE, 140428214276096, 140428214284287, ++STORE, 140428209991680, 140428212170751, ++SNULL, 140428209991680, 140428210069503, ++STORE, 140428210069504, 140428212170751, ++STORE, 140428209991680, 140428210069503, ++SNULL, 140428212162559, 140428212170751, ++STORE, 140428210069504, 140428212162559, ++STORE, 140428212162560, 140428212170751, ++ERASE, 140428212162560, 140428212170751, ++STORE, 140428212162560, 140428212170751, ++STORE, 140428207874048, 140428209991679, ++SNULL, 140428207874048, 140428207890431, ++STORE, 140428207890432, 140428209991679, ++STORE, 140428207874048, 140428207890431, ++SNULL, 140428209983487, 140428209991679, ++STORE, 140428207890432, 140428209983487, ++STORE, 140428209983488, 140428209991679, ++ERASE, 140428209983488, 140428209991679, ++STORE, 140428209983488, 140428209991679, ++STORE, 140428248244224, 140428248268799, ++STORE, 140428248231936, 140428248268799, ++SNULL, 140428241600511, 140428241608703, ++STORE, 140428241584128, 140428241600511, ++STORE, 140428241600512, 140428241608703, ++SNULL, 140428209987583, 140428209991679, ++STORE, 140428209983488, 140428209987583, ++STORE, 140428209987584, 140428209991679, ++SNULL, 140428212166655, 140428212170751, ++STORE, 140428212162560, 140428212166655, ++STORE, 140428212166656, 140428212170751, ++SNULL, 140428214280191, 140428214284287, ++STORE, 140428214276096, 140428214280191, ++STORE, 140428214280192, 140428214284287, ++SNULL, 140428243820543, 140428243824639, ++STORE, 140428243816448, 140428243820543, ++STORE, 140428243820544, 140428243824639, ++SNULL, 140428216848383, 140428216852479, ++STORE, 140428216844288, 140428216848383, ++STORE, 140428216848384, 140428216852479, ++SNULL, 140428219232255, 140428219236351, ++STORE, 140428219138048, 140428219232255, ++STORE, 140428219232256, 140428219236351, ++SNULL, 140428221542399, 140428221546495, ++STORE, 140428221538304, 140428221542399, ++STORE, 140428221542400, 140428221546495, ++SNULL, 140428223676415, 140428223680511, ++STORE, 140428223672320, 140428223676415, ++STORE, 140428223676416, 140428223680511, ++SNULL, 140428226863103, 140428226891775, ++STORE, 140428226854912, 140428226863103, ++STORE, 140428226863104, 140428226891775, ++SNULL, 140428229058559, 140428229062655, ++STORE, 140428229054464, 140428229058559, ++STORE, 140428229058560, 140428229062655, ++SNULL, 140428231311359, 140428231315455, ++STORE, 140428231307264, 140428231311359, ++STORE, 140428231311360, 140428231315455, ++SNULL, 140428233441279, 140428233445375, ++STORE, 140428233437184, 140428233441279, ++STORE, 140428233441280, 140428233445375, ++SNULL, 140428235563007, 140428235567103, ++STORE, 140428235558912, 140428235563007, ++STORE, 140428235563008, 140428235567103, ++SNULL, 140428237815807, 140428237819903, ++STORE, 140428237811712, 140428237815807, ++STORE, 140428237815808, 140428237819903, ++SNULL, 140428246056959, 140428246061055, ++STORE, 140428245491712, 140428246056959, ++STORE, 140428246056960, 140428246061055, ++SNULL, 94717834924031, 94717834928127, ++STORE, 94717834919936, 94717834924031, ++STORE, 94717834924032, 94717834928127, ++SNULL, 140428248309759, 140428248313855, ++STORE, 140428248305664, 140428248309759, ++STORE, 140428248309760, 140428248313855, ++ERASE, 140428248268800, 140428248297471, ++STORE, 94717843058688, 94717843193855, ++STORE, 94749677137920, 94749677559807, ++STORE, 94749677563904, 94749677604863, ++STORE, 94749677604864, 94749677608959, ++STORE, 94749710970880, 94749711241215, ++STORE, 140490884894720, 140490884935679, ++STORE, 140490884935680, 140490887032831, ++STORE, 140490887032832, 140490887036927, ++STORE, 140490887036928, 140490887041023, ++STORE, 140490887041024, 140490887065599, ++STORE, 140490887065600, 140490887110655, ++STORE, 140490887110656, 140490889203711, ++STORE, 140490889203712, 140490889207807, ++STORE, 140490889207808, 140490889211903, ++STORE, 140490889211904, 140490889293823, ++STORE, 140490889293824, 140490891390975, ++STORE, 140490891390976, 140490891395071, ++STORE, 140490891395072, 140490891399167, ++STORE, 140490891399168, 140490891407359, ++STORE, 140490891407360, 140490891436031, ++STORE, 140490891436032, 140490893529087, ++STORE, 140490893529088, 140490893533183, ++STORE, 140490893533184, 140490893537279, ++STORE, 140490893537280, 140490901979135, ++STORE, 140490901979136, 140490901991423, ++STORE, 140490901991424, 140490904084479, ++STORE, 140490904084480, 140490904088575, ++STORE, 140490904088576, 140490904092671, ++STORE, 140490904092672, 140490904559615, ++STORE, 140490904559616, 140490906652671, ++STORE, 140490906652672, 140490906656767, ++STORE, 140490906656768, 140490906660863, ++STORE, 140490906660864, 140490906677247, ++STORE, 140490906677248, 140490908770303, ++STORE, 140490908770304, 140490908774399, ++STORE, 140490908774400, 140490908778495, ++STORE, 140490908778496, 140490908794879, ++STORE, 140490908794880, 140490910887935, ++STORE, 140490910887936, 140490910892031, ++STORE, 140490910892032, 140490910896127, ++STORE, 140490910896128, 140490912555007, ++STORE, 140490912555008, 140490914652159, ++STORE, 140490914652160, 140490914668543, ++STORE, 140490914668544, 140490914676735, ++STORE, 140490914676736, 140490914693119, ++STORE, 140490914693120, 140490914791423, ++STORE, 140490914791424, 140490916884479, ++STORE, 140490916884480, 140490916888575, ++STORE, 140490916888576, 140490916892671, ++STORE, 140490916892672, 140490916909055, ++STORE, 140490916909056, 140490916937727, ++STORE, 140490916937728, 140490919030783, ++STORE, 140490919030784, 140490919034879, ++STORE, 140490919034880, 140490919038975, ++STORE, 140490919038976, 140490919190527, ++STORE, 140490919190528, 140490921283583, ++STORE, 140490921283584, 140490921287679, ++STORE, 140490921287680, 140490921291775, ++STORE, 140490921291776, 140490921299967, ++STORE, 140490921299968, 140490921390079, ++STORE, 140490921390080, 140490923483135, ++STORE, 140490923483136, 140490923487231, ++STORE, 140490923487232, 140490923491327, ++STORE, 140490923491328, 140490923757567, ++STORE, 140490923757568, 140490925850623, ++STORE, 140490925850624, 140490925867007, ++STORE, 140490925867008, 140490925871103, ++STORE, 140490925871104, 140490925875199, ++STORE, 140490925875200, 140490925903871, ++STORE, 140490925903872, 140490928001023, ++STORE, 140490928001024, 140490928005119, ++STORE, 140490928005120, 140490928009215, ++STORE, 140490928009216, 140490928152575, ++STORE, 140490930184192, 140490930221055, ++STORE, 140490930221056, 140490930237439, ++STORE, 140490930237440, 140490930241535, ++STORE, 140490930241536, 140490930245631, ++STORE, 140490930245632, 140490930249727, ++STORE, 140490930249728, 140490930253823, ++STORE, 140490930253824, 140490930257919, ++STORE, 140490930257920, 140490930262015, ++STORE, 140724611694592, 140724611829759, ++STORE, 140724612427776, 140724612440063, ++STORE, 140724612440064, 140724612444159, ++STORE, 94103163662336, 94103163772927, ++STORE, 94103165865984, 94103165874175, ++STORE, 94103165874176, 94103165878271, ++STORE, 94103165878272, 94103165886463, ++STORE, 94103182548992, 94103182684159, ++STORE, 140092694708224, 140092696367103, ++STORE, 140092696367104, 140092698464255, ++STORE, 140092698464256, 140092698480639, ++STORE, 140092698480640, 140092698488831, ++STORE, 140092698488832, 140092698505215, ++STORE, 140092698505216, 140092698648575, ++STORE, 140092700708864, 140092700717055, ++STORE, 140092700745728, 140092700749823, ++STORE, 140092700749824, 140092700753919, ++STORE, 140092700753920, 140092700758015, ++STORE, 140736800911360, 140736801046527, ++STORE, 140736802308096, 140736802320383, ++STORE, 140736802320384, 140736802324479, ++STORE, 93948802064384, 93948802174975, ++STORE, 93948804268032, 93948804276223, ++STORE, 93948804276224, 93948804280319, ++STORE, 93948804280320, 93948804288511, ++STORE, 93948806266880, 93948806402047, ++STORE, 140222999113728, 140223000772607, ++STORE, 140223000772608, 140223002869759, ++STORE, 140223002869760, 140223002886143, ++STORE, 140223002886144, 140223002894335, ++STORE, 140223002894336, 140223002910719, ++STORE, 140223002910720, 140223003054079, ++STORE, 140223005114368, 140223005122559, ++STORE, 140223005151232, 140223005155327, ++STORE, 140223005155328, 140223005159423, ++STORE, 140223005159424, 140223005163519, ++STORE, 140720877506560, 140720877641727, ++STORE, 140720878231552, 140720878243839, ++STORE, 140720878243840, 140720878247935, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733232087040, 140737488351231, ++SNULL, 140733232091135, 140737488351231, ++STORE, 140733232087040, 140733232091135, ++STORE, 140733231955968, 140733232091135, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140161681321984, 140161683574783, ++SNULL, 140161681465343, 140161683574783, ++STORE, 140161681321984, 140161681465343, ++STORE, 140161681465344, 140161683574783, ++ERASE, 140161681465344, 140161683574783, ++STORE, 140161683562496, 140161683570687, ++STORE, 140161683570688, 140161683574783, ++STORE, 140733232214016, 140733232218111, ++STORE, 140733232201728, 140733232214015, ++STORE, 140161683533824, 140161683562495, ++STORE, 140161683525632, 140161683533823, ++STORE, 140161678159872, 140161681321983, ++SNULL, 140161678159872, 140161679220735, ++STORE, 140161679220736, 140161681321983, ++STORE, 140161678159872, 140161679220735, ++SNULL, 140161681313791, 140161681321983, ++STORE, 140161679220736, 140161681313791, ++STORE, 140161681313792, 140161681321983, ++ERASE, 140161681313792, 140161681321983, ++STORE, 140161681313792, 140161681321983, ++STORE, 140161674362880, 140161678159871, ++SNULL, 140161674362880, 140161676021759, ++STORE, 140161676021760, 140161678159871, ++STORE, 140161674362880, 140161676021759, ++SNULL, 140161678118911, 140161678159871, ++STORE, 140161676021760, 140161678118911, ++STORE, 140161678118912, 140161678159871, ++SNULL, 140161678118912, 140161678143487, ++STORE, 140161678143488, 140161678159871, ++STORE, 140161678118912, 140161678143487, ++ERASE, 140161678118912, 140161678143487, ++STORE, 140161678118912, 140161678143487, ++ERASE, 140161678143488, 140161678159871, ++STORE, 140161678143488, 140161678159871, ++STORE, 140161683513344, 140161683533823, ++SNULL, 140161678135295, 140161678143487, ++STORE, 140161678118912, 140161678135295, ++STORE, 140161678135296, 140161678143487, ++SNULL, 140161681317887, 140161681321983, ++STORE, 140161681313792, 140161681317887, ++STORE, 140161681317888, 140161681321983, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140161683566591, 140161683570687, ++STORE, 140161683562496, 140161683566591, ++STORE, 140161683566592, 140161683570687, ++ERASE, 140161683533824, 140161683562495, ++STORE, 25477120, 25612287, ++STORE, 25477120, 25759743, ++STORE, 140161681829888, 140161683513343, ++STORE, 25477120, 25915391, ++STORE, 25477120, 26054655, ++SNULL, 25800703, 26054655, ++STORE, 25477120, 25800703, ++STORE, 25800704, 26054655, ++ERASE, 25800704, 26054655, ++STORE, 140737488347136, 140737488351231, ++STORE, 140723218452480, 140737488351231, ++SNULL, 140723218456575, 140737488351231, ++STORE, 140723218452480, 140723218456575, ++STORE, 140723218321408, 140723218456575, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 140398872264704, 140398874517503, ++SNULL, 140398872408063, 140398874517503, ++STORE, 140398872264704, 140398872408063, ++STORE, 140398872408064, 140398874517503, ++ERASE, 140398872408064, 140398874517503, ++STORE, 140398874505216, 140398874513407, ++STORE, 140398874513408, 140398874517503, ++STORE, 140723219247104, 140723219251199, ++STORE, 140723219234816, 140723219247103, ++STORE, 140398874476544, 140398874505215, ++STORE, 140398874468352, 140398874476543, ++STORE, 140398868430848, 140398872264703, ++SNULL, 140398868430848, 140398870138879, ++STORE, 140398870138880, 140398872264703, ++STORE, 140398868430848, 140398870138879, ++SNULL, 140398872231935, 140398872264703, ++STORE, 140398870138880, 140398872231935, ++STORE, 140398872231936, 140398872264703, ++ERASE, 140398872231936, 140398872264703, ++STORE, 140398872231936, 140398872264703, ++STORE, 140398866235392, 140398868430847, ++SNULL, 140398866235392, 140398866329599, ++STORE, 140398866329600, 140398868430847, ++STORE, 140398866235392, 140398866329599, ++SNULL, 140398868422655, 140398868430847, ++STORE, 140398866329600, 140398868422655, ++STORE, 140398868422656, 140398868430847, ++ERASE, 140398868422656, 140398868430847, ++STORE, 140398868422656, 140398868430847, ++STORE, 140398863716352, 140398866235391, ++SNULL, 140398863716352, 140398864130047, ++STORE, 140398864130048, 140398866235391, ++STORE, 140398863716352, 140398864130047, ++SNULL, 140398866223103, 140398866235391, ++STORE, 140398864130048, 140398866223103, ++STORE, 140398866223104, 140398866235391, ++ERASE, 140398866223104, 140398866235391, ++STORE, 140398866223104, 140398866235391, ++STORE, 140398861082624, 140398863716351, ++SNULL, 140398861082624, 140398861611007, ++STORE, 140398861611008, 140398863716351, ++STORE, 140398861082624, 140398861611007, ++SNULL, 140398863708159, 140398863716351, ++STORE, 140398861611008, 140398863708159, ++STORE, 140398863708160, 140398863716351, ++ERASE, 140398863708160, 140398863716351, ++STORE, 140398863708160, 140398863716351, ++STORE, 140398858969088, 140398861082623, ++SNULL, 140398858969088, 140398858981375, ++STORE, 140398858981376, 140398861082623, ++STORE, 140398858969088, 140398858981375, ++SNULL, 140398861074431, 140398861082623, ++STORE, 140398858981376, 140398861074431, ++STORE, 140398861074432, 140398861082623, ++ERASE, 140398861074432, 140398861082623, ++STORE, 140398861074432, 140398861082623, ++STORE, 140398856765440, 140398858969087, ++SNULL, 140398856765440, 140398856867839, ++STORE, 140398856867840, 140398858969087, ++STORE, 140398856765440, 140398856867839, ++SNULL, 140398858960895, 140398858969087, ++STORE, 140398856867840, 140398858960895, ++STORE, 140398858960896, 140398858969087, ++ERASE, 140398858960896, 140398858969087, ++STORE, 140398858960896, 140398858969087, ++STORE, 140398874460160, 140398874476543, ++STORE, 140398853603328, 140398856765439, ++SNULL, 140398853603328, 140398854664191, ++STORE, 140398854664192, 140398856765439, ++STORE, 140398853603328, 140398854664191, ++SNULL, 140398856757247, 140398856765439, ++STORE, 140398854664192, 140398856757247, ++STORE, 140398856757248, 140398856765439, ++ERASE, 140398856757248, 140398856765439, ++STORE, 140398856757248, 140398856765439, ++STORE, 140398849806336, 140398853603327, ++SNULL, 140398849806336, 140398851465215, ++STORE, 140398851465216, 140398853603327, ++STORE, 140398849806336, 140398851465215, ++SNULL, 140398853562367, 140398853603327, ++STORE, 140398851465216, 140398853562367, ++STORE, 140398853562368, 140398853603327, ++SNULL, 140398853562368, 140398853586943, ++STORE, 140398853586944, 140398853603327, ++STORE, 140398853562368, 140398853586943, ++ERASE, 140398853562368, 140398853586943, ++STORE, 140398853562368, 140398853586943, ++ERASE, 140398853586944, 140398853603327, ++STORE, 140398853586944, 140398853603327, ++STORE, 140398874447872, 140398874476543, ++SNULL, 140398853578751, 140398853586943, ++STORE, 140398853562368, 140398853578751, ++STORE, 140398853578752, 140398853586943, ++SNULL, 140398856761343, 140398856765439, ++STORE, 140398856757248, 140398856761343, ++STORE, 140398856761344, 140398856765439, ++SNULL, 140398858964991, 140398858969087, ++STORE, 140398858960896, 140398858964991, ++STORE, 140398858964992, 140398858969087, ++SNULL, 140398861078527, 140398861082623, ++STORE, 140398861074432, 140398861078527, ++STORE, 140398861078528, 140398861082623, ++SNULL, 140398863712255, 140398863716351, ++STORE, 140398863708160, 140398863712255, ++STORE, 140398863712256, 140398863716351, ++SNULL, 140398866231295, 140398866235391, ++STORE, 140398866223104, 140398866231295, ++STORE, 140398866231296, 140398866235391, ++SNULL, 140398868426751, 140398868430847, ++STORE, 140398868422656, 140398868426751, ++STORE, 140398868426752, 140398868430847, ++SNULL, 140398872236031, 140398872264703, ++STORE, 140398872231936, 140398872236031, ++STORE, 140398872236032, 140398872264703, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 140398874509311, 140398874513407, ++STORE, 140398874505216, 140398874509311, ++STORE, 140398874509312, 140398874513407, ++ERASE, 140398874476544, 140398874505215, ++STORE, 43278336, 43413503, ++STORE, 140398872764416, 140398874447871, ++STORE, 140398874501120, 140398874505215, ++STORE, 140398872629248, 140398872764415, ++STORE, 43278336, 43556863, ++STORE, 140398847709184, 140398849806335, ++STORE, 140398874492928, 140398874505215, ++STORE, 140398874484736, 140398874505215, ++STORE, 140398874447872, 140398874484735, ++STORE, 140398872612864, 140398872764415, ++STORE, 43278336, 43692031, ++STORE, 43278336, 43880447, ++STORE, 140398872604672, 140398872764415, ++STORE, 140398872596480, 140398872764415, ++STORE, 43278336, 44044287, ++STORE, 140398872580096, 140398872764415, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734403092480, 140737488351231, ++SNULL, 140734403096575, 140737488351231, ++STORE, 140734403092480, 140734403096575, ++STORE, 140734402961408, 140734403096575, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140240662380544, 140240664633343, ++SNULL, 140240662523903, 140240664633343, ++STORE, 140240662380544, 140240662523903, ++STORE, 140240662523904, 140240664633343, ++ERASE, 140240662523904, 140240664633343, ++STORE, 140240664621056, 140240664629247, ++STORE, 140240664629248, 140240664633343, ++STORE, 140734403145728, 140734403149823, ++STORE, 140734403133440, 140734403145727, ++STORE, 140240664592384, 140240664621055, ++STORE, 140240664584192, 140240664592383, ++STORE, 140240659218432, 140240662380543, ++SNULL, 140240659218432, 140240660279295, ++STORE, 140240660279296, 140240662380543, ++STORE, 140240659218432, 140240660279295, ++SNULL, 140240662372351, 140240662380543, ++STORE, 140240660279296, 140240662372351, ++STORE, 140240662372352, 140240662380543, ++ERASE, 140240662372352, 140240662380543, ++STORE, 140240662372352, 140240662380543, ++STORE, 140240655421440, 140240659218431, ++SNULL, 140240655421440, 140240657080319, ++STORE, 140240657080320, 140240659218431, ++STORE, 140240655421440, 140240657080319, ++SNULL, 140240659177471, 140240659218431, ++STORE, 140240657080320, 140240659177471, ++STORE, 140240659177472, 140240659218431, ++SNULL, 140240659177472, 140240659202047, ++STORE, 140240659202048, 140240659218431, ++STORE, 140240659177472, 140240659202047, ++ERASE, 140240659177472, 140240659202047, ++STORE, 140240659177472, 140240659202047, ++ERASE, 140240659202048, 140240659218431, ++STORE, 140240659202048, 140240659218431, ++STORE, 140240664571904, 140240664592383, ++SNULL, 140240659193855, 140240659202047, ++STORE, 140240659177472, 140240659193855, ++STORE, 140240659193856, 140240659202047, ++SNULL, 140240662376447, 140240662380543, ++STORE, 140240662372352, 140240662376447, ++STORE, 140240662376448, 140240662380543, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140240664625151, 140240664629247, ++STORE, 140240664621056, 140240664625151, ++STORE, 140240664625152, 140240664629247, ++ERASE, 140240664592384, 140240664621055, ++STORE, 30646272, 30781439, ++STORE, 30646272, 30928895, ++STORE, 140240662888448, 140240664571903, ++STORE, 94256659468288, 94256659578879, ++STORE, 94256661671936, 94256661680127, ++STORE, 94256661680128, 94256661684223, ++STORE, 94256661684224, 94256661692415, ++STORE, 94256687980544, 94256688115711, ++STORE, 139801712504832, 139801714163711, ++STORE, 139801714163712, 139801716260863, ++STORE, 139801716260864, 139801716277247, ++STORE, 139801716277248, 139801716285439, ++STORE, 139801716285440, 139801716301823, ++STORE, 139801716301824, 139801716445183, ++STORE, 139801718505472, 139801718513663, ++STORE, 139801718542336, 139801718546431, ++STORE, 139801718546432, 139801718550527, ++STORE, 139801718550528, 139801718554623, ++STORE, 140721575538688, 140721575673855, ++STORE, 140721577013248, 140721577025535, ++STORE, 140721577025536, 140721577029631, ++STORE, 140737488347136, 140737488351231, ++STORE, 140729259393024, 140737488351231, ++SNULL, 140729259397119, 140737488351231, ++STORE, 140729259393024, 140729259397119, ++STORE, 140729259261952, 140729259397119, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 139682376638464, 139682378891263, ++SNULL, 139682376781823, 139682378891263, ++STORE, 139682376638464, 139682376781823, ++STORE, 139682376781824, 139682378891263, ++ERASE, 139682376781824, 139682378891263, ++STORE, 139682378878976, 139682378887167, ++STORE, 139682378887168, 139682378891263, ++STORE, 140729260462080, 140729260466175, ++STORE, 140729260449792, 140729260462079, ++STORE, 139682378850304, 139682378878975, ++STORE, 139682378842112, 139682378850303, ++STORE, 139682373476352, 139682376638463, ++SNULL, 139682373476352, 139682374537215, ++STORE, 139682374537216, 139682376638463, ++STORE, 139682373476352, 139682374537215, ++SNULL, 139682376630271, 139682376638463, ++STORE, 139682374537216, 139682376630271, ++STORE, 139682376630272, 139682376638463, ++ERASE, 139682376630272, 139682376638463, ++STORE, 139682376630272, 139682376638463, ++STORE, 139682369679360, 139682373476351, ++SNULL, 139682369679360, 139682371338239, ++STORE, 139682371338240, 139682373476351, ++STORE, 139682369679360, 139682371338239, ++SNULL, 139682373435391, 139682373476351, ++STORE, 139682371338240, 139682373435391, ++STORE, 139682373435392, 139682373476351, ++SNULL, 139682373435392, 139682373459967, ++STORE, 139682373459968, 139682373476351, ++STORE, 139682373435392, 139682373459967, ++ERASE, 139682373435392, 139682373459967, ++STORE, 139682373435392, 139682373459967, ++ERASE, 139682373459968, 139682373476351, ++STORE, 139682373459968, 139682373476351, ++STORE, 139682378829824, 139682378850303, ++SNULL, 139682373451775, 139682373459967, ++STORE, 139682373435392, 139682373451775, ++STORE, 139682373451776, 139682373459967, ++SNULL, 139682376634367, 139682376638463, ++STORE, 139682376630272, 139682376634367, ++STORE, 139682376634368, 139682376638463, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 139682378883071, 139682378887167, ++STORE, 139682378878976, 139682378883071, ++STORE, 139682378883072, 139682378887167, ++ERASE, 139682378850304, 139682378878975, ++STORE, 10022912, 10158079, ++STORE, 10022912, 10305535, ++STORE, 139682377146368, 139682378829823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140731831926784, 140737488351231, ++SNULL, 140731831930879, 140737488351231, ++STORE, 140731831926784, 140731831930879, ++STORE, 140731831795712, 140731831930879, ++STORE, 94615305261056, 94615307485183, ++SNULL, 94615305371647, 94615307485183, ++STORE, 94615305261056, 94615305371647, ++STORE, 94615305371648, 94615307485183, ++ERASE, 94615305371648, 94615307485183, ++STORE, 94615307464704, 94615307476991, ++STORE, 94615307476992, 94615307485183, ++STORE, 140163912994816, 140163915247615, ++SNULL, 140163913138175, 140163915247615, ++STORE, 140163912994816, 140163913138175, ++STORE, 140163913138176, 140163915247615, ++ERASE, 140163913138176, 140163915247615, ++STORE, 140163915235328, 140163915243519, ++STORE, 140163915243520, 140163915247615, ++STORE, 140731832217600, 140731832221695, ++STORE, 140731832205312, 140731832217599, ++STORE, 140163915206656, 140163915235327, ++STORE, 140163915198464, 140163915206655, ++STORE, 140163909197824, 140163912994815, ++SNULL, 140163909197824, 140163910856703, ++STORE, 140163910856704, 140163912994815, ++STORE, 140163909197824, 140163910856703, ++SNULL, 140163912953855, 140163912994815, ++STORE, 140163910856704, 140163912953855, ++STORE, 140163912953856, 140163912994815, ++SNULL, 140163912953856, 140163912978431, ++STORE, 140163912978432, 140163912994815, ++STORE, 140163912953856, 140163912978431, ++ERASE, 140163912953856, 140163912978431, ++STORE, 140163912953856, 140163912978431, ++ERASE, 140163912978432, 140163912994815, ++STORE, 140163912978432, 140163912994815, ++SNULL, 140163912970239, 140163912978431, ++STORE, 140163912953856, 140163912970239, ++STORE, 140163912970240, 140163912978431, ++SNULL, 94615307472895, 94615307476991, ++STORE, 94615307464704, 94615307472895, ++STORE, 94615307472896, 94615307476991, ++SNULL, 140163915239423, 140163915243519, ++STORE, 140163915235328, 140163915239423, ++STORE, 140163915239424, 140163915243519, ++ERASE, 140163915206656, 140163915235327, ++STORE, 94615330672640, 94615330807807, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725254479872, 140737488351231, ++SNULL, 140725254488063, 140737488351231, ++STORE, 140725254479872, 140725254488063, ++STORE, 140725254348800, 140725254488063, ++STORE, 94572781277184, 94572785741823, ++SNULL, 94572783312895, 94572785741823, ++STORE, 94572781277184, 94572783312895, ++STORE, 94572783312896, 94572785741823, ++ERASE, 94572783312896, 94572785741823, ++STORE, 94572785405952, 94572785455103, ++STORE, 94572785455104, 94572785741823, ++STORE, 139636001341440, 139636003594239, ++SNULL, 139636001484799, 139636003594239, ++STORE, 139636001341440, 139636001484799, ++STORE, 139636001484800, 139636003594239, ++ERASE, 139636001484800, 139636003594239, ++STORE, 139636003581952, 139636003590143, ++STORE, 139636003590144, 139636003594239, ++STORE, 140725255557120, 140725255561215, ++STORE, 140725255544832, 140725255557119, ++STORE, 139636003553280, 139636003581951, ++STORE, 139636003545088, 139636003553279, ++STORE, 139635998773248, 139636001341439, ++SNULL, 139635998773248, 139635999240191, ++STORE, 139635999240192, 139636001341439, ++STORE, 139635998773248, 139635999240191, ++SNULL, 139636001333247, 139636001341439, ++STORE, 139635999240192, 139636001333247, ++STORE, 139636001333248, 139636001341439, ++ERASE, 139636001333248, 139636001341439, ++STORE, 139636001333248, 139636001341439, ++STORE, 139635996569600, 139635998773247, ++SNULL, 139635996569600, 139635996671999, ++STORE, 139635996672000, 139635998773247, ++STORE, 139635996569600, 139635996671999, ++SNULL, 139635998765055, 139635998773247, ++STORE, 139635996672000, 139635998765055, ++STORE, 139635998765056, 139635998773247, ++ERASE, 139635998765056, 139635998773247, ++STORE, 139635998765056, 139635998773247, ++STORE, 139635994353664, 139635996569599, ++SNULL, 139635994353664, 139635994451967, ++STORE, 139635994451968, 139635996569599, ++STORE, 139635994353664, 139635994451967, ++SNULL, 139635996545023, 139635996569599, ++STORE, 139635994451968, 139635996545023, ++STORE, 139635996545024, 139635996569599, ++SNULL, 139635996545024, 139635996553215, ++STORE, 139635996553216, 139635996569599, ++STORE, 139635996545024, 139635996553215, ++ERASE, 139635996545024, 139635996553215, ++STORE, 139635996545024, 139635996553215, ++ERASE, 139635996553216, 139635996569599, ++STORE, 139635996553216, 139635996569599, ++STORE, 139635992223744, 139635994353663, ++SNULL, 139635992223744, 139635992252415, ++STORE, 139635992252416, 139635994353663, ++STORE, 139635992223744, 139635992252415, ++SNULL, 139635994345471, 139635994353663, ++STORE, 139635992252416, 139635994345471, ++STORE, 139635994345472, 139635994353663, ++ERASE, 139635994345472, 139635994353663, ++STORE, 139635994345472, 139635994353663, ++STORE, 139635988426752, 139635992223743, ++SNULL, 139635988426752, 139635990085631, ++STORE, 139635990085632, 139635992223743, ++STORE, 139635988426752, 139635990085631, ++SNULL, 139635992182783, 139635992223743, ++STORE, 139635990085632, 139635992182783, ++STORE, 139635992182784, 139635992223743, ++SNULL, 139635992182784, 139635992207359, ++STORE, 139635992207360, 139635992223743, ++STORE, 139635992182784, 139635992207359, ++ERASE, 139635992182784, 139635992207359, ++STORE, 139635992182784, 139635992207359, ++ERASE, 139635992207360, 139635992223743, ++STORE, 139635992207360, 139635992223743, ++STORE, 139636003536896, 139636003553279, ++SNULL, 139635992199167, 139635992207359, ++STORE, 139635992182784, 139635992199167, ++STORE, 139635992199168, 139635992207359, ++SNULL, 139635996549119, 139635996553215, ++STORE, 139635996545024, 139635996549119, ++STORE, 139635996549120, 139635996553215, ++SNULL, 139635994349567, 139635994353663, ++STORE, 139635994345472, 139635994349567, ++STORE, 139635994349568, 139635994353663, ++SNULL, 139635998769151, 139635998773247, ++STORE, 139635998765056, 139635998769151, ++STORE, 139635998769152, 139635998773247, ++SNULL, 139636001337343, 139636001341439, ++STORE, 139636001333248, 139636001337343, ++STORE, 139636001337344, 139636001341439, ++SNULL, 94572785418239, 94572785455103, ++STORE, 94572785405952, 94572785418239, ++STORE, 94572785418240, 94572785455103, ++SNULL, 139636003586047, 139636003590143, ++STORE, 139636003581952, 139636003586047, ++STORE, 139636003586048, 139636003590143, ++ERASE, 139636003553280, 139636003581951, ++STORE, 94572798435328, 94572798570495, ++STORE, 139636001853440, 139636003536895, ++STORE, 139635981426688, 139635988426751, ++STORE, 139635980615680, 139635981426687, ++STORE, 94572798435328, 94572798705663, ++STORE, 94572798435328, 94572798840831, ++STORE, 94572798435328, 94572798975999, ++STORE, 94572798435328, 94572799111167, ++STORE, 94572798435328, 94572799246335, ++STORE, 94572798435328, 94572799381503, ++STORE, 94572798435328, 94572799516671, ++STORE, 94572798435328, 94572799651839, ++STORE, 94572798435328, 94572799787007, ++STORE, 94572798435328, 94572799922175, ++STORE, 94572798435328, 94572800057343, ++STORE, 94572798435328, 94572800192511, ++STORE, 94572798435328, 94572800327679, ++STORE, 94572798435328, 94572800462847, ++STORE, 94572798435328, 94572800598015, ++STORE, 94572798435328, 94572800733183, ++STORE, 94572798435328, 94572800868351, ++STORE, 94572798435328, 94572801003519, ++STORE, 94572798435328, 94572801138687, ++STORE, 94572798435328, 94572801273855, ++STORE, 94572798435328, 94572801409023, ++STORE, 94572798435328, 94572801544191, ++STORE, 94572798435328, 94572801679359, ++STORE, 94572798435328, 94572801814527, ++STORE, 94572798435328, 94572801949695, ++STORE, 94572798435328, 94572802084863, ++STORE, 94572798435328, 94572802220031, ++STORE, 94572798435328, 94572802355199, ++STORE, 94572798435328, 94572802490367, ++STORE, 94572798435328, 94572802625535, ++STORE, 94572798435328, 94572802760703, ++STORE, 94572798435328, 94572802895871, ++STORE, 94572798435328, 94572803031039, ++STORE, 94572798435328, 94572803166207, ++STORE, 94572798435328, 94572803301375, ++STORE, 94572798435328, 94572803436543, ++STORE, 94572798435328, 94572803571711, ++STORE, 94572798435328, 94572803706879, ++STORE, 94572798435328, 94572803842047, ++STORE, 94572798435328, 94572803977215, ++STORE, 94572798435328, 94572804112383, ++STORE, 94572798435328, 94572804247551, ++STORE, 94572798435328, 94572804382719, ++STORE, 94572798435328, 94572804517887, ++STORE, 94572798435328, 94572804653055, ++STORE, 94572798435328, 94572804788223, ++STORE, 94572798435328, 94572804923391, ++STORE, 94572798435328, 94572805058559, ++STORE, 94572798435328, 94572805193727, ++STORE, 94572798435328, 94572805328895, ++STORE, 94572798435328, 94572805464063, ++STORE, 94572798435328, 94572805599231, ++STORE, 94572798435328, 94572805734399, ++STORE, 94572798435328, 94572805869567, ++STORE, 94572798435328, 94572806004735, ++STORE, 94572798435328, 94572806139903, ++STORE, 94572798435328, 94572806275071, ++STORE, 94572798435328, 94572806410239, ++STORE, 94572798435328, 94572806545407, ++STORE, 94572798435328, 94572806680575, ++STORE, 94572798435328, 94572806815743, ++STORE, 94572798435328, 94572806950911, ++STORE, 94572798435328, 94572807086079, ++STORE, 94572798435328, 94572807221247, ++STORE, 94572798435328, 94572807356415, ++STORE, 94572798435328, 94572807491583, ++STORE, 94572798435328, 94572807626751, ++STORE, 94572798435328, 94572807761919, ++STORE, 94572798435328, 94572807897087, ++STORE, 94572798435328, 94572808032255, ++STORE, 94572798435328, 94572808167423, ++STORE, 94572798435328, 94572808302591, ++STORE, 94572798435328, 94572808437759, ++STORE, 94572798435328, 94572808572927, ++ERASE, 139635981426688, 139635988426751, ++STORE, 139635985088512, 139635988426751, ++STORE, 139635778273280, 139635980615679, ++STORE, 139635567632384, 139635778273279, ++STORE, 94572798435328, 94572808716287, ++STORE, 139635984564224, 139635985088511, ++STORE, 139635559239680, 139635567632383, ++SNULL, 139635559243775, 139635567632383, ++STORE, 139635559239680, 139635559243775, ++STORE, 139635559243776, 139635567632383, ++STORE, 139635550846976, 139635559239679, ++SNULL, 139635550851071, 139635559239679, ++STORE, 139635550846976, 139635550851071, ++STORE, 139635550851072, 139635559239679, ++STORE, 139635542454272, 139635550846975, ++STORE, 139635408236544, 139635542454271, ++SNULL, 139635408236544, 139635426590719, ++STORE, 139635426590720, 139635542454271, ++STORE, 139635408236544, 139635426590719, ++ERASE, 139635408236544, 139635426590719, ++STORE, 139635292372992, 139635542454271, ++SNULL, 139635359481855, 139635542454271, ++STORE, 139635292372992, 139635359481855, ++STORE, 139635359481856, 139635542454271, ++SNULL, 139635359481856, 139635426590719, ++STORE, 139635426590720, 139635542454271, ++STORE, 139635359481856, 139635426590719, ++ERASE, 139635359481856, 139635426590719, ++SNULL, 139635542458367, 139635550846975, ++STORE, 139635542454272, 139635542458367, ++STORE, 139635542458368, 139635550846975, ++STORE, 139635418198016, 139635426590719, ++SNULL, 139635493699583, 139635542454271, ++STORE, 139635426590720, 139635493699583, ++STORE, 139635493699584, 139635542454271, ++ERASE, 139635493699584, 139635542454271, ++SNULL, 139635426725887, 139635493699583, ++STORE, 139635426590720, 139635426725887, ++STORE, 139635426725888, 139635493699583, ++SNULL, 139635292508159, 139635359481855, ++STORE, 139635292372992, 139635292508159, ++STORE, 139635292508160, 139635359481855, ++SNULL, 139635418202111, 139635426590719, ++STORE, 139635418198016, 139635418202111, ++STORE, 139635418202112, 139635426590719, ++STORE, 139635225264128, 139635292372991, ++STORE, 139635534061568, 139635542454271, ++SNULL, 139635534065663, 139635542454271, ++STORE, 139635534061568, 139635534065663, ++STORE, 139635534065664, 139635542454271, ++STORE, 139635525668864, 139635534061567, ++SNULL, 139635525672959, 139635534061567, ++STORE, 139635525668864, 139635525672959, ++STORE, 139635525672960, 139635534061567, ++SNULL, 139635225399295, 139635292372991, ++STORE, 139635225264128, 139635225399295, ++STORE, 139635225399296, 139635292372991, ++STORE, 139635091046400, 139635225264127, ++SNULL, 139635158155263, 139635225264127, ++STORE, 139635091046400, 139635158155263, ++STORE, 139635158155264, 139635225264127, ++ERASE, 139635158155264, 139635225264127, ++STORE, 139634956828672, 139635158155263, ++STORE, 139635517276160, 139635525668863, ++SNULL, 139635517280255, 139635525668863, ++STORE, 139635517276160, 139635517280255, ++STORE, 139635517280256, 139635525668863, ++SNULL, 139634956828672, 139635091046399, ++STORE, 139635091046400, 139635158155263, ++STORE, 139634956828672, 139635091046399, ++SNULL, 139635091181567, 139635158155263, ++STORE, 139635091046400, 139635091181567, ++STORE, 139635091181568, 139635158155263, ++SNULL, 139635023937535, 139635091046399, ++STORE, 139634956828672, 139635023937535, ++STORE, 139635023937536, 139635091046399, ++ERASE, 139635023937536, 139635091046399, ++STORE, 139634956828672, 139635091046399, ++SNULL, 139634956828672, 139635023937535, ++STORE, 139635023937536, 139635091046399, ++STORE, 139634956828672, 139635023937535, ++SNULL, 139635024072703, 139635091046399, ++STORE, 139635023937536, 139635024072703, ++STORE, 139635024072704, 139635091046399, ++STORE, 139635508883456, 139635517276159, ++SNULL, 139635508887551, 139635517276159, ++STORE, 139635508883456, 139635508887551, ++STORE, 139635508887552, 139635517276159, ++STORE, 139634822610944, 139635023937535, ++SNULL, 139634822610944, 139634956828671, ++STORE, 139634956828672, 139635023937535, ++STORE, 139634822610944, 139634956828671, ++SNULL, 139634956963839, 139635023937535, ++STORE, 139634956828672, 139634956963839, ++STORE, 139634956963840, 139635023937535, ++STORE, 139635500490752, 139635508883455, ++SNULL, 139634889719807, 139634956828671, ++STORE, 139634822610944, 139634889719807, ++STORE, 139634889719808, 139634956828671, ++ERASE, 139634889719808, 139634956828671, ++SNULL, 139635500494847, 139635508883455, ++STORE, 139635500490752, 139635500494847, ++STORE, 139635500494848, 139635508883455, ++SNULL, 139634822746111, 139634889719807, ++STORE, 139634822610944, 139634822746111, ++STORE, 139634822746112, 139634889719807, ++STORE, 139635409805312, 139635418198015, ++STORE, 139634822746112, 139634956828671, ++SNULL, 139634822746112, 139634889719807, ++STORE, 139634889719808, 139634956828671, ++STORE, 139634822746112, 139634889719807, ++SNULL, 139634889854975, 139634956828671, ++STORE, 139634889719808, 139634889854975, ++STORE, 139634889854976, 139634956828671, ++SNULL, 139635409809407, 139635418198015, ++STORE, 139635409805312, 139635409809407, ++STORE, 139635409809408, 139635418198015, ++STORE, 139635401412608, 139635409805311, ++STORE, 139634688393216, 139634822610943, ++SNULL, 139634755502079, 139634822610943, ++STORE, 139634688393216, 139634755502079, ++STORE, 139634755502080, 139634822610943, ++ERASE, 139634755502080, 139634822610943, ++SNULL, 139635401416703, 139635409805311, ++STORE, 139635401412608, 139635401416703, ++STORE, 139635401416704, 139635409805311, ++STORE, 139634554175488, 139634755502079, ++SNULL, 139634554175488, 139634688393215, ++STORE, 139634688393216, 139634755502079, ++STORE, 139634554175488, 139634688393215, ++SNULL, 139634688528383, 139634755502079, ++STORE, 139634688393216, 139634688528383, ++STORE, 139634688528384, 139634755502079, ++STORE, 139635393019904, 139635401412607, ++SNULL, 139634621284351, 139634688393215, ++STORE, 139634554175488, 139634621284351, ++STORE, 139634621284352, 139634688393215, ++ERASE, 139634621284352, 139634688393215, ++SNULL, 139634554310655, 139634621284351, ++STORE, 139634554175488, 139634554310655, ++STORE, 139634554310656, 139634621284351, ++STORE, 139634554310656, 139634688393215, ++SNULL, 139635393023999, 139635401412607, ++STORE, 139635393019904, 139635393023999, ++STORE, 139635393024000, 139635401412607, ++SNULL, 139634554310656, 139634621284351, ++STORE, 139634621284352, 139634688393215, ++STORE, 139634554310656, 139634621284351, ++SNULL, 139634621419519, 139634688393215, ++STORE, 139634621284352, 139634621419519, ++STORE, 139634621419520, 139634688393215, ++STORE, 139635384627200, 139635393019903, ++SNULL, 139635384631295, 139635393019903, ++STORE, 139635384627200, 139635384631295, ++STORE, 139635384631296, 139635393019903, ++STORE, 139635376234496, 139635384627199, ++SNULL, 139635376238591, 139635384627199, ++STORE, 139635376234496, 139635376238591, ++STORE, 139635376238592, 139635384627199, ++STORE, 139635367841792, 139635376234495, ++SNULL, 139635367845887, 139635376234495, ++STORE, 139635367841792, 139635367845887, ++STORE, 139635367845888, 139635376234495, ++STORE, 139634419957760, 139634554175487, ++SNULL, 139634487066623, 139634554175487, ++STORE, 139634419957760, 139634487066623, ++STORE, 139634487066624, 139634554175487, ++ERASE, 139634487066624, 139634554175487, ++STORE, 139635216871424, 139635225264127, ++SNULL, 139635216875519, 139635225264127, ++STORE, 139635216871424, 139635216875519, ++STORE, 139635216875520, 139635225264127, ++SNULL, 139634420092927, 139634487066623, ++STORE, 139634419957760, 139634420092927, ++STORE, 139634420092928, 139634487066623, ++STORE, 139635208478720, 139635216871423, ++SNULL, 139635208482815, 139635216871423, ++STORE, 139635208478720, 139635208482815, ++STORE, 139635208482816, 139635216871423, ++STORE, 139635200086016, 139635208478719, ++SNULL, 139635200090111, 139635208478719, ++STORE, 139635200086016, 139635200090111, ++STORE, 139635200090112, 139635208478719, ++STORE, 139635191693312, 139635200086015, ++SNULL, 139635191697407, 139635200086015, ++STORE, 139635191693312, 139635191697407, ++STORE, 139635191697408, 139635200086015, ++STORE, 139635183300608, 139635191693311, ++SNULL, 139635183304703, 139635191693311, ++STORE, 139635183300608, 139635183304703, ++STORE, 139635183304704, 139635191693311, ++STORE, 139634420092928, 139634554175487, ++SNULL, 139634420092928, 139634487066623, ++STORE, 139634487066624, 139634554175487, ++STORE, 139634420092928, 139634487066623, ++SNULL, 139634487201791, 139634554175487, ++STORE, 139634487066624, 139634487201791, ++STORE, 139634487201792, 139634554175487, ++ERASE, 139635559239680, 139635559243775, ++ERASE, 139635559243776, 139635567632383, ++ERASE, 139635550846976, 139635550851071, ++ERASE, 139635550851072, 139635559239679, ++ERASE, 139635542454272, 139635542458367, ++ERASE, 139635542458368, 139635550846975, ++ERASE, 139635418198016, 139635418202111, ++ERASE, 139635418202112, 139635426590719, ++ERASE, 139635534061568, 139635534065663, ++ERASE, 139635534065664, 139635542454271, ++ERASE, 139635525668864, 139635525672959, ++ERASE, 139635525672960, 139635534061567, ++ERASE, 139635517276160, 139635517280255, ++ERASE, 139635517280256, 139635525668863, ++ERASE, 139635508883456, 139635508887551, ++ERASE, 139635508887552, 139635517276159, ++ERASE, 139635500490752, 139635500494847, ++ERASE, 139635500494848, 139635508883455, ++ERASE, 139635409805312, 139635409809407, ++ERASE, 139635409809408, 139635418198015, ++ERASE, 139635401412608, 139635401416703, ++ERASE, 139635401416704, 139635409805311, ++ERASE, 139635393019904, 139635393023999, ++ERASE, 139635393024000, 139635401412607, ++ERASE, 139635384627200, 139635384631295, ++ERASE, 139635384631296, 139635393019903, ++ }; ++ unsigned long set25[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722547441664, 140737488351231, ++SNULL, 140722547449855, 140737488351231, ++STORE, 140722547441664, 140722547449855, ++STORE, 140722547310592, 140722547449855, ++STORE, 94827521732608, 94827523956735, ++SNULL, 94827521843199, 94827523956735, ++STORE, 94827521732608, 94827521843199, ++STORE, 94827521843200, 94827523956735, ++ERASE, 94827521843200, 94827523956735, ++STORE, 94827523936256, 94827523948543, ++STORE, 94827523948544, 94827523956735, ++STORE, 139816136847360, 139816139100159, ++SNULL, 139816136990719, 139816139100159, ++STORE, 139816136847360, 139816136990719, ++STORE, 139816136990720, 139816139100159, ++ERASE, 139816136990720, 139816139100159, ++STORE, 139816139087872, 139816139096063, ++STORE, 139816139096064, 139816139100159, ++STORE, 140722548142080, 140722548146175, ++STORE, 140722548129792, 140722548142079, ++STORE, 139816139059200, 139816139087871, ++STORE, 139816139051008, 139816139059199, ++STORE, 139816133050368, 139816136847359, ++SNULL, 139816133050368, 139816134709247, ++STORE, 139816134709248, 139816136847359, ++STORE, 139816133050368, 139816134709247, ++SNULL, 139816136806399, 139816136847359, ++STORE, 139816134709248, 139816136806399, ++STORE, 139816136806400, 139816136847359, ++SNULL, 139816136806400, 139816136830975, ++STORE, 139816136830976, 139816136847359, ++STORE, 139816136806400, 139816136830975, ++ERASE, 139816136806400, 139816136830975, ++STORE, 139816136806400, 139816136830975, ++ERASE, 139816136830976, 139816136847359, ++STORE, 139816136830976, 139816136847359, ++SNULL, 139816136822783, 139816136830975, ++STORE, 139816136806400, 139816136822783, ++STORE, 139816136822784, 139816136830975, ++SNULL, 94827523944447, 94827523948543, ++STORE, 94827523936256, 94827523944447, ++STORE, 94827523944448, 94827523948543, ++SNULL, 139816139091967, 139816139096063, ++STORE, 139816139087872, 139816139091967, ++STORE, 139816139091968, 139816139096063, ++ERASE, 139816139059200, 139816139087871, ++STORE, 94827534970880, 94827535106047, ++STORE, 94114394132480, 94114394345471, ++STORE, 94114396442624, 94114396446719, ++STORE, 94114396446720, 94114396454911, ++STORE, 94114396454912, 94114396467199, ++STORE, 94114421575680, 94114427715583, ++STORE, 139934313955328, 139934315614207, ++STORE, 139934315614208, 139934317711359, ++STORE, 139934317711360, 139934317727743, ++STORE, 139934317727744, 139934317735935, ++STORE, 139934317735936, 139934317752319, ++STORE, 139934317752320, 139934317764607, ++STORE, 139934317764608, 139934319857663, ++STORE, 139934319857664, 139934319861759, ++STORE, 139934319861760, 139934319865855, ++STORE, 139934319865856, 139934320009215, ++STORE, 139934320377856, 139934322061311, ++STORE, 139934322061312, 139934322077695, ++STORE, 139934322106368, 139934322110463, ++STORE, 139934322110464, 139934322114559, ++STORE, 139934322114560, 139934322118655, ++STORE, 140731200376832, 140731200516095, ++STORE, 140731200929792, 140731200942079, ++STORE, 140731200942080, 140731200946175, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140734133174272, 140737488351231, ++SNULL, 140734133182463, 140737488351231, ++STORE, 140734133174272, 140734133182463, ++STORE, 140734133043200, 140734133182463, ++STORE, 94412675600384, 94412677824511, ++SNULL, 94412675710975, 94412677824511, ++STORE, 94412675600384, 94412675710975, ++STORE, 94412675710976, 94412677824511, ++ERASE, 94412675710976, 94412677824511, ++STORE, 94412677804032, 94412677816319, ++STORE, 94412677816320, 94412677824511, ++STORE, 140320087945216, 140320090198015, ++SNULL, 140320088088575, 140320090198015, ++STORE, 140320087945216, 140320088088575, ++STORE, 140320088088576, 140320090198015, ++ERASE, 140320088088576, 140320090198015, ++STORE, 140320090185728, 140320090193919, ++STORE, 140320090193920, 140320090198015, ++STORE, 140734134591488, 140734134595583, ++STORE, 140734134579200, 140734134591487, ++STORE, 140320090157056, 140320090185727, ++STORE, 140320090148864, 140320090157055, ++STORE, 140320084148224, 140320087945215, ++SNULL, 140320084148224, 140320085807103, ++STORE, 140320085807104, 140320087945215, ++STORE, 140320084148224, 140320085807103, ++SNULL, 140320087904255, 140320087945215, ++STORE, 140320085807104, 140320087904255, ++STORE, 140320087904256, 140320087945215, ++SNULL, 140320087904256, 140320087928831, ++STORE, 140320087928832, 140320087945215, ++STORE, 140320087904256, 140320087928831, ++ERASE, 140320087904256, 140320087928831, ++STORE, 140320087904256, 140320087928831, ++ERASE, 140320087928832, 140320087945215, ++STORE, 140320087928832, 140320087945215, ++SNULL, 140320087920639, 140320087928831, ++STORE, 140320087904256, 140320087920639, ++STORE, 140320087920640, 140320087928831, ++SNULL, 94412677812223, 94412677816319, ++STORE, 94412677804032, 94412677812223, ++STORE, 94412677812224, 94412677816319, ++SNULL, 140320090189823, 140320090193919, ++STORE, 140320090185728, 140320090189823, ++STORE, 140320090189824, 140320090193919, ++ERASE, 140320090157056, 140320090185727, ++STORE, 94412684546048, 94412684681215, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723005485056, 140737488351231, ++SNULL, 140723005493247, 140737488351231, ++STORE, 140723005485056, 140723005493247, ++STORE, 140723005353984, 140723005493247, ++STORE, 94387431936000, 94387434160127, ++SNULL, 94387432046591, 94387434160127, ++STORE, 94387431936000, 94387432046591, ++STORE, 94387432046592, 94387434160127, ++ERASE, 94387432046592, 94387434160127, ++STORE, 94387434139648, 94387434151935, ++STORE, 94387434151936, 94387434160127, ++STORE, 140151675392000, 140151677644799, ++SNULL, 140151675535359, 140151677644799, ++STORE, 140151675392000, 140151675535359, ++STORE, 140151675535360, 140151677644799, ++ERASE, 140151675535360, 140151677644799, ++STORE, 140151677632512, 140151677640703, ++STORE, 140151677640704, 140151677644799, ++STORE, 140723005784064, 140723005788159, ++STORE, 140723005771776, 140723005784063, ++STORE, 140151677603840, 140151677632511, ++STORE, 140151677595648, 140151677603839, ++STORE, 140151671595008, 140151675391999, ++SNULL, 140151671595008, 140151673253887, ++STORE, 140151673253888, 140151675391999, ++STORE, 140151671595008, 140151673253887, ++SNULL, 140151675351039, 140151675391999, ++STORE, 140151673253888, 140151675351039, ++STORE, 140151675351040, 140151675391999, ++SNULL, 140151675351040, 140151675375615, ++STORE, 140151675375616, 140151675391999, ++STORE, 140151675351040, 140151675375615, ++ERASE, 140151675351040, 140151675375615, ++STORE, 140151675351040, 140151675375615, ++ERASE, 140151675375616, 140151675391999, ++STORE, 140151675375616, 140151675391999, ++SNULL, 140151675367423, 140151675375615, ++STORE, 140151675351040, 140151675367423, ++STORE, 140151675367424, 140151675375615, ++SNULL, 94387434147839, 94387434151935, ++STORE, 94387434139648, 94387434147839, ++STORE, 94387434147840, 94387434151935, ++SNULL, 140151677636607, 140151677640703, ++STORE, 140151677632512, 140151677636607, ++STORE, 140151677636608, 140151677640703, ++ERASE, 140151677603840, 140151677632511, ++STORE, 94387458818048, 94387458953215, ++STORE, 94909010997248, 94909011210239, ++STORE, 94909013307392, 94909013311487, ++STORE, 94909013311488, 94909013319679, ++STORE, 94909013319680, 94909013331967, ++STORE, 94909014827008, 94909023371263, ++STORE, 140712411975680, 140712413634559, ++STORE, 140712413634560, 140712415731711, ++STORE, 140712415731712, 140712415748095, ++STORE, 140712415748096, 140712415756287, ++STORE, 140712415756288, 140712415772671, ++STORE, 140712415772672, 140712415784959, ++STORE, 140712415784960, 140712417878015, ++STORE, 140712417878016, 140712417882111, ++STORE, 140712417882112, 140712417886207, ++STORE, 140712417886208, 140712418029567, ++STORE, 140712418398208, 140712420081663, ++STORE, 140712420081664, 140712420098047, ++STORE, 140712420126720, 140712420130815, ++STORE, 140712420130816, 140712420134911, ++STORE, 140712420134912, 140712420139007, ++STORE, 140729293111296, 140729293250559, ++STORE, 140729293307904, 140729293320191, ++STORE, 140729293320192, 140729293324287, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720541691904, 140737488351231, ++SNULL, 140720541700095, 140737488351231, ++STORE, 140720541691904, 140720541700095, ++STORE, 140720541560832, 140720541700095, ++STORE, 94203603419136, 94203605643263, ++SNULL, 94203603529727, 94203605643263, ++STORE, 94203603419136, 94203603529727, ++STORE, 94203603529728, 94203605643263, ++ERASE, 94203603529728, 94203605643263, ++STORE, 94203605622784, 94203605635071, ++STORE, 94203605635072, 94203605643263, ++STORE, 139847623081984, 139847625334783, ++SNULL, 139847623225343, 139847625334783, ++STORE, 139847623081984, 139847623225343, ++STORE, 139847623225344, 139847625334783, ++ERASE, 139847623225344, 139847625334783, ++STORE, 139847625322496, 139847625330687, ++STORE, 139847625330688, 139847625334783, ++STORE, 140720542547968, 140720542552063, ++STORE, 140720542535680, 140720542547967, ++STORE, 139847625293824, 139847625322495, ++STORE, 139847625285632, 139847625293823, ++STORE, 139847619284992, 139847623081983, ++SNULL, 139847619284992, 139847620943871, ++STORE, 139847620943872, 139847623081983, ++STORE, 139847619284992, 139847620943871, ++SNULL, 139847623041023, 139847623081983, ++STORE, 139847620943872, 139847623041023, ++STORE, 139847623041024, 139847623081983, ++SNULL, 139847623041024, 139847623065599, ++STORE, 139847623065600, 139847623081983, ++STORE, 139847623041024, 139847623065599, ++ERASE, 139847623041024, 139847623065599, ++STORE, 139847623041024, 139847623065599, ++ERASE, 139847623065600, 139847623081983, ++STORE, 139847623065600, 139847623081983, ++SNULL, 139847623057407, 139847623065599, ++STORE, 139847623041024, 139847623057407, ++STORE, 139847623057408, 139847623065599, ++SNULL, 94203605630975, 94203605635071, ++STORE, 94203605622784, 94203605630975, ++STORE, 94203605630976, 94203605635071, ++SNULL, 139847625326591, 139847625330687, ++STORE, 139847625322496, 139847625326591, ++STORE, 139847625326592, 139847625330687, ++ERASE, 139847625293824, 139847625322495, ++STORE, 94203634880512, 94203635015679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721428738048, 140737488351231, ++SNULL, 140721428746239, 140737488351231, ++STORE, 140721428738048, 140721428746239, ++STORE, 140721428606976, 140721428746239, ++STORE, 93968808378368, 93968810602495, ++SNULL, 93968808488959, 93968810602495, ++STORE, 93968808378368, 93968808488959, ++STORE, 93968808488960, 93968810602495, ++ERASE, 93968808488960, 93968810602495, ++STORE, 93968810582016, 93968810594303, ++STORE, 93968810594304, 93968810602495, ++STORE, 140397757026304, 140397759279103, ++SNULL, 140397757169663, 140397759279103, ++STORE, 140397757026304, 140397757169663, ++STORE, 140397757169664, 140397759279103, ++ERASE, 140397757169664, 140397759279103, ++STORE, 140397759266816, 140397759275007, ++STORE, 140397759275008, 140397759279103, ++STORE, 140721430368256, 140721430372351, ++STORE, 140721430355968, 140721430368255, ++STORE, 140397759238144, 140397759266815, ++STORE, 140397759229952, 140397759238143, ++STORE, 140397753229312, 140397757026303, ++SNULL, 140397753229312, 140397754888191, ++STORE, 140397754888192, 140397757026303, ++STORE, 140397753229312, 140397754888191, ++SNULL, 140397756985343, 140397757026303, ++STORE, 140397754888192, 140397756985343, ++STORE, 140397756985344, 140397757026303, ++SNULL, 140397756985344, 140397757009919, ++STORE, 140397757009920, 140397757026303, ++STORE, 140397756985344, 140397757009919, ++ERASE, 140397756985344, 140397757009919, ++STORE, 140397756985344, 140397757009919, ++ERASE, 140397757009920, 140397757026303, ++STORE, 140397757009920, 140397757026303, ++SNULL, 140397757001727, 140397757009919, ++STORE, 140397756985344, 140397757001727, ++STORE, 140397757001728, 140397757009919, ++SNULL, 93968810590207, 93968810594303, ++STORE, 93968810582016, 93968810590207, ++STORE, 93968810590208, 93968810594303, ++SNULL, 140397759270911, 140397759275007, ++STORE, 140397759266816, 140397759270911, ++STORE, 140397759270912, 140397759275007, ++ERASE, 140397759238144, 140397759266815, ++STORE, 93968837025792, 93968837160959, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721751044096, 140737488351231, ++SNULL, 140721751052287, 140737488351231, ++STORE, 140721751044096, 140721751052287, ++STORE, 140721750913024, 140721751052287, ++STORE, 94426051657728, 94426053881855, ++SNULL, 94426051768319, 94426053881855, ++STORE, 94426051657728, 94426051768319, ++STORE, 94426051768320, 94426053881855, ++ERASE, 94426051768320, 94426053881855, ++STORE, 94426053861376, 94426053873663, ++STORE, 94426053873664, 94426053881855, ++STORE, 140228456181760, 140228458434559, ++SNULL, 140228456325119, 140228458434559, ++STORE, 140228456181760, 140228456325119, ++STORE, 140228456325120, 140228458434559, ++ERASE, 140228456325120, 140228458434559, ++STORE, 140228458422272, 140228458430463, ++STORE, 140228458430464, 140228458434559, ++STORE, 140721751117824, 140721751121919, ++STORE, 140721751105536, 140721751117823, ++STORE, 140228458393600, 140228458422271, ++STORE, 140228458385408, 140228458393599, ++STORE, 140228452384768, 140228456181759, ++SNULL, 140228452384768, 140228454043647, ++STORE, 140228454043648, 140228456181759, ++STORE, 140228452384768, 140228454043647, ++SNULL, 140228456140799, 140228456181759, ++STORE, 140228454043648, 140228456140799, ++STORE, 140228456140800, 140228456181759, ++SNULL, 140228456140800, 140228456165375, ++STORE, 140228456165376, 140228456181759, ++STORE, 140228456140800, 140228456165375, ++ERASE, 140228456140800, 140228456165375, ++STORE, 140228456140800, 140228456165375, ++ERASE, 140228456165376, 140228456181759, ++STORE, 140228456165376, 140228456181759, ++SNULL, 140228456157183, 140228456165375, ++STORE, 140228456140800, 140228456157183, ++STORE, 140228456157184, 140228456165375, ++SNULL, 94426053869567, 94426053873663, ++STORE, 94426053861376, 94426053869567, ++STORE, 94426053869568, 94426053873663, ++SNULL, 140228458426367, 140228458430463, ++STORE, 140228458422272, 140228458426367, ++STORE, 140228458426368, 140228458430463, ++ERASE, 140228458393600, 140228458422271, ++STORE, 94426073681920, 94426073817087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732727623680, 140737488351231, ++SNULL, 140732727631871, 140737488351231, ++STORE, 140732727623680, 140732727631871, ++STORE, 140732727492608, 140732727631871, ++STORE, 94537485996032, 94537488220159, ++SNULL, 94537486106623, 94537488220159, ++STORE, 94537485996032, 94537486106623, ++STORE, 94537486106624, 94537488220159, ++ERASE, 94537486106624, 94537488220159, ++STORE, 94537488199680, 94537488211967, ++STORE, 94537488211968, 94537488220159, ++STORE, 140446578036736, 140446580289535, ++SNULL, 140446578180095, 140446580289535, ++STORE, 140446578036736, 140446578180095, ++STORE, 140446578180096, 140446580289535, ++ERASE, 140446578180096, 140446580289535, ++STORE, 140446580277248, 140446580285439, ++STORE, 140446580285440, 140446580289535, ++STORE, 140732727758848, 140732727762943, ++STORE, 140732727746560, 140732727758847, ++STORE, 140446580248576, 140446580277247, ++STORE, 140446580240384, 140446580248575, ++STORE, 140446574239744, 140446578036735, ++SNULL, 140446574239744, 140446575898623, ++STORE, 140446575898624, 140446578036735, ++STORE, 140446574239744, 140446575898623, ++SNULL, 140446577995775, 140446578036735, ++STORE, 140446575898624, 140446577995775, ++STORE, 140446577995776, 140446578036735, ++SNULL, 140446577995776, 140446578020351, ++STORE, 140446578020352, 140446578036735, ++STORE, 140446577995776, 140446578020351, ++ERASE, 140446577995776, 140446578020351, ++STORE, 140446577995776, 140446578020351, ++ERASE, 140446578020352, 140446578036735, ++STORE, 140446578020352, 140446578036735, ++SNULL, 140446578012159, 140446578020351, ++STORE, 140446577995776, 140446578012159, ++STORE, 140446578012160, 140446578020351, ++SNULL, 94537488207871, 94537488211967, ++STORE, 94537488199680, 94537488207871, ++STORE, 94537488207872, 94537488211967, ++SNULL, 140446580281343, 140446580285439, ++STORE, 140446580277248, 140446580281343, ++STORE, 140446580281344, 140446580285439, ++ERASE, 140446580248576, 140446580277247, ++STORE, 94537489014784, 94537489149951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728766808064, 140737488351231, ++SNULL, 140728766816255, 140737488351231, ++STORE, 140728766808064, 140728766816255, ++STORE, 140728766676992, 140728766816255, ++STORE, 94418513866752, 94418516090879, ++SNULL, 94418513977343, 94418516090879, ++STORE, 94418513866752, 94418513977343, ++STORE, 94418513977344, 94418516090879, ++ERASE, 94418513977344, 94418516090879, ++STORE, 94418516070400, 94418516082687, ++STORE, 94418516082688, 94418516090879, ++STORE, 140556479520768, 140556481773567, ++SNULL, 140556479664127, 140556481773567, ++STORE, 140556479520768, 140556479664127, ++STORE, 140556479664128, 140556481773567, ++ERASE, 140556479664128, 140556481773567, ++STORE, 140556481761280, 140556481769471, ++STORE, 140556481769472, 140556481773567, ++STORE, 140728767148032, 140728767152127, ++STORE, 140728767135744, 140728767148031, ++STORE, 140556481732608, 140556481761279, ++STORE, 140556481724416, 140556481732607, ++STORE, 140556475723776, 140556479520767, ++SNULL, 140556475723776, 140556477382655, ++STORE, 140556477382656, 140556479520767, ++STORE, 140556475723776, 140556477382655, ++SNULL, 140556479479807, 140556479520767, ++STORE, 140556477382656, 140556479479807, ++STORE, 140556479479808, 140556479520767, ++SNULL, 140556479479808, 140556479504383, ++STORE, 140556479504384, 140556479520767, ++STORE, 140556479479808, 140556479504383, ++ERASE, 140556479479808, 140556479504383, ++STORE, 140556479479808, 140556479504383, ++ERASE, 140556479504384, 140556479520767, ++STORE, 140556479504384, 140556479520767, ++SNULL, 140556479496191, 140556479504383, ++STORE, 140556479479808, 140556479496191, ++STORE, 140556479496192, 140556479504383, ++SNULL, 94418516078591, 94418516082687, ++STORE, 94418516070400, 94418516078591, ++STORE, 94418516078592, 94418516082687, ++SNULL, 140556481765375, 140556481769471, ++STORE, 140556481761280, 140556481765375, ++STORE, 140556481765376, 140556481769471, ++ERASE, 140556481732608, 140556481761279, ++STORE, 94418541113344, 94418541248511, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723945873408, 140737488351231, ++SNULL, 140723945881599, 140737488351231, ++STORE, 140723945873408, 140723945881599, ++STORE, 140723945742336, 140723945881599, ++STORE, 94543169773568, 94543171997695, ++SNULL, 94543169884159, 94543171997695, ++STORE, 94543169773568, 94543169884159, ++STORE, 94543169884160, 94543171997695, ++ERASE, 94543169884160, 94543171997695, ++STORE, 94543171977216, 94543171989503, ++STORE, 94543171989504, 94543171997695, ++STORE, 139890420883456, 139890423136255, ++SNULL, 139890421026815, 139890423136255, ++STORE, 139890420883456, 139890421026815, ++STORE, 139890421026816, 139890423136255, ++ERASE, 139890421026816, 139890423136255, ++STORE, 139890423123968, 139890423132159, ++STORE, 139890423132160, 139890423136255, ++STORE, 140723946102784, 140723946106879, ++STORE, 140723946090496, 140723946102783, ++STORE, 139890423095296, 139890423123967, ++STORE, 139890423087104, 139890423095295, ++STORE, 139890417086464, 139890420883455, ++SNULL, 139890417086464, 139890418745343, ++STORE, 139890418745344, 139890420883455, ++STORE, 139890417086464, 139890418745343, ++SNULL, 139890420842495, 139890420883455, ++STORE, 139890418745344, 139890420842495, ++STORE, 139890420842496, 139890420883455, ++SNULL, 139890420842496, 139890420867071, ++STORE, 139890420867072, 139890420883455, ++STORE, 139890420842496, 139890420867071, ++ERASE, 139890420842496, 139890420867071, ++STORE, 139890420842496, 139890420867071, ++ERASE, 139890420867072, 139890420883455, ++STORE, 139890420867072, 139890420883455, ++SNULL, 139890420858879, 139890420867071, ++STORE, 139890420842496, 139890420858879, ++STORE, 139890420858880, 139890420867071, ++SNULL, 94543171985407, 94543171989503, ++STORE, 94543171977216, 94543171985407, ++STORE, 94543171985408, 94543171989503, ++SNULL, 139890423128063, 139890423132159, ++STORE, 139890423123968, 139890423128063, ++STORE, 139890423128064, 139890423132159, ++ERASE, 139890423095296, 139890423123967, ++STORE, 94543197097984, 94543197233151, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736205979648, 140737488351231, ++SNULL, 140736205987839, 140737488351231, ++STORE, 140736205979648, 140736205987839, ++STORE, 140736205848576, 140736205987839, ++STORE, 94913209913344, 94913212137471, ++SNULL, 94913210023935, 94913212137471, ++STORE, 94913209913344, 94913210023935, ++STORE, 94913210023936, 94913212137471, ++ERASE, 94913210023936, 94913212137471, ++STORE, 94913212116992, 94913212129279, ++STORE, 94913212129280, 94913212137471, ++STORE, 140006323052544, 140006325305343, ++SNULL, 140006323195903, 140006325305343, ++STORE, 140006323052544, 140006323195903, ++STORE, 140006323195904, 140006325305343, ++ERASE, 140006323195904, 140006325305343, ++STORE, 140006325293056, 140006325301247, ++STORE, 140006325301248, 140006325305343, ++STORE, 140736206716928, 140736206721023, ++STORE, 140736206704640, 140736206716927, ++STORE, 140006325264384, 140006325293055, ++STORE, 140006325256192, 140006325264383, ++STORE, 140006319255552, 140006323052543, ++SNULL, 140006319255552, 140006320914431, ++STORE, 140006320914432, 140006323052543, ++STORE, 140006319255552, 140006320914431, ++SNULL, 140006323011583, 140006323052543, ++STORE, 140006320914432, 140006323011583, ++STORE, 140006323011584, 140006323052543, ++SNULL, 140006323011584, 140006323036159, ++STORE, 140006323036160, 140006323052543, ++STORE, 140006323011584, 140006323036159, ++ERASE, 140006323011584, 140006323036159, ++STORE, 140006323011584, 140006323036159, ++ERASE, 140006323036160, 140006323052543, ++STORE, 140006323036160, 140006323052543, ++SNULL, 140006323027967, 140006323036159, ++STORE, 140006323011584, 140006323027967, ++STORE, 140006323027968, 140006323036159, ++SNULL, 94913212125183, 94913212129279, ++STORE, 94913212116992, 94913212125183, ++STORE, 94913212125184, 94913212129279, ++SNULL, 140006325297151, 140006325301247, ++STORE, 140006325293056, 140006325297151, ++STORE, 140006325297152, 140006325301247, ++ERASE, 140006325264384, 140006325293055, ++STORE, 94913239932928, 94913240068095, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726926897152, 140737488351231, ++SNULL, 140726926905343, 140737488351231, ++STORE, 140726926897152, 140726926905343, ++STORE, 140726926766080, 140726926905343, ++STORE, 94213246820352, 94213249044479, ++SNULL, 94213246930943, 94213249044479, ++STORE, 94213246820352, 94213246930943, ++STORE, 94213246930944, 94213249044479, ++ERASE, 94213246930944, 94213249044479, ++STORE, 94213249024000, 94213249036287, ++STORE, 94213249036288, 94213249044479, ++STORE, 140368830242816, 140368832495615, ++SNULL, 140368830386175, 140368832495615, ++STORE, 140368830242816, 140368830386175, ++STORE, 140368830386176, 140368832495615, ++ERASE, 140368830386176, 140368832495615, ++STORE, 140368832483328, 140368832491519, ++STORE, 140368832491520, 140368832495615, ++STORE, 140726926999552, 140726927003647, ++STORE, 140726926987264, 140726926999551, ++STORE, 140368832454656, 140368832483327, ++STORE, 140368832446464, 140368832454655, ++STORE, 140368826445824, 140368830242815, ++SNULL, 140368826445824, 140368828104703, ++STORE, 140368828104704, 140368830242815, ++STORE, 140368826445824, 140368828104703, ++SNULL, 140368830201855, 140368830242815, ++STORE, 140368828104704, 140368830201855, ++STORE, 140368830201856, 140368830242815, ++SNULL, 140368830201856, 140368830226431, ++STORE, 140368830226432, 140368830242815, ++STORE, 140368830201856, 140368830226431, ++ERASE, 140368830201856, 140368830226431, ++STORE, 140368830201856, 140368830226431, ++ERASE, 140368830226432, 140368830242815, ++STORE, 140368830226432, 140368830242815, ++SNULL, 140368830218239, 140368830226431, ++STORE, 140368830201856, 140368830218239, ++STORE, 140368830218240, 140368830226431, ++SNULL, 94213249032191, 94213249036287, ++STORE, 94213249024000, 94213249032191, ++STORE, 94213249032192, 94213249036287, ++SNULL, 140368832487423, 140368832491519, ++STORE, 140368832483328, 140368832487423, ++STORE, 140368832487424, 140368832491519, ++ERASE, 140368832454656, 140368832483327, ++STORE, 94213267435520, 94213267570687, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728954130432, 140737488351231, ++SNULL, 140728954138623, 140737488351231, ++STORE, 140728954130432, 140728954138623, ++STORE, 140728953999360, 140728954138623, ++STORE, 94672570966016, 94672573190143, ++SNULL, 94672571076607, 94672573190143, ++STORE, 94672570966016, 94672571076607, ++STORE, 94672571076608, 94672573190143, ++ERASE, 94672571076608, 94672573190143, ++STORE, 94672573169664, 94672573181951, ++STORE, 94672573181952, 94672573190143, ++STORE, 140201696735232, 140201698988031, ++SNULL, 140201696878591, 140201698988031, ++STORE, 140201696735232, 140201696878591, ++STORE, 140201696878592, 140201698988031, ++ERASE, 140201696878592, 140201698988031, ++STORE, 140201698975744, 140201698983935, ++STORE, 140201698983936, 140201698988031, ++STORE, 140728954163200, 140728954167295, ++STORE, 140728954150912, 140728954163199, ++STORE, 140201698947072, 140201698975743, ++STORE, 140201698938880, 140201698947071, ++STORE, 140201692938240, 140201696735231, ++SNULL, 140201692938240, 140201694597119, ++STORE, 140201694597120, 140201696735231, ++STORE, 140201692938240, 140201694597119, ++SNULL, 140201696694271, 140201696735231, ++STORE, 140201694597120, 140201696694271, ++STORE, 140201696694272, 140201696735231, ++SNULL, 140201696694272, 140201696718847, ++STORE, 140201696718848, 140201696735231, ++STORE, 140201696694272, 140201696718847, ++ERASE, 140201696694272, 140201696718847, ++STORE, 140201696694272, 140201696718847, ++ERASE, 140201696718848, 140201696735231, ++STORE, 140201696718848, 140201696735231, ++SNULL, 140201696710655, 140201696718847, ++STORE, 140201696694272, 140201696710655, ++STORE, 140201696710656, 140201696718847, ++SNULL, 94672573177855, 94672573181951, ++STORE, 94672573169664, 94672573177855, ++STORE, 94672573177856, 94672573181951, ++SNULL, 140201698979839, 140201698983935, ++STORE, 140201698975744, 140201698979839, ++STORE, 140201698979840, 140201698983935, ++ERASE, 140201698947072, 140201698975743, ++STORE, 94672595689472, 94672595824639, ++STORE, 94114394132480, 94114394345471, ++STORE, 94114396442624, 94114396446719, ++STORE, 94114396446720, 94114396454911, ++STORE, 94114396454912, 94114396467199, ++STORE, 94114421575680, 94114428256255, ++STORE, 139934313955328, 139934315614207, ++STORE, 139934315614208, 139934317711359, ++STORE, 139934317711360, 139934317727743, ++STORE, 139934317727744, 139934317735935, ++STORE, 139934317735936, 139934317752319, ++STORE, 139934317752320, 139934317764607, ++STORE, 139934317764608, 139934319857663, ++STORE, 139934319857664, 139934319861759, ++STORE, 139934319861760, 139934319865855, ++STORE, 139934319865856, 139934320009215, ++STORE, 139934320377856, 139934322061311, ++STORE, 139934322061312, 139934322077695, ++STORE, 139934322106368, 139934322110463, ++STORE, 139934322110464, 139934322114559, ++STORE, 139934322114560, 139934322118655, ++STORE, 140731200376832, 140731200516095, ++STORE, 140731200929792, 140731200942079, ++STORE, 140731200942080, 140731200946175, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721532362752, 140737488351231, ++SNULL, 140721532370943, 140737488351231, ++STORE, 140721532362752, 140721532370943, ++STORE, 140721532231680, 140721532370943, ++STORE, 94467222597632, 94467224821759, ++SNULL, 94467222708223, 94467224821759, ++STORE, 94467222597632, 94467222708223, ++STORE, 94467222708224, 94467224821759, ++ERASE, 94467222708224, 94467224821759, ++STORE, 94467224801280, 94467224813567, ++STORE, 94467224813568, 94467224821759, ++STORE, 140191433543680, 140191435796479, ++SNULL, 140191433687039, 140191435796479, ++STORE, 140191433543680, 140191433687039, ++STORE, 140191433687040, 140191435796479, ++ERASE, 140191433687040, 140191435796479, ++STORE, 140191435784192, 140191435792383, ++STORE, 140191435792384, 140191435796479, ++STORE, 140721533034496, 140721533038591, ++STORE, 140721533022208, 140721533034495, ++STORE, 140191435755520, 140191435784191, ++STORE, 140191435747328, 140191435755519, ++STORE, 140191429746688, 140191433543679, ++SNULL, 140191429746688, 140191431405567, ++STORE, 140191431405568, 140191433543679, ++STORE, 140191429746688, 140191431405567, ++SNULL, 140191433502719, 140191433543679, ++STORE, 140191431405568, 140191433502719, ++STORE, 140191433502720, 140191433543679, ++SNULL, 140191433502720, 140191433527295, ++STORE, 140191433527296, 140191433543679, ++STORE, 140191433502720, 140191433527295, ++ERASE, 140191433502720, 140191433527295, ++STORE, 140191433502720, 140191433527295, ++ERASE, 140191433527296, 140191433543679, ++STORE, 140191433527296, 140191433543679, ++SNULL, 140191433519103, 140191433527295, ++STORE, 140191433502720, 140191433519103, ++STORE, 140191433519104, 140191433527295, ++SNULL, 94467224809471, 94467224813567, ++STORE, 94467224801280, 94467224809471, ++STORE, 94467224809472, 94467224813567, ++SNULL, 140191435788287, 140191435792383, ++STORE, 140191435784192, 140191435788287, ++STORE, 140191435788288, 140191435792383, ++ERASE, 140191435755520, 140191435784191, ++STORE, 94467251847168, 94467251982335, ++STORE, 94367895400448, 94367895613439, ++STORE, 94367897710592, 94367897714687, ++STORE, 94367897714688, 94367897722879, ++STORE, 94367897722880, 94367897735167, ++STORE, 94367925264384, 94367926861823, ++STORE, 139801317548032, 139801319206911, ++STORE, 139801319206912, 139801321304063, ++STORE, 139801321304064, 139801321320447, ++STORE, 139801321320448, 139801321328639, ++STORE, 139801321328640, 139801321345023, ++STORE, 139801321345024, 139801321357311, ++STORE, 139801321357312, 139801323450367, ++STORE, 139801323450368, 139801323454463, ++STORE, 139801323454464, 139801323458559, ++STORE, 139801323458560, 139801323601919, ++STORE, 139801323970560, 139801325654015, ++STORE, 139801325654016, 139801325670399, ++STORE, 139801325699072, 139801325703167, ++STORE, 139801325703168, 139801325707263, ++STORE, 139801325707264, 139801325711359, ++STORE, 140724442861568, 140724443000831, ++STORE, 140724443611136, 140724443623423, ++STORE, 140724443623424, 140724443627519, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731353149440, 140737488351231, ++SNULL, 140731353157631, 140737488351231, ++STORE, 140731353149440, 140731353157631, ++STORE, 140731353018368, 140731353157631, ++STORE, 94310379503616, 94310381838335, ++SNULL, 94310379716607, 94310381838335, ++STORE, 94310379503616, 94310379716607, ++STORE, 94310379716608, 94310381838335, ++ERASE, 94310379716608, 94310381838335, ++STORE, 94310381813760, 94310381826047, ++STORE, 94310381826048, 94310381838335, ++STORE, 140515434659840, 140515436912639, ++SNULL, 140515434803199, 140515436912639, ++STORE, 140515434659840, 140515434803199, ++STORE, 140515434803200, 140515436912639, ++ERASE, 140515434803200, 140515436912639, ++STORE, 140515436900352, 140515436908543, ++STORE, 140515436908544, 140515436912639, ++STORE, 140731353886720, 140731353890815, ++STORE, 140731353874432, 140731353886719, ++STORE, 140515436871680, 140515436900351, ++STORE, 140515436863488, 140515436871679, ++STORE, 140515432546304, 140515434659839, ++SNULL, 140515432546304, 140515432558591, ++STORE, 140515432558592, 140515434659839, ++STORE, 140515432546304, 140515432558591, ++SNULL, 140515434651647, 140515434659839, ++STORE, 140515432558592, 140515434651647, ++STORE, 140515434651648, 140515434659839, ++ERASE, 140515434651648, 140515434659839, ++STORE, 140515434651648, 140515434659839, ++STORE, 140515428749312, 140515432546303, ++SNULL, 140515428749312, 140515430408191, ++STORE, 140515430408192, 140515432546303, ++STORE, 140515428749312, 140515430408191, ++SNULL, 140515432505343, 140515432546303, ++STORE, 140515430408192, 140515432505343, ++STORE, 140515432505344, 140515432546303, ++SNULL, 140515432505344, 140515432529919, ++STORE, 140515432529920, 140515432546303, ++STORE, 140515432505344, 140515432529919, ++ERASE, 140515432505344, 140515432529919, ++STORE, 140515432505344, 140515432529919, ++ERASE, 140515432529920, 140515432546303, ++STORE, 140515432529920, 140515432546303, ++STORE, 140515436855296, 140515436871679, ++SNULL, 140515432521727, 140515432529919, ++STORE, 140515432505344, 140515432521727, ++STORE, 140515432521728, 140515432529919, ++SNULL, 140515434655743, 140515434659839, ++STORE, 140515434651648, 140515434655743, ++STORE, 140515434655744, 140515434659839, ++SNULL, 94310381817855, 94310381826047, ++STORE, 94310381813760, 94310381817855, ++STORE, 94310381817856, 94310381826047, ++SNULL, 140515436904447, 140515436908543, ++STORE, 140515436900352, 140515436904447, ++STORE, 140515436904448, 140515436908543, ++ERASE, 140515436871680, 140515436900351, ++STORE, 94310395457536, 94310395592703, ++STORE, 140515435171840, 140515436855295, ++STORE, 94310395457536, 94310395727871, ++STORE, 94310395457536, 94310395863039, ++STORE, 94310395457536, 94310396047359, ++SNULL, 94310396022783, 94310396047359, ++STORE, 94310395457536, 94310396022783, ++STORE, 94310396022784, 94310396047359, ++ERASE, 94310396022784, 94310396047359, ++STORE, 94310395457536, 94310396157951, ++STORE, 94310395457536, 94310396293119, ++SNULL, 94310396276735, 94310396293119, ++STORE, 94310395457536, 94310396276735, ++STORE, 94310396276736, 94310396293119, ++ERASE, 94310396276736, 94310396293119, ++STORE, 94310395457536, 94310396411903, ++SNULL, 94310396383231, 94310396411903, ++STORE, 94310395457536, 94310396383231, ++STORE, 94310396383232, 94310396411903, ++ERASE, 94310396383232, 94310396411903, ++STORE, 94310395457536, 94310396522495, ++STORE, 94310395457536, 94310396674047, ++SNULL, 94310396657663, 94310396674047, ++STORE, 94310395457536, 94310396657663, ++STORE, 94310396657664, 94310396674047, ++ERASE, 94310396657664, 94310396674047, ++SNULL, 94310396624895, 94310396657663, ++STORE, 94310395457536, 94310396624895, ++STORE, 94310396624896, 94310396657663, ++ERASE, 94310396624896, 94310396657663, ++STORE, 94310395457536, 94310396776447, ++SNULL, 94310396764159, 94310396776447, ++STORE, 94310395457536, 94310396764159, ++STORE, 94310396764160, 94310396776447, ++ERASE, 94310396764160, 94310396776447, ++SNULL, 94310396739583, 94310396764159, ++STORE, 94310395457536, 94310396739583, ++STORE, 94310396739584, 94310396764159, ++ERASE, 94310396739584, 94310396764159, ++STORE, 94310395457536, 94310396882943, ++STORE, 94310395457536, 94310397018111, ++STORE, 94310395457536, 94310397161471, ++STORE, 94310395457536, 94310397300735, ++SNULL, 94310397292543, 94310397300735, ++STORE, 94310395457536, 94310397292543, ++STORE, 94310397292544, 94310397300735, ++ERASE, 94310397292544, 94310397300735, ++STORE, 94359222210560, 94359222423551, ++STORE, 94359224520704, 94359224524799, ++STORE, 94359224524800, 94359224532991, ++STORE, 94359224532992, 94359224545279, ++STORE, 94359238348800, 94359239385087, ++STORE, 140675699838976, 140675701497855, ++STORE, 140675701497856, 140675703595007, ++STORE, 140675703595008, 140675703611391, ++STORE, 140675703611392, 140675703619583, ++STORE, 140675703619584, 140675703635967, ++STORE, 140675703635968, 140675703648255, ++STORE, 140675703648256, 140675705741311, ++STORE, 140675705741312, 140675705745407, ++STORE, 140675705745408, 140675705749503, ++STORE, 140675705749504, 140675705892863, ++STORE, 140675706261504, 140675707944959, ++STORE, 140675707944960, 140675707961343, ++STORE, 140675707990016, 140675707994111, ++STORE, 140675707994112, 140675707998207, ++STORE, 140675707998208, 140675708002303, ++STORE, 140721324634112, 140721324773375, ++STORE, 140721324810240, 140721324822527, ++STORE, 140721324822528, 140721324826623, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724099678208, 140737488351231, ++SNULL, 140724099686399, 140737488351231, ++STORE, 140724099678208, 140724099686399, ++STORE, 140724099547136, 140724099686399, ++STORE, 94586638516224, 94586640850943, ++SNULL, 94586638729215, 94586640850943, ++STORE, 94586638516224, 94586638729215, ++STORE, 94586638729216, 94586640850943, ++ERASE, 94586638729216, 94586640850943, ++STORE, 94586640826368, 94586640838655, ++STORE, 94586640838656, 94586640850943, ++STORE, 140371033796608, 140371036049407, ++SNULL, 140371033939967, 140371036049407, ++STORE, 140371033796608, 140371033939967, ++STORE, 140371033939968, 140371036049407, ++ERASE, 140371033939968, 140371036049407, ++STORE, 140371036037120, 140371036045311, ++STORE, 140371036045312, 140371036049407, ++STORE, 140724100001792, 140724100005887, ++STORE, 140724099989504, 140724100001791, ++STORE, 140371036008448, 140371036037119, ++STORE, 140371036000256, 140371036008447, ++STORE, 140371031683072, 140371033796607, ++SNULL, 140371031683072, 140371031695359, ++STORE, 140371031695360, 140371033796607, ++STORE, 140371031683072, 140371031695359, ++SNULL, 140371033788415, 140371033796607, ++STORE, 140371031695360, 140371033788415, ++STORE, 140371033788416, 140371033796607, ++ERASE, 140371033788416, 140371033796607, ++STORE, 140371033788416, 140371033796607, ++STORE, 140371027886080, 140371031683071, ++SNULL, 140371027886080, 140371029544959, ++STORE, 140371029544960, 140371031683071, ++STORE, 140371027886080, 140371029544959, ++SNULL, 140371031642111, 140371031683071, ++STORE, 140371029544960, 140371031642111, ++STORE, 140371031642112, 140371031683071, ++SNULL, 140371031642112, 140371031666687, ++STORE, 140371031666688, 140371031683071, ++STORE, 140371031642112, 140371031666687, ++ERASE, 140371031642112, 140371031666687, ++STORE, 140371031642112, 140371031666687, ++ERASE, 140371031666688, 140371031683071, ++STORE, 140371031666688, 140371031683071, ++STORE, 140371035992064, 140371036008447, ++SNULL, 140371031658495, 140371031666687, ++STORE, 140371031642112, 140371031658495, ++STORE, 140371031658496, 140371031666687, ++SNULL, 140371033792511, 140371033796607, ++STORE, 140371033788416, 140371033792511, ++STORE, 140371033792512, 140371033796607, ++SNULL, 94586640830463, 94586640838655, ++STORE, 94586640826368, 94586640830463, ++STORE, 94586640830464, 94586640838655, ++SNULL, 140371036041215, 140371036045311, ++STORE, 140371036037120, 140371036041215, ++STORE, 140371036041216, 140371036045311, ++ERASE, 140371036008448, 140371036037119, ++STORE, 94586663849984, 94586663985151, ++STORE, 140371034308608, 140371035992063, ++STORE, 94586663849984, 94586664120319, ++STORE, 94586663849984, 94586664255487, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727532937216, 140737488351231, ++SNULL, 140727532945407, 140737488351231, ++STORE, 140727532937216, 140727532945407, ++STORE, 140727532806144, 140727532945407, ++STORE, 94849780191232, 94849782525951, ++SNULL, 94849780404223, 94849782525951, ++STORE, 94849780191232, 94849780404223, ++STORE, 94849780404224, 94849782525951, ++ERASE, 94849780404224, 94849782525951, ++STORE, 94849782501376, 94849782513663, ++STORE, 94849782513664, 94849782525951, ++STORE, 140382070218752, 140382072471551, ++SNULL, 140382070362111, 140382072471551, ++STORE, 140382070218752, 140382070362111, ++STORE, 140382070362112, 140382072471551, ++ERASE, 140382070362112, 140382072471551, ++STORE, 140382072459264, 140382072467455, ++STORE, 140382072467456, 140382072471551, ++STORE, 140727533092864, 140727533096959, ++STORE, 140727533080576, 140727533092863, ++STORE, 140382072430592, 140382072459263, ++STORE, 140382072422400, 140382072430591, ++STORE, 140382068105216, 140382070218751, ++SNULL, 140382068105216, 140382068117503, ++STORE, 140382068117504, 140382070218751, ++STORE, 140382068105216, 140382068117503, ++SNULL, 140382070210559, 140382070218751, ++STORE, 140382068117504, 140382070210559, ++STORE, 140382070210560, 140382070218751, ++ERASE, 140382070210560, 140382070218751, ++STORE, 140382070210560, 140382070218751, ++STORE, 140382064308224, 140382068105215, ++SNULL, 140382064308224, 140382065967103, ++STORE, 140382065967104, 140382068105215, ++STORE, 140382064308224, 140382065967103, ++SNULL, 140382068064255, 140382068105215, ++STORE, 140382065967104, 140382068064255, ++STORE, 140382068064256, 140382068105215, ++SNULL, 140382068064256, 140382068088831, ++STORE, 140382068088832, 140382068105215, ++STORE, 140382068064256, 140382068088831, ++ERASE, 140382068064256, 140382068088831, ++STORE, 140382068064256, 140382068088831, ++ERASE, 140382068088832, 140382068105215, ++STORE, 140382068088832, 140382068105215, ++STORE, 140382072414208, 140382072430591, ++SNULL, 140382068080639, 140382068088831, ++STORE, 140382068064256, 140382068080639, ++STORE, 140382068080640, 140382068088831, ++SNULL, 140382070214655, 140382070218751, ++STORE, 140382070210560, 140382070214655, ++STORE, 140382070214656, 140382070218751, ++SNULL, 94849782505471, 94849782513663, ++STORE, 94849782501376, 94849782505471, ++STORE, 94849782505472, 94849782513663, ++SNULL, 140382072463359, 140382072467455, ++STORE, 140382072459264, 140382072463359, ++STORE, 140382072463360, 140382072467455, ++ERASE, 140382072430592, 140382072459263, ++STORE, 94849782845440, 94849782980607, ++STORE, 140382070730752, 140382072414207, ++STORE, 94849782845440, 94849783115775, ++STORE, 94849782845440, 94849783250943, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722594377728, 140737488351231, ++SNULL, 140722594385919, 140737488351231, ++STORE, 140722594377728, 140722594385919, ++STORE, 140722594246656, 140722594385919, ++STORE, 94421466353664, 94421468577791, ++SNULL, 94421466464255, 94421468577791, ++STORE, 94421466353664, 94421466464255, ++STORE, 94421466464256, 94421468577791, ++ERASE, 94421466464256, 94421468577791, ++STORE, 94421468557312, 94421468569599, ++STORE, 94421468569600, 94421468577791, ++STORE, 140345458057216, 140345460310015, ++SNULL, 140345458200575, 140345460310015, ++STORE, 140345458057216, 140345458200575, ++STORE, 140345458200576, 140345460310015, ++ERASE, 140345458200576, 140345460310015, ++STORE, 140345460297728, 140345460305919, ++STORE, 140345460305920, 140345460310015, ++STORE, 140722595557376, 140722595561471, ++STORE, 140722595545088, 140722595557375, ++STORE, 140345460269056, 140345460297727, ++STORE, 140345460260864, 140345460269055, ++STORE, 140345454260224, 140345458057215, ++SNULL, 140345454260224, 140345455919103, ++STORE, 140345455919104, 140345458057215, ++STORE, 140345454260224, 140345455919103, ++SNULL, 140345458016255, 140345458057215, ++STORE, 140345455919104, 140345458016255, ++STORE, 140345458016256, 140345458057215, ++SNULL, 140345458016256, 140345458040831, ++STORE, 140345458040832, 140345458057215, ++STORE, 140345458016256, 140345458040831, ++ERASE, 140345458016256, 140345458040831, ++STORE, 140345458016256, 140345458040831, ++ERASE, 140345458040832, 140345458057215, ++STORE, 140345458040832, 140345458057215, ++SNULL, 140345458032639, 140345458040831, ++STORE, 140345458016256, 140345458032639, ++STORE, 140345458032640, 140345458040831, ++SNULL, 94421468565503, 94421468569599, ++STORE, 94421468557312, 94421468565503, ++STORE, 94421468565504, 94421468569599, ++SNULL, 140345460301823, 140345460305919, ++STORE, 140345460297728, 140345460301823, ++STORE, 140345460301824, 140345460305919, ++ERASE, 140345460269056, 140345460297727, ++STORE, 94421496004608, 94421496139775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726096302080, 140737488351231, ++SNULL, 140726096310271, 140737488351231, ++STORE, 140726096302080, 140726096310271, ++STORE, 140726096171008, 140726096310271, ++STORE, 94101992124416, 94101994459135, ++SNULL, 94101992337407, 94101994459135, ++STORE, 94101992124416, 94101992337407, ++STORE, 94101992337408, 94101994459135, ++ERASE, 94101992337408, 94101994459135, ++STORE, 94101994434560, 94101994446847, ++STORE, 94101994446848, 94101994459135, ++STORE, 140192085594112, 140192087846911, ++SNULL, 140192085737471, 140192087846911, ++STORE, 140192085594112, 140192085737471, ++STORE, 140192085737472, 140192087846911, ++ERASE, 140192085737472, 140192087846911, ++STORE, 140192087834624, 140192087842815, ++STORE, 140192087842816, 140192087846911, ++STORE, 140726096375808, 140726096379903, ++STORE, 140726096363520, 140726096375807, ++STORE, 140192087805952, 140192087834623, ++STORE, 140192087797760, 140192087805951, ++STORE, 140192083480576, 140192085594111, ++SNULL, 140192083480576, 140192083492863, ++STORE, 140192083492864, 140192085594111, ++STORE, 140192083480576, 140192083492863, ++SNULL, 140192085585919, 140192085594111, ++STORE, 140192083492864, 140192085585919, ++STORE, 140192085585920, 140192085594111, ++ERASE, 140192085585920, 140192085594111, ++STORE, 140192085585920, 140192085594111, ++STORE, 140192079683584, 140192083480575, ++SNULL, 140192079683584, 140192081342463, ++STORE, 140192081342464, 140192083480575, ++STORE, 140192079683584, 140192081342463, ++SNULL, 140192083439615, 140192083480575, ++STORE, 140192081342464, 140192083439615, ++STORE, 140192083439616, 140192083480575, ++SNULL, 140192083439616, 140192083464191, ++STORE, 140192083464192, 140192083480575, ++STORE, 140192083439616, 140192083464191, ++ERASE, 140192083439616, 140192083464191, ++STORE, 140192083439616, 140192083464191, ++ERASE, 140192083464192, 140192083480575, ++STORE, 140192083464192, 140192083480575, ++STORE, 140192087789568, 140192087805951, ++SNULL, 140192083455999, 140192083464191, ++STORE, 140192083439616, 140192083455999, ++STORE, 140192083456000, 140192083464191, ++SNULL, 140192085590015, 140192085594111, ++STORE, 140192085585920, 140192085590015, ++STORE, 140192085590016, 140192085594111, ++SNULL, 94101994438655, 94101994446847, ++STORE, 94101994434560, 94101994438655, ++STORE, 94101994438656, 94101994446847, ++SNULL, 140192087838719, 140192087842815, ++STORE, 140192087834624, 140192087838719, ++STORE, 140192087838720, 140192087842815, ++ERASE, 140192087805952, 140192087834623, ++STORE, 94102011887616, 94102012022783, ++STORE, 140192086106112, 140192087789567, ++STORE, 94102011887616, 94102012157951, ++STORE, 94102011887616, 94102012293119, ++STORE, 94102011887616, 94102012440575, ++SNULL, 94102012428287, 94102012440575, ++STORE, 94102011887616, 94102012428287, ++STORE, 94102012428288, 94102012440575, ++ERASE, 94102012428288, 94102012440575, ++STORE, 94102011887616, 94102012579839, ++STORE, 94102011887616, 94102012715007, ++SNULL, 94102012694527, 94102012715007, ++STORE, 94102011887616, 94102012694527, ++STORE, 94102012694528, 94102012715007, ++ERASE, 94102012694528, 94102012715007, ++STORE, 94102011887616, 94102012833791, ++STORE, 94102011887616, 94102012968959, ++SNULL, 94102012927999, 94102012968959, ++STORE, 94102011887616, 94102012927999, ++STORE, 94102012928000, 94102012968959, ++ERASE, 94102012928000, 94102012968959, ++STORE, 94102011887616, 94102013091839, ++SNULL, 94102013075455, 94102013091839, ++STORE, 94102011887616, 94102013075455, ++STORE, 94102013075456, 94102013091839, ++ERASE, 94102013075456, 94102013091839, ++STORE, 94102011887616, 94102013210623, ++STORE, 94102011887616, 94102013345791, ++STORE, 93968727965696, 93968728178687, ++STORE, 93968730275840, 93968730279935, ++STORE, 93968730279936, 93968730288127, ++STORE, 93968730288128, 93968730300415, ++STORE, 93968731140096, 93968732704767, ++STORE, 140588443168768, 140588444827647, ++STORE, 140588444827648, 140588446924799, ++STORE, 140588446924800, 140588446941183, ++STORE, 140588446941184, 140588446949375, ++STORE, 140588446949376, 140588446965759, ++STORE, 140588446965760, 140588446978047, ++STORE, 140588446978048, 140588449071103, ++STORE, 140588449071104, 140588449075199, ++STORE, 140588449075200, 140588449079295, ++STORE, 140588449079296, 140588449222655, ++STORE, 140588449591296, 140588451274751, ++STORE, 140588451274752, 140588451291135, ++STORE, 140588451319808, 140588451323903, ++STORE, 140588451323904, 140588451327999, ++STORE, 140588451328000, 140588451332095, ++STORE, 140733877239808, 140733877379071, ++STORE, 140733878702080, 140733878714367, ++STORE, 140733878714368, 140733878718463, ++STORE, 93968727965696, 93968728178687, ++STORE, 93968730275840, 93968730279935, ++STORE, 93968730279936, 93968730288127, ++STORE, 93968730288128, 93968730300415, ++STORE, 93968731140096, 93968732991487, ++STORE, 140588443168768, 140588444827647, ++STORE, 140588444827648, 140588446924799, ++STORE, 140588446924800, 140588446941183, ++STORE, 140588446941184, 140588446949375, ++STORE, 140588446949376, 140588446965759, ++STORE, 140588446965760, 140588446978047, ++STORE, 140588446978048, 140588449071103, ++STORE, 140588449071104, 140588449075199, ++STORE, 140588449075200, 140588449079295, ++STORE, 140588449079296, 140588449222655, ++STORE, 140588449591296, 140588451274751, ++STORE, 140588451274752, 140588451291135, ++STORE, 140588451319808, 140588451323903, ++STORE, 140588451323904, 140588451327999, ++STORE, 140588451328000, 140588451332095, ++STORE, 140733877239808, 140733877379071, ++STORE, 140733878702080, 140733878714367, ++STORE, 140733878714368, 140733878718463, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733054472192, 140737488351231, ++SNULL, 140733054480383, 140737488351231, ++STORE, 140733054472192, 140733054480383, ++STORE, 140733054341120, 140733054480383, ++STORE, 93992873623552, 93992875847679, ++SNULL, 93992873734143, 93992875847679, ++STORE, 93992873623552, 93992873734143, ++STORE, 93992873734144, 93992875847679, ++ERASE, 93992873734144, 93992875847679, ++STORE, 93992875827200, 93992875839487, ++STORE, 93992875839488, 93992875847679, ++STORE, 139790881488896, 139790883741695, ++SNULL, 139790881632255, 139790883741695, ++STORE, 139790881488896, 139790881632255, ++STORE, 139790881632256, 139790883741695, ++ERASE, 139790881632256, 139790883741695, ++STORE, 139790883729408, 139790883737599, ++STORE, 139790883737600, 139790883741695, ++STORE, 140733054754816, 140733054758911, ++STORE, 140733054742528, 140733054754815, ++STORE, 139790883700736, 139790883729407, ++STORE, 139790883692544, 139790883700735, ++STORE, 139790877691904, 139790881488895, ++SNULL, 139790877691904, 139790879350783, ++STORE, 139790879350784, 139790881488895, ++STORE, 139790877691904, 139790879350783, ++SNULL, 139790881447935, 139790881488895, ++STORE, 139790879350784, 139790881447935, ++STORE, 139790881447936, 139790881488895, ++SNULL, 139790881447936, 139790881472511, ++STORE, 139790881472512, 139790881488895, ++STORE, 139790881447936, 139790881472511, ++ERASE, 139790881447936, 139790881472511, ++STORE, 139790881447936, 139790881472511, ++ERASE, 139790881472512, 139790881488895, ++STORE, 139790881472512, 139790881488895, ++SNULL, 139790881464319, 139790881472511, ++STORE, 139790881447936, 139790881464319, ++STORE, 139790881464320, 139790881472511, ++SNULL, 93992875835391, 93992875839487, ++STORE, 93992875827200, 93992875835391, ++STORE, 93992875835392, 93992875839487, ++SNULL, 139790883733503, 139790883737599, ++STORE, 139790883729408, 139790883733503, ++STORE, 139790883733504, 139790883737599, ++ERASE, 139790883700736, 139790883729407, ++STORE, 93992877031424, 93992877166591, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728550887424, 140737488351231, ++SNULL, 140728550895615, 140737488351231, ++STORE, 140728550887424, 140728550895615, ++STORE, 140728550756352, 140728550895615, ++STORE, 94707634077696, 94707636301823, ++SNULL, 94707634188287, 94707636301823, ++STORE, 94707634077696, 94707634188287, ++STORE, 94707634188288, 94707636301823, ++ERASE, 94707634188288, 94707636301823, ++STORE, 94707636281344, 94707636293631, ++STORE, 94707636293632, 94707636301823, ++STORE, 140553545666560, 140553547919359, ++SNULL, 140553545809919, 140553547919359, ++STORE, 140553545666560, 140553545809919, ++STORE, 140553545809920, 140553547919359, ++ERASE, 140553545809920, 140553547919359, ++STORE, 140553547907072, 140553547915263, ++STORE, 140553547915264, 140553547919359, ++STORE, 140728552374272, 140728552378367, ++STORE, 140728552361984, 140728552374271, ++STORE, 140553547878400, 140553547907071, ++STORE, 140553547870208, 140553547878399, ++STORE, 140553541869568, 140553545666559, ++SNULL, 140553541869568, 140553543528447, ++STORE, 140553543528448, 140553545666559, ++STORE, 140553541869568, 140553543528447, ++SNULL, 140553545625599, 140553545666559, ++STORE, 140553543528448, 140553545625599, ++STORE, 140553545625600, 140553545666559, ++SNULL, 140553545625600, 140553545650175, ++STORE, 140553545650176, 140553545666559, ++STORE, 140553545625600, 140553545650175, ++ERASE, 140553545625600, 140553545650175, ++STORE, 140553545625600, 140553545650175, ++ERASE, 140553545650176, 140553545666559, ++STORE, 140553545650176, 140553545666559, ++SNULL, 140553545641983, 140553545650175, ++STORE, 140553545625600, 140553545641983, ++STORE, 140553545641984, 140553545650175, ++SNULL, 94707636289535, 94707636293631, ++STORE, 94707636281344, 94707636289535, ++STORE, 94707636289536, 94707636293631, ++SNULL, 140553547911167, 140553547915263, ++STORE, 140553547907072, 140553547911167, ++STORE, 140553547911168, 140553547915263, ++ERASE, 140553547878400, 140553547907071, ++STORE, 94707651411968, 94707651547135, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732168695808, 140737488351231, ++SNULL, 140732168703999, 140737488351231, ++STORE, 140732168695808, 140732168703999, ++STORE, 140732168564736, 140732168703999, ++STORE, 94454287859712, 94454290083839, ++SNULL, 94454287970303, 94454290083839, ++STORE, 94454287859712, 94454287970303, ++STORE, 94454287970304, 94454290083839, ++ERASE, 94454287970304, 94454290083839, ++STORE, 94454290063360, 94454290075647, ++STORE, 94454290075648, 94454290083839, ++STORE, 140564947107840, 140564949360639, ++SNULL, 140564947251199, 140564949360639, ++STORE, 140564947107840, 140564947251199, ++STORE, 140564947251200, 140564949360639, ++ERASE, 140564947251200, 140564949360639, ++STORE, 140564949348352, 140564949356543, ++STORE, 140564949356544, 140564949360639, ++STORE, 140732168843264, 140732168847359, ++STORE, 140732168830976, 140732168843263, ++STORE, 140564949319680, 140564949348351, ++STORE, 140564949311488, 140564949319679, ++STORE, 140564943310848, 140564947107839, ++SNULL, 140564943310848, 140564944969727, ++STORE, 140564944969728, 140564947107839, ++STORE, 140564943310848, 140564944969727, ++SNULL, 140564947066879, 140564947107839, ++STORE, 140564944969728, 140564947066879, ++STORE, 140564947066880, 140564947107839, ++SNULL, 140564947066880, 140564947091455, ++STORE, 140564947091456, 140564947107839, ++STORE, 140564947066880, 140564947091455, ++ERASE, 140564947066880, 140564947091455, ++STORE, 140564947066880, 140564947091455, ++ERASE, 140564947091456, 140564947107839, ++STORE, 140564947091456, 140564947107839, ++SNULL, 140564947083263, 140564947091455, ++STORE, 140564947066880, 140564947083263, ++STORE, 140564947083264, 140564947091455, ++SNULL, 94454290071551, 94454290075647, ++STORE, 94454290063360, 94454290071551, ++STORE, 94454290071552, 94454290075647, ++SNULL, 140564949352447, 140564949356543, ++STORE, 140564949348352, 140564949352447, ++STORE, 140564949352448, 140564949356543, ++ERASE, 140564949319680, 140564949348351, ++STORE, 94454316236800, 94454316371967, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735155617792, 140737488351231, ++SNULL, 140735155625983, 140737488351231, ++STORE, 140735155617792, 140735155625983, ++STORE, 140735155486720, 140735155625983, ++STORE, 93915969556480, 93915971780607, ++SNULL, 93915969667071, 93915971780607, ++STORE, 93915969556480, 93915969667071, ++STORE, 93915969667072, 93915971780607, ++ERASE, 93915969667072, 93915971780607, ++STORE, 93915971760128, 93915971772415, ++STORE, 93915971772416, 93915971780607, ++STORE, 140141164605440, 140141166858239, ++SNULL, 140141164748799, 140141166858239, ++STORE, 140141164605440, 140141164748799, ++STORE, 140141164748800, 140141166858239, ++ERASE, 140141164748800, 140141166858239, ++STORE, 140141166845952, 140141166854143, ++STORE, 140141166854144, 140141166858239, ++STORE, 140735155691520, 140735155695615, ++STORE, 140735155679232, 140735155691519, ++STORE, 140141166817280, 140141166845951, ++STORE, 140141166809088, 140141166817279, ++STORE, 140141160808448, 140141164605439, ++SNULL, 140141160808448, 140141162467327, ++STORE, 140141162467328, 140141164605439, ++STORE, 140141160808448, 140141162467327, ++SNULL, 140141164564479, 140141164605439, ++STORE, 140141162467328, 140141164564479, ++STORE, 140141164564480, 140141164605439, ++SNULL, 140141164564480, 140141164589055, ++STORE, 140141164589056, 140141164605439, ++STORE, 140141164564480, 140141164589055, ++ERASE, 140141164564480, 140141164589055, ++STORE, 140141164564480, 140141164589055, ++ERASE, 140141164589056, 140141164605439, ++STORE, 140141164589056, 140141164605439, ++SNULL, 140141164580863, 140141164589055, ++STORE, 140141164564480, 140141164580863, ++STORE, 140141164580864, 140141164589055, ++SNULL, 93915971768319, 93915971772415, ++STORE, 93915971760128, 93915971768319, ++STORE, 93915971768320, 93915971772415, ++SNULL, 140141166850047, 140141166854143, ++STORE, 140141166845952, 140141166850047, ++STORE, 140141166850048, 140141166854143, ++ERASE, 140141166817280, 140141166845951, ++STORE, 93916002775040, 93916002910207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728988409856, 140737488351231, ++SNULL, 140728988418047, 140737488351231, ++STORE, 140728988409856, 140728988418047, ++STORE, 140728988278784, 140728988418047, ++STORE, 94021634813952, 94021637038079, ++SNULL, 94021634924543, 94021637038079, ++STORE, 94021634813952, 94021634924543, ++STORE, 94021634924544, 94021637038079, ++ERASE, 94021634924544, 94021637038079, ++STORE, 94021637017600, 94021637029887, ++STORE, 94021637029888, 94021637038079, ++STORE, 140638014038016, 140638016290815, ++SNULL, 140638014181375, 140638016290815, ++STORE, 140638014038016, 140638014181375, ++STORE, 140638014181376, 140638016290815, ++ERASE, 140638014181376, 140638016290815, ++STORE, 140638016278528, 140638016286719, ++STORE, 140638016286720, 140638016290815, ++STORE, 140728988536832, 140728988540927, ++STORE, 140728988524544, 140728988536831, ++STORE, 140638016249856, 140638016278527, ++STORE, 140638016241664, 140638016249855, ++STORE, 140638010241024, 140638014038015, ++SNULL, 140638010241024, 140638011899903, ++STORE, 140638011899904, 140638014038015, ++STORE, 140638010241024, 140638011899903, ++SNULL, 140638013997055, 140638014038015, ++STORE, 140638011899904, 140638013997055, ++STORE, 140638013997056, 140638014038015, ++SNULL, 140638013997056, 140638014021631, ++STORE, 140638014021632, 140638014038015, ++STORE, 140638013997056, 140638014021631, ++ERASE, 140638013997056, 140638014021631, ++STORE, 140638013997056, 140638014021631, ++ERASE, 140638014021632, 140638014038015, ++STORE, 140638014021632, 140638014038015, ++SNULL, 140638014013439, 140638014021631, ++STORE, 140638013997056, 140638014013439, ++STORE, 140638014013440, 140638014021631, ++SNULL, 94021637025791, 94021637029887, ++STORE, 94021637017600, 94021637025791, ++STORE, 94021637025792, 94021637029887, ++SNULL, 140638016282623, 140638016286719, ++STORE, 140638016278528, 140638016282623, ++STORE, 140638016282624, 140638016286719, ++ERASE, 140638016249856, 140638016278527, ++STORE, 94021643124736, 94021643259903, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731219275776, 140737488351231, ++SNULL, 140731219283967, 140737488351231, ++STORE, 140731219275776, 140731219283967, ++STORE, 140731219144704, 140731219283967, ++STORE, 93888803647488, 93888805871615, ++SNULL, 93888803758079, 93888805871615, ++STORE, 93888803647488, 93888803758079, ++STORE, 93888803758080, 93888805871615, ++ERASE, 93888803758080, 93888805871615, ++STORE, 93888805851136, 93888805863423, ++STORE, 93888805863424, 93888805871615, ++STORE, 139630576934912, 139630579187711, ++SNULL, 139630577078271, 139630579187711, ++STORE, 139630576934912, 139630577078271, ++STORE, 139630577078272, 139630579187711, ++ERASE, 139630577078272, 139630579187711, ++STORE, 139630579175424, 139630579183615, ++STORE, 139630579183616, 139630579187711, ++STORE, 140731219718144, 140731219722239, ++STORE, 140731219705856, 140731219718143, ++STORE, 139630579146752, 139630579175423, ++STORE, 139630579138560, 139630579146751, ++STORE, 139630573137920, 139630576934911, ++SNULL, 139630573137920, 139630574796799, ++STORE, 139630574796800, 139630576934911, ++STORE, 139630573137920, 139630574796799, ++SNULL, 139630576893951, 139630576934911, ++STORE, 139630574796800, 139630576893951, ++STORE, 139630576893952, 139630576934911, ++SNULL, 139630576893952, 139630576918527, ++STORE, 139630576918528, 139630576934911, ++STORE, 139630576893952, 139630576918527, ++ERASE, 139630576893952, 139630576918527, ++STORE, 139630576893952, 139630576918527, ++ERASE, 139630576918528, 139630576934911, ++STORE, 139630576918528, 139630576934911, ++SNULL, 139630576910335, 139630576918527, ++STORE, 139630576893952, 139630576910335, ++STORE, 139630576910336, 139630576918527, ++SNULL, 93888805859327, 93888805863423, ++STORE, 93888805851136, 93888805859327, ++STORE, 93888805859328, 93888805863423, ++SNULL, 139630579179519, 139630579183615, ++STORE, 139630579175424, 139630579179519, ++STORE, 139630579179520, 139630579183615, ++ERASE, 139630579146752, 139630579175423, ++STORE, 93888822235136, 93888822370303, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733391151104, 140737488351231, ++SNULL, 140733391159295, 140737488351231, ++STORE, 140733391151104, 140733391159295, ++STORE, 140733391020032, 140733391159295, ++STORE, 94393875324928, 94393877549055, ++SNULL, 94393875435519, 94393877549055, ++STORE, 94393875324928, 94393875435519, ++STORE, 94393875435520, 94393877549055, ++ERASE, 94393875435520, 94393877549055, ++STORE, 94393877528576, 94393877540863, ++STORE, 94393877540864, 94393877549055, ++STORE, 140292111740928, 140292113993727, ++SNULL, 140292111884287, 140292113993727, ++STORE, 140292111740928, 140292111884287, ++STORE, 140292111884288, 140292113993727, ++ERASE, 140292111884288, 140292113993727, ++STORE, 140292113981440, 140292113989631, ++STORE, 140292113989632, 140292113993727, ++STORE, 140733391532032, 140733391536127, ++STORE, 140733391519744, 140733391532031, ++STORE, 140292113952768, 140292113981439, ++STORE, 140292113944576, 140292113952767, ++STORE, 140292107943936, 140292111740927, ++SNULL, 140292107943936, 140292109602815, ++STORE, 140292109602816, 140292111740927, ++STORE, 140292107943936, 140292109602815, ++SNULL, 140292111699967, 140292111740927, ++STORE, 140292109602816, 140292111699967, ++STORE, 140292111699968, 140292111740927, ++SNULL, 140292111699968, 140292111724543, ++STORE, 140292111724544, 140292111740927, ++STORE, 140292111699968, 140292111724543, ++ERASE, 140292111699968, 140292111724543, ++STORE, 140292111699968, 140292111724543, ++ERASE, 140292111724544, 140292111740927, ++STORE, 140292111724544, 140292111740927, ++SNULL, 140292111716351, 140292111724543, ++STORE, 140292111699968, 140292111716351, ++STORE, 140292111716352, 140292111724543, ++SNULL, 94393877536767, 94393877540863, ++STORE, 94393877528576, 94393877536767, ++STORE, 94393877536768, 94393877540863, ++SNULL, 140292113985535, 140292113989631, ++STORE, 140292113981440, 140292113985535, ++STORE, 140292113985536, 140292113989631, ++ERASE, 140292113952768, 140292113981439, ++STORE, 94393909342208, 94393909477375, ++STORE, 94458367512576, 94458367725567, ++STORE, 94458369822720, 94458369826815, ++STORE, 94458369826816, 94458369835007, ++STORE, 94458369835008, 94458369847295, ++STORE, 94458393292800, 94458399666175, ++STORE, 140619773841408, 140619775500287, ++STORE, 140619775500288, 140619777597439, ++STORE, 140619777597440, 140619777613823, ++STORE, 140619777613824, 140619777622015, ++STORE, 140619777622016, 140619777638399, ++STORE, 140619777638400, 140619777650687, ++STORE, 140619777650688, 140619779743743, ++STORE, 140619779743744, 140619779747839, ++STORE, 140619779747840, 140619779751935, ++STORE, 140619779751936, 140619779895295, ++STORE, 140619780263936, 140619781947391, ++STORE, 140619781947392, 140619781963775, ++STORE, 140619781992448, 140619781996543, ++STORE, 140619781996544, 140619782000639, ++STORE, 140619782000640, 140619782004735, ++STORE, 140725811675136, 140725811814399, ++STORE, 140725812813824, 140725812826111, ++STORE, 140725812826112, 140725812830207, ++STORE, 94458367512576, 94458367725567, ++STORE, 94458369822720, 94458369826815, ++STORE, 94458369826816, 94458369835007, ++STORE, 94458369835008, 94458369847295, ++STORE, 94458393292800, 94458400366591, ++STORE, 140619773841408, 140619775500287, ++STORE, 140619775500288, 140619777597439, ++STORE, 140619777597440, 140619777613823, ++STORE, 140619777613824, 140619777622015, ++STORE, 140619777622016, 140619777638399, ++STORE, 140619777638400, 140619777650687, ++STORE, 140619777650688, 140619779743743, ++STORE, 140619779743744, 140619779747839, ++STORE, 140619779747840, 140619779751935, ++STORE, 140619779751936, 140619779895295, ++STORE, 140619780263936, 140619781947391, ++STORE, 140619781947392, 140619781963775, ++STORE, 140619781992448, 140619781996543, ++STORE, 140619781996544, 140619782000639, ++STORE, 140619782000640, 140619782004735, ++STORE, 140725811675136, 140725811814399, ++STORE, 140725812813824, 140725812826111, ++STORE, 140725812826112, 140725812830207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728740679680, 140737488351231, ++SNULL, 140728740687871, 140737488351231, ++STORE, 140728740679680, 140728740687871, ++STORE, 140728740548608, 140728740687871, ++STORE, 94764075249664, 94764077473791, ++SNULL, 94764075360255, 94764077473791, ++STORE, 94764075249664, 94764075360255, ++STORE, 94764075360256, 94764077473791, ++ERASE, 94764075360256, 94764077473791, ++STORE, 94764077453312, 94764077465599, ++STORE, 94764077465600, 94764077473791, ++STORE, 139766406791168, 139766409043967, ++SNULL, 139766406934527, 139766409043967, ++STORE, 139766406791168, 139766406934527, ++STORE, 139766406934528, 139766409043967, ++ERASE, 139766406934528, 139766409043967, ++STORE, 139766409031680, 139766409039871, ++STORE, 139766409039872, 139766409043967, ++STORE, 140728740913152, 140728740917247, ++STORE, 140728740900864, 140728740913151, ++STORE, 139766409003008, 139766409031679, ++STORE, 139766408994816, 139766409003007, ++STORE, 139766402994176, 139766406791167, ++SNULL, 139766402994176, 139766404653055, ++STORE, 139766404653056, 139766406791167, ++STORE, 139766402994176, 139766404653055, ++SNULL, 139766406750207, 139766406791167, ++STORE, 139766404653056, 139766406750207, ++STORE, 139766406750208, 139766406791167, ++SNULL, 139766406750208, 139766406774783, ++STORE, 139766406774784, 139766406791167, ++STORE, 139766406750208, 139766406774783, ++ERASE, 139766406750208, 139766406774783, ++STORE, 139766406750208, 139766406774783, ++ERASE, 139766406774784, 139766406791167, ++STORE, 139766406774784, 139766406791167, ++SNULL, 139766406766591, 139766406774783, ++STORE, 139766406750208, 139766406766591, ++STORE, 139766406766592, 139766406774783, ++SNULL, 94764077461503, 94764077465599, ++STORE, 94764077453312, 94764077461503, ++STORE, 94764077461504, 94764077465599, ++SNULL, 139766409035775, 139766409039871, ++STORE, 139766409031680, 139766409035775, ++STORE, 139766409035776, 139766409039871, ++ERASE, 139766409003008, 139766409031679, ++STORE, 94764090458112, 94764090593279, ++STORE, 94758057480192, 94758057590783, ++STORE, 94758059683840, 94758059692031, ++STORE, 94758059692032, 94758059696127, ++STORE, 94758059696128, 94758059704319, ++STORE, 94758083215360, 94758083350527, ++STORE, 139951456772096, 139951458430975, ++STORE, 139951458430976, 139951460528127, ++STORE, 139951460528128, 139951460544511, ++STORE, 139951460544512, 139951460552703, ++STORE, 139951460552704, 139951460569087, ++STORE, 139951460569088, 139951460712447, ++STORE, 139951462772736, 139951462780927, ++STORE, 139951462809600, 139951462813695, ++STORE, 139951462813696, 139951462817791, ++STORE, 139951462817792, 139951462821887, ++STORE, 140734098313216, 140734098452479, ++STORE, 140734098911232, 140734098923519, ++STORE, 140734098923520, 140734098927615, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724904095744, 140737488351231, ++SNULL, 140724904103935, 140737488351231, ++STORE, 140724904095744, 140724904103935, ++STORE, 140724903964672, 140724904103935, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140408497864704, 140408500117503, ++SNULL, 140408498008063, 140408500117503, ++STORE, 140408497864704, 140408498008063, ++STORE, 140408498008064, 140408500117503, ++ERASE, 140408498008064, 140408500117503, ++STORE, 140408500105216, 140408500113407, ++STORE, 140408500113408, 140408500117503, ++STORE, 140724905369600, 140724905373695, ++STORE, 140724905357312, 140724905369599, ++STORE, 140408500076544, 140408500105215, ++STORE, 140408500068352, 140408500076543, ++STORE, 140408494702592, 140408497864703, ++SNULL, 140408494702592, 140408495763455, ++STORE, 140408495763456, 140408497864703, ++STORE, 140408494702592, 140408495763455, ++SNULL, 140408497856511, 140408497864703, ++STORE, 140408495763456, 140408497856511, ++STORE, 140408497856512, 140408497864703, ++ERASE, 140408497856512, 140408497864703, ++STORE, 140408497856512, 140408497864703, ++STORE, 140408490905600, 140408494702591, ++SNULL, 140408490905600, 140408492564479, ++STORE, 140408492564480, 140408494702591, ++STORE, 140408490905600, 140408492564479, ++SNULL, 140408494661631, 140408494702591, ++STORE, 140408492564480, 140408494661631, ++STORE, 140408494661632, 140408494702591, ++SNULL, 140408494661632, 140408494686207, ++STORE, 140408494686208, 140408494702591, ++STORE, 140408494661632, 140408494686207, ++ERASE, 140408494661632, 140408494686207, ++STORE, 140408494661632, 140408494686207, ++ERASE, 140408494686208, 140408494702591, ++STORE, 140408494686208, 140408494702591, ++STORE, 140408500056064, 140408500076543, ++SNULL, 140408494678015, 140408494686207, ++STORE, 140408494661632, 140408494678015, ++STORE, 140408494678016, 140408494686207, ++SNULL, 140408497860607, 140408497864703, ++STORE, 140408497856512, 140408497860607, ++STORE, 140408497860608, 140408497864703, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140408500109311, 140408500113407, ++STORE, 140408500105216, 140408500109311, ++STORE, 140408500109312, 140408500113407, ++ERASE, 140408500076544, 140408500105215, ++STORE, 25235456, 25370623, ++STORE, 25235456, 25518079, ++STORE, 140408498372608, 140408500056063, ++STORE, 94543937388544, 94543937499135, ++STORE, 94543939592192, 94543939600383, ++STORE, 94543939600384, 94543939604479, ++STORE, 94543939604480, 94543939612671, ++STORE, 94543941447680, 94543941582847, ++STORE, 140282621947904, 140282623606783, ++STORE, 140282623606784, 140282625703935, ++STORE, 140282625703936, 140282625720319, ++STORE, 140282625720320, 140282625728511, ++STORE, 140282625728512, 140282625744895, ++STORE, 140282625744896, 140282625888255, ++STORE, 140282627948544, 140282627956735, ++STORE, 140282627985408, 140282627989503, ++STORE, 140282627989504, 140282627993599, ++STORE, 140282627993600, 140282627997695, ++STORE, 140728295723008, 140728295862271, ++STORE, 140728296476672, 140728296488959, ++STORE, 140728296488960, 140728296493055, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431510691839, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431510691839, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140736944451584, 140737488351231, ++SNULL, 140736944463871, 140737488351231, ++STORE, 140736944451584, 140736944463871, ++STORE, 140736944320512, 140736944463871, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 139693609893888, 139693612146687, ++SNULL, 139693610037247, 139693612146687, ++STORE, 139693609893888, 139693610037247, ++STORE, 139693610037248, 139693612146687, ++ERASE, 139693610037248, 139693612146687, ++STORE, 139693612134400, 139693612142591, ++STORE, 139693612142592, 139693612146687, ++STORE, 140736945152000, 140736945156095, ++STORE, 140736945139712, 140736945151999, ++STORE, 139693612105728, 139693612134399, ++STORE, 139693612097536, 139693612105727, ++STORE, 139693606060032, 139693609893887, ++SNULL, 139693606060032, 139693607768063, ++STORE, 139693607768064, 139693609893887, ++STORE, 139693606060032, 139693607768063, ++SNULL, 139693609861119, 139693609893887, ++STORE, 139693607768064, 139693609861119, ++STORE, 139693609861120, 139693609893887, ++ERASE, 139693609861120, 139693609893887, ++STORE, 139693609861120, 139693609893887, ++STORE, 139693603864576, 139693606060031, ++SNULL, 139693603864576, 139693603958783, ++STORE, 139693603958784, 139693606060031, ++STORE, 139693603864576, 139693603958783, ++SNULL, 139693606051839, 139693606060031, ++STORE, 139693603958784, 139693606051839, ++STORE, 139693606051840, 139693606060031, ++ERASE, 139693606051840, 139693606060031, ++STORE, 139693606051840, 139693606060031, ++STORE, 139693601345536, 139693603864575, ++SNULL, 139693601345536, 139693601759231, ++STORE, 139693601759232, 139693603864575, ++STORE, 139693601345536, 139693601759231, ++SNULL, 139693603852287, 139693603864575, ++STORE, 139693601759232, 139693603852287, ++STORE, 139693603852288, 139693603864575, ++ERASE, 139693603852288, 139693603864575, ++STORE, 139693603852288, 139693603864575, ++STORE, 139693598711808, 139693601345535, ++SNULL, 139693598711808, 139693599240191, ++STORE, 139693599240192, 139693601345535, ++STORE, 139693598711808, 139693599240191, ++SNULL, 139693601337343, 139693601345535, ++STORE, 139693599240192, 139693601337343, ++STORE, 139693601337344, 139693601345535, ++ERASE, 139693601337344, 139693601345535, ++STORE, 139693601337344, 139693601345535, ++STORE, 139693596598272, 139693598711807, ++SNULL, 139693596598272, 139693596610559, ++STORE, 139693596610560, 139693598711807, ++STORE, 139693596598272, 139693596610559, ++SNULL, 139693598703615, 139693598711807, ++STORE, 139693596610560, 139693598703615, ++STORE, 139693598703616, 139693598711807, ++ERASE, 139693598703616, 139693598711807, ++STORE, 139693598703616, 139693598711807, ++STORE, 139693594394624, 139693596598271, ++SNULL, 139693594394624, 139693594497023, ++STORE, 139693594497024, 139693596598271, ++STORE, 139693594394624, 139693594497023, ++SNULL, 139693596590079, 139693596598271, ++STORE, 139693594497024, 139693596590079, ++STORE, 139693596590080, 139693596598271, ++ERASE, 139693596590080, 139693596598271, ++STORE, 139693596590080, 139693596598271, ++STORE, 139693612089344, 139693612105727, ++STORE, 139693591232512, 139693594394623, ++SNULL, 139693591232512, 139693592293375, ++STORE, 139693592293376, 139693594394623, ++STORE, 139693591232512, 139693592293375, ++SNULL, 139693594386431, 139693594394623, ++STORE, 139693592293376, 139693594386431, ++STORE, 139693594386432, 139693594394623, ++ERASE, 139693594386432, 139693594394623, ++STORE, 139693594386432, 139693594394623, ++STORE, 139693587435520, 139693591232511, ++SNULL, 139693587435520, 139693589094399, ++STORE, 139693589094400, 139693591232511, ++STORE, 139693587435520, 139693589094399, ++SNULL, 139693591191551, 139693591232511, ++STORE, 139693589094400, 139693591191551, ++STORE, 139693591191552, 139693591232511, ++SNULL, 139693591191552, 139693591216127, ++STORE, 139693591216128, 139693591232511, ++STORE, 139693591191552, 139693591216127, ++ERASE, 139693591191552, 139693591216127, ++STORE, 139693591191552, 139693591216127, ++ERASE, 139693591216128, 139693591232511, ++STORE, 139693591216128, 139693591232511, ++STORE, 139693612077056, 139693612105727, ++SNULL, 139693591207935, 139693591216127, ++STORE, 139693591191552, 139693591207935, ++STORE, 139693591207936, 139693591216127, ++SNULL, 139693594390527, 139693594394623, ++STORE, 139693594386432, 139693594390527, ++STORE, 139693594390528, 139693594394623, ++SNULL, 139693596594175, 139693596598271, ++STORE, 139693596590080, 139693596594175, ++STORE, 139693596594176, 139693596598271, ++SNULL, 139693598707711, 139693598711807, ++STORE, 139693598703616, 139693598707711, ++STORE, 139693598707712, 139693598711807, ++SNULL, 139693601341439, 139693601345535, ++STORE, 139693601337344, 139693601341439, ++STORE, 139693601341440, 139693601345535, ++SNULL, 139693603860479, 139693603864575, ++STORE, 139693603852288, 139693603860479, ++STORE, 139693603860480, 139693603864575, ++SNULL, 139693606055935, 139693606060031, ++STORE, 139693606051840, 139693606055935, ++STORE, 139693606055936, 139693606060031, ++SNULL, 139693609865215, 139693609893887, ++STORE, 139693609861120, 139693609865215, ++STORE, 139693609865216, 139693609893887, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 139693612138495, 139693612142591, ++STORE, 139693612134400, 139693612138495, ++STORE, 139693612138496, 139693612142591, ++ERASE, 139693612105728, 139693612134399, ++STORE, 39976960, 40112127, ++STORE, 139693610393600, 139693612077055, ++STORE, 139693612130304, 139693612134399, ++STORE, 139693610258432, 139693610393599, ++STORE, 39976960, 40255487, ++STORE, 139693585338368, 139693587435519, ++STORE, 139693612122112, 139693612134399, ++STORE, 139693612113920, 139693612134399, ++STORE, 139693612077056, 139693612113919, ++STORE, 139693610242048, 139693610393599, ++STORE, 39976960, 40390655, ++STORE, 39976960, 40546303, ++STORE, 139693610233856, 139693610393599, ++STORE, 139693610225664, 139693610393599, ++STORE, 39976960, 40714239, ++STORE, 139693610209280, 139693610393599, ++STORE, 39976960, 40861695, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431528759295, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140729993904128, 140737488351231, ++SNULL, 140729993912319, 140737488351231, ++STORE, 140729993904128, 140729993912319, ++STORE, 140729993773056, 140729993912319, ++STORE, 93926271991808, 93926274215935, ++SNULL, 93926272102399, 93926274215935, ++STORE, 93926271991808, 93926272102399, ++STORE, 93926272102400, 93926274215935, ++ERASE, 93926272102400, 93926274215935, ++STORE, 93926274195456, 93926274207743, ++STORE, 93926274207744, 93926274215935, ++STORE, 139962167296000, 139962169548799, ++SNULL, 139962167439359, 139962169548799, ++STORE, 139962167296000, 139962167439359, ++STORE, 139962167439360, 139962169548799, ++ERASE, 139962167439360, 139962169548799, ++STORE, 139962169536512, 139962169544703, ++STORE, 139962169544704, 139962169548799, ++STORE, 140729995096064, 140729995100159, ++STORE, 140729995083776, 140729995096063, ++STORE, 139962169507840, 139962169536511, ++STORE, 139962169499648, 139962169507839, ++STORE, 139962163499008, 139962167295999, ++SNULL, 139962163499008, 139962165157887, ++STORE, 139962165157888, 139962167295999, ++STORE, 139962163499008, 139962165157887, ++SNULL, 139962167255039, 139962167295999, ++STORE, 139962165157888, 139962167255039, ++STORE, 139962167255040, 139962167295999, ++SNULL, 139962167255040, 139962167279615, ++STORE, 139962167279616, 139962167295999, ++STORE, 139962167255040, 139962167279615, ++ERASE, 139962167255040, 139962167279615, ++STORE, 139962167255040, 139962167279615, ++ERASE, 139962167279616, 139962167295999, ++STORE, 139962167279616, 139962167295999, ++SNULL, 139962167271423, 139962167279615, ++STORE, 139962167255040, 139962167271423, ++STORE, 139962167271424, 139962167279615, ++SNULL, 93926274203647, 93926274207743, ++STORE, 93926274195456, 93926274203647, ++STORE, 93926274203648, 93926274207743, ++SNULL, 139962169540607, 139962169544703, ++STORE, 139962169536512, 139962169540607, ++STORE, 139962169540608, 139962169544703, ++ERASE, 139962169507840, 139962169536511, ++STORE, 93926291120128, 93926291255295, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724960579584, 140737488351231, ++SNULL, 140724960587775, 140737488351231, ++STORE, 140724960579584, 140724960587775, ++STORE, 140724960448512, 140724960587775, ++STORE, 94246489489408, 94246491713535, ++SNULL, 94246489599999, 94246491713535, ++STORE, 94246489489408, 94246489599999, ++STORE, 94246489600000, 94246491713535, ++ERASE, 94246489600000, 94246491713535, ++STORE, 94246491693056, 94246491705343, ++STORE, 94246491705344, 94246491713535, ++STORE, 140098174926848, 140098177179647, ++SNULL, 140098175070207, 140098177179647, ++STORE, 140098174926848, 140098175070207, ++STORE, 140098175070208, 140098177179647, ++ERASE, 140098175070208, 140098177179647, ++STORE, 140098177167360, 140098177175551, ++STORE, 140098177175552, 140098177179647, ++STORE, 140724961439744, 140724961443839, ++STORE, 140724961427456, 140724961439743, ++STORE, 140098177138688, 140098177167359, ++STORE, 140098177130496, 140098177138687, ++STORE, 140098171129856, 140098174926847, ++SNULL, 140098171129856, 140098172788735, ++STORE, 140098172788736, 140098174926847, ++STORE, 140098171129856, 140098172788735, ++SNULL, 140098174885887, 140098174926847, ++STORE, 140098172788736, 140098174885887, ++STORE, 140098174885888, 140098174926847, ++SNULL, 140098174885888, 140098174910463, ++STORE, 140098174910464, 140098174926847, ++STORE, 140098174885888, 140098174910463, ++ERASE, 140098174885888, 140098174910463, ++STORE, 140098174885888, 140098174910463, ++ERASE, 140098174910464, 140098174926847, ++STORE, 140098174910464, 140098174926847, ++SNULL, 140098174902271, 140098174910463, ++STORE, 140098174885888, 140098174902271, ++STORE, 140098174902272, 140098174910463, ++SNULL, 94246491701247, 94246491705343, ++STORE, 94246491693056, 94246491701247, ++STORE, 94246491701248, 94246491705343, ++SNULL, 140098177171455, 140098177175551, ++STORE, 140098177167360, 140098177171455, ++STORE, 140098177171456, 140098177175551, ++ERASE, 140098177138688, 140098177167359, ++STORE, 94246516998144, 94246517133311, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730522918912, 140737488351231, ++SNULL, 140730522927103, 140737488351231, ++STORE, 140730522918912, 140730522927103, ++STORE, 140730522787840, 140730522927103, ++STORE, 94196043120640, 94196045344767, ++SNULL, 94196043231231, 94196045344767, ++STORE, 94196043120640, 94196043231231, ++STORE, 94196043231232, 94196045344767, ++ERASE, 94196043231232, 94196045344767, ++STORE, 94196045324288, 94196045336575, ++STORE, 94196045336576, 94196045344767, ++STORE, 139815918940160, 139815921192959, ++SNULL, 139815919083519, 139815921192959, ++STORE, 139815918940160, 139815919083519, ++STORE, 139815919083520, 139815921192959, ++ERASE, 139815919083520, 139815921192959, ++STORE, 139815921180672, 139815921188863, ++STORE, 139815921188864, 139815921192959, ++STORE, 140730523344896, 140730523348991, ++STORE, 140730523332608, 140730523344895, ++STORE, 139815921152000, 139815921180671, ++STORE, 139815921143808, 139815921151999, ++STORE, 139815915143168, 139815918940159, ++SNULL, 139815915143168, 139815916802047, ++STORE, 139815916802048, 139815918940159, ++STORE, 139815915143168, 139815916802047, ++SNULL, 139815918899199, 139815918940159, ++STORE, 139815916802048, 139815918899199, ++STORE, 139815918899200, 139815918940159, ++SNULL, 139815918899200, 139815918923775, ++STORE, 139815918923776, 139815918940159, ++STORE, 139815918899200, 139815918923775, ++ERASE, 139815918899200, 139815918923775, ++STORE, 139815918899200, 139815918923775, ++ERASE, 139815918923776, 139815918940159, ++STORE, 139815918923776, 139815918940159, ++SNULL, 139815918915583, 139815918923775, ++STORE, 139815918899200, 139815918915583, ++STORE, 139815918915584, 139815918923775, ++SNULL, 94196045332479, 94196045336575, ++STORE, 94196045324288, 94196045332479, ++STORE, 94196045332480, 94196045336575, ++SNULL, 139815921184767, 139815921188863, ++STORE, 139815921180672, 139815921184767, ++STORE, 139815921184768, 139815921188863, ++ERASE, 139815921152000, 139815921180671, ++STORE, 94196076183552, 94196076318719, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722460393472, 140737488351231, ++SNULL, 140722460401663, 140737488351231, ++STORE, 140722460393472, 140722460401663, ++STORE, 140722460262400, 140722460401663, ++STORE, 94569810399232, 94569812623359, ++SNULL, 94569810509823, 94569812623359, ++STORE, 94569810399232, 94569810509823, ++STORE, 94569810509824, 94569812623359, ++ERASE, 94569810509824, 94569812623359, ++STORE, 94569812602880, 94569812615167, ++STORE, 94569812615168, 94569812623359, ++STORE, 139681565450240, 139681567703039, ++SNULL, 139681565593599, 139681567703039, ++STORE, 139681565450240, 139681565593599, ++STORE, 139681565593600, 139681567703039, ++ERASE, 139681565593600, 139681567703039, ++STORE, 139681567690752, 139681567698943, ++STORE, 139681567698944, 139681567703039, ++STORE, 140722460569600, 140722460573695, ++STORE, 140722460557312, 140722460569599, ++STORE, 139681567662080, 139681567690751, ++STORE, 139681567653888, 139681567662079, ++STORE, 139681561653248, 139681565450239, ++SNULL, 139681561653248, 139681563312127, ++STORE, 139681563312128, 139681565450239, ++STORE, 139681561653248, 139681563312127, ++SNULL, 139681565409279, 139681565450239, ++STORE, 139681563312128, 139681565409279, ++STORE, 139681565409280, 139681565450239, ++SNULL, 139681565409280, 139681565433855, ++STORE, 139681565433856, 139681565450239, ++STORE, 139681565409280, 139681565433855, ++ERASE, 139681565409280, 139681565433855, ++STORE, 139681565409280, 139681565433855, ++ERASE, 139681565433856, 139681565450239, ++STORE, 139681565433856, 139681565450239, ++SNULL, 139681565425663, 139681565433855, ++STORE, 139681565409280, 139681565425663, ++STORE, 139681565425664, 139681565433855, ++SNULL, 94569812611071, 94569812615167, ++STORE, 94569812602880, 94569812611071, ++STORE, 94569812611072, 94569812615167, ++SNULL, 139681567694847, 139681567698943, ++STORE, 139681567690752, 139681567694847, ++STORE, 139681567694848, 139681567698943, ++ERASE, 139681567662080, 139681567690751, ++STORE, 94569818066944, 94569818202111, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431534280703, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725452365824, 140737488351231, ++SNULL, 140725452374015, 140737488351231, ++STORE, 140725452365824, 140725452374015, ++STORE, 140725452234752, 140725452374015, ++STORE, 94395067465728, 94395069689855, ++SNULL, 94395067576319, 94395069689855, ++STORE, 94395067465728, 94395067576319, ++STORE, 94395067576320, 94395069689855, ++ERASE, 94395067576320, 94395069689855, ++STORE, 94395069669376, 94395069681663, ++STORE, 94395069681664, 94395069689855, ++STORE, 140269941211136, 140269943463935, ++SNULL, 140269941354495, 140269943463935, ++STORE, 140269941211136, 140269941354495, ++STORE, 140269941354496, 140269943463935, ++ERASE, 140269941354496, 140269943463935, ++STORE, 140269943451648, 140269943459839, ++STORE, 140269943459840, 140269943463935, ++STORE, 140725452558336, 140725452562431, ++STORE, 140725452546048, 140725452558335, ++STORE, 140269943422976, 140269943451647, ++STORE, 140269943414784, 140269943422975, ++STORE, 140269937414144, 140269941211135, ++SNULL, 140269937414144, 140269939073023, ++STORE, 140269939073024, 140269941211135, ++STORE, 140269937414144, 140269939073023, ++SNULL, 140269941170175, 140269941211135, ++STORE, 140269939073024, 140269941170175, ++STORE, 140269941170176, 140269941211135, ++SNULL, 140269941170176, 140269941194751, ++STORE, 140269941194752, 140269941211135, ++STORE, 140269941170176, 140269941194751, ++ERASE, 140269941170176, 140269941194751, ++STORE, 140269941170176, 140269941194751, ++ERASE, 140269941194752, 140269941211135, ++STORE, 140269941194752, 140269941211135, ++SNULL, 140269941186559, 140269941194751, ++STORE, 140269941170176, 140269941186559, ++STORE, 140269941186560, 140269941194751, ++SNULL, 94395069677567, 94395069681663, ++STORE, 94395069669376, 94395069677567, ++STORE, 94395069677568, 94395069681663, ++SNULL, 140269943455743, 140269943459839, ++STORE, 140269943451648, 140269943455743, ++STORE, 140269943455744, 140269943459839, ++ERASE, 140269943422976, 140269943451647, ++STORE, 94395101691904, 94395101827071, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733860118528, 140737488351231, ++SNULL, 140733860126719, 140737488351231, ++STORE, 140733860118528, 140733860126719, ++STORE, 140733859987456, 140733860126719, ++STORE, 94484752990208, 94484755214335, ++SNULL, 94484753100799, 94484755214335, ++STORE, 94484752990208, 94484753100799, ++STORE, 94484753100800, 94484755214335, ++ERASE, 94484753100800, 94484755214335, ++STORE, 94484755193856, 94484755206143, ++STORE, 94484755206144, 94484755214335, ++STORE, 139958922309632, 139958924562431, ++SNULL, 139958922452991, 139958924562431, ++STORE, 139958922309632, 139958922452991, ++STORE, 139958922452992, 139958924562431, ++ERASE, 139958922452992, 139958924562431, ++STORE, 139958924550144, 139958924558335, ++STORE, 139958924558336, 139958924562431, ++STORE, 140733860253696, 140733860257791, ++STORE, 140733860241408, 140733860253695, ++STORE, 139958924521472, 139958924550143, ++STORE, 139958924513280, 139958924521471, ++STORE, 139958918512640, 139958922309631, ++SNULL, 139958918512640, 139958920171519, ++STORE, 139958920171520, 139958922309631, ++STORE, 139958918512640, 139958920171519, ++SNULL, 139958922268671, 139958922309631, ++STORE, 139958920171520, 139958922268671, ++STORE, 139958922268672, 139958922309631, ++SNULL, 139958922268672, 139958922293247, ++STORE, 139958922293248, 139958922309631, ++STORE, 139958922268672, 139958922293247, ++ERASE, 139958922268672, 139958922293247, ++STORE, 139958922268672, 139958922293247, ++ERASE, 139958922293248, 139958922309631, ++STORE, 139958922293248, 139958922309631, ++SNULL, 139958922285055, 139958922293247, ++STORE, 139958922268672, 139958922285055, ++STORE, 139958922285056, 139958922293247, ++SNULL, 94484755202047, 94484755206143, ++STORE, 94484755193856, 94484755202047, ++STORE, 94484755202048, 94484755206143, ++SNULL, 139958924554239, 139958924558335, ++STORE, 139958924550144, 139958924554239, ++STORE, 139958924554240, 139958924558335, ++ERASE, 139958924521472, 139958924550143, ++STORE, 94484777615360, 94484777750527, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731051036672, 140737488351231, ++SNULL, 140731051044863, 140737488351231, ++STORE, 140731051036672, 140731051044863, ++STORE, 140731050905600, 140731051044863, ++STORE, 93945822998528, 93945825222655, ++SNULL, 93945823109119, 93945825222655, ++STORE, 93945822998528, 93945823109119, ++STORE, 93945823109120, 93945825222655, ++ERASE, 93945823109120, 93945825222655, ++STORE, 93945825202176, 93945825214463, ++STORE, 93945825214464, 93945825222655, ++STORE, 140153503997952, 140153506250751, ++SNULL, 140153504141311, 140153506250751, ++STORE, 140153503997952, 140153504141311, ++STORE, 140153504141312, 140153506250751, ++ERASE, 140153504141312, 140153506250751, ++STORE, 140153506238464, 140153506246655, ++STORE, 140153506246656, 140153506250751, ++STORE, 140731051331584, 140731051335679, ++STORE, 140731051319296, 140731051331583, ++STORE, 140153506209792, 140153506238463, ++STORE, 140153506201600, 140153506209791, ++STORE, 140153500200960, 140153503997951, ++SNULL, 140153500200960, 140153501859839, ++STORE, 140153501859840, 140153503997951, ++STORE, 140153500200960, 140153501859839, ++SNULL, 140153503956991, 140153503997951, ++STORE, 140153501859840, 140153503956991, ++STORE, 140153503956992, 140153503997951, ++SNULL, 140153503956992, 140153503981567, ++STORE, 140153503981568, 140153503997951, ++STORE, 140153503956992, 140153503981567, ++ERASE, 140153503956992, 140153503981567, ++STORE, 140153503956992, 140153503981567, ++ERASE, 140153503981568, 140153503997951, ++STORE, 140153503981568, 140153503997951, ++SNULL, 140153503973375, 140153503981567, ++STORE, 140153503956992, 140153503973375, ++STORE, 140153503973376, 140153503981567, ++SNULL, 93945825210367, 93945825214463, ++STORE, 93945825202176, 93945825210367, ++STORE, 93945825210368, 93945825214463, ++SNULL, 140153506242559, 140153506246655, ++STORE, 140153506238464, 140153506242559, ++STORE, 140153506242560, 140153506246655, ++ERASE, 140153506209792, 140153506238463, ++STORE, 93945854537728, 93945854672895, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431537885183, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736025325568, 140737488351231, ++SNULL, 140736025333759, 140737488351231, ++STORE, 140736025325568, 140736025333759, ++STORE, 140736025194496, 140736025333759, ++STORE, 94809095172096, 94809097396223, ++SNULL, 94809095282687, 94809097396223, ++STORE, 94809095172096, 94809095282687, ++STORE, 94809095282688, 94809097396223, ++ERASE, 94809095282688, 94809097396223, ++STORE, 94809097375744, 94809097388031, ++STORE, 94809097388032, 94809097396223, ++STORE, 140194992517120, 140194994769919, ++SNULL, 140194992660479, 140194994769919, ++STORE, 140194992517120, 140194992660479, ++STORE, 140194992660480, 140194994769919, ++ERASE, 140194992660480, 140194994769919, ++STORE, 140194994757632, 140194994765823, ++STORE, 140194994765824, 140194994769919, ++STORE, 140736026173440, 140736026177535, ++STORE, 140736026161152, 140736026173439, ++STORE, 140194994728960, 140194994757631, ++STORE, 140194994720768, 140194994728959, ++STORE, 140194988720128, 140194992517119, ++SNULL, 140194988720128, 140194990379007, ++STORE, 140194990379008, 140194992517119, ++STORE, 140194988720128, 140194990379007, ++SNULL, 140194992476159, 140194992517119, ++STORE, 140194990379008, 140194992476159, ++STORE, 140194992476160, 140194992517119, ++SNULL, 140194992476160, 140194992500735, ++STORE, 140194992500736, 140194992517119, ++STORE, 140194992476160, 140194992500735, ++ERASE, 140194992476160, 140194992500735, ++STORE, 140194992476160, 140194992500735, ++ERASE, 140194992500736, 140194992517119, ++STORE, 140194992500736, 140194992517119, ++SNULL, 140194992492543, 140194992500735, ++STORE, 140194992476160, 140194992492543, ++STORE, 140194992492544, 140194992500735, ++SNULL, 94809097383935, 94809097388031, ++STORE, 94809097375744, 94809097383935, ++STORE, 94809097383936, 94809097388031, ++SNULL, 140194994761727, 140194994765823, ++STORE, 140194994757632, 140194994761727, ++STORE, 140194994761728, 140194994765823, ++ERASE, 140194994728960, 140194994757631, ++STORE, 94809124286464, 94809124421631, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726342660096, 140737488351231, ++SNULL, 140726342668287, 140737488351231, ++STORE, 140726342660096, 140726342668287, ++STORE, 140726342529024, 140726342668287, ++STORE, 94140331462656, 94140333686783, ++SNULL, 94140331573247, 94140333686783, ++STORE, 94140331462656, 94140331573247, ++STORE, 94140331573248, 94140333686783, ++ERASE, 94140331573248, 94140333686783, ++STORE, 94140333666304, 94140333678591, ++STORE, 94140333678592, 94140333686783, ++STORE, 140714077208576, 140714079461375, ++SNULL, 140714077351935, 140714079461375, ++STORE, 140714077208576, 140714077351935, ++STORE, 140714077351936, 140714079461375, ++ERASE, 140714077351936, 140714079461375, ++STORE, 140714079449088, 140714079457279, ++STORE, 140714079457280, 140714079461375, ++STORE, 140726343933952, 140726343938047, ++STORE, 140726343921664, 140726343933951, ++STORE, 140714079420416, 140714079449087, ++STORE, 140714079412224, 140714079420415, ++STORE, 140714073411584, 140714077208575, ++SNULL, 140714073411584, 140714075070463, ++STORE, 140714075070464, 140714077208575, ++STORE, 140714073411584, 140714075070463, ++SNULL, 140714077167615, 140714077208575, ++STORE, 140714075070464, 140714077167615, ++STORE, 140714077167616, 140714077208575, ++SNULL, 140714077167616, 140714077192191, ++STORE, 140714077192192, 140714077208575, ++STORE, 140714077167616, 140714077192191, ++ERASE, 140714077167616, 140714077192191, ++STORE, 140714077167616, 140714077192191, ++ERASE, 140714077192192, 140714077208575, ++STORE, 140714077192192, 140714077208575, ++SNULL, 140714077183999, 140714077192191, ++STORE, 140714077167616, 140714077183999, ++STORE, 140714077184000, 140714077192191, ++SNULL, 94140333674495, 94140333678591, ++STORE, 94140333666304, 94140333674495, ++STORE, 94140333674496, 94140333678591, ++SNULL, 140714079453183, 140714079457279, ++STORE, 140714079449088, 140714079453183, ++STORE, 140714079453184, 140714079457279, ++ERASE, 140714079420416, 140714079449087, ++STORE, 94140341432320, 94140341567487, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431539601407, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725843607552, 140737488351231, ++SNULL, 140725843615743, 140737488351231, ++STORE, 140725843607552, 140725843615743, ++STORE, 140725843476480, 140725843615743, ++STORE, 94889043505152, 94889045839871, ++SNULL, 94889043718143, 94889045839871, ++STORE, 94889043505152, 94889043718143, ++STORE, 94889043718144, 94889045839871, ++ERASE, 94889043718144, 94889045839871, ++STORE, 94889045815296, 94889045827583, ++STORE, 94889045827584, 94889045839871, ++STORE, 140250965946368, 140250968199167, ++SNULL, 140250966089727, 140250968199167, ++STORE, 140250965946368, 140250966089727, ++STORE, 140250966089728, 140250968199167, ++ERASE, 140250966089728, 140250968199167, ++STORE, 140250968186880, 140250968195071, ++STORE, 140250968195072, 140250968199167, ++STORE, 140725844500480, 140725844504575, ++STORE, 140725844488192, 140725844500479, ++STORE, 140250968158208, 140250968186879, ++STORE, 140250968150016, 140250968158207, ++STORE, 140250963832832, 140250965946367, ++SNULL, 140250963832832, 140250963845119, ++STORE, 140250963845120, 140250965946367, ++STORE, 140250963832832, 140250963845119, ++SNULL, 140250965938175, 140250965946367, ++STORE, 140250963845120, 140250965938175, ++STORE, 140250965938176, 140250965946367, ++ERASE, 140250965938176, 140250965946367, ++STORE, 140250965938176, 140250965946367, ++STORE, 140250960035840, 140250963832831, ++SNULL, 140250960035840, 140250961694719, ++STORE, 140250961694720, 140250963832831, ++STORE, 140250960035840, 140250961694719, ++SNULL, 140250963791871, 140250963832831, ++STORE, 140250961694720, 140250963791871, ++STORE, 140250963791872, 140250963832831, ++SNULL, 140250963791872, 140250963816447, ++STORE, 140250963816448, 140250963832831, ++STORE, 140250963791872, 140250963816447, ++ERASE, 140250963791872, 140250963816447, ++STORE, 140250963791872, 140250963816447, ++ERASE, 140250963816448, 140250963832831, ++STORE, 140250963816448, 140250963832831, ++STORE, 140250968141824, 140250968158207, ++SNULL, 140250963808255, 140250963816447, ++STORE, 140250963791872, 140250963808255, ++STORE, 140250963808256, 140250963816447, ++SNULL, 140250965942271, 140250965946367, ++STORE, 140250965938176, 140250965942271, ++STORE, 140250965942272, 140250965946367, ++SNULL, 94889045819391, 94889045827583, ++STORE, 94889045815296, 94889045819391, ++STORE, 94889045819392, 94889045827583, ++SNULL, 140250968190975, 140250968195071, ++STORE, 140250968186880, 140250968190975, ++STORE, 140250968190976, 140250968195071, ++ERASE, 140250968158208, 140250968186879, ++STORE, 94889052213248, 94889052348415, ++STORE, 140250966458368, 140250968141823, ++STORE, 94889052213248, 94889052483583, ++STORE, 94889052213248, 94889052618751, ++STORE, 94170851819520, 94170852032511, ++STORE, 94170854129664, 94170854133759, ++STORE, 94170854133760, 94170854141951, ++STORE, 94170854141952, 94170854154239, ++STORE, 94170866515968, 94170867740671, ++STORE, 140062030422016, 140062032080895, ++STORE, 140062032080896, 140062034178047, ++STORE, 140062034178048, 140062034194431, ++STORE, 140062034194432, 140062034202623, ++STORE, 140062034202624, 140062034219007, ++STORE, 140062034219008, 140062034231295, ++STORE, 140062034231296, 140062036324351, ++STORE, 140062036324352, 140062036328447, ++STORE, 140062036328448, 140062036332543, ++STORE, 140062036332544, 140062036475903, ++STORE, 140062036844544, 140062038527999, ++STORE, 140062038528000, 140062038544383, ++STORE, 140062038573056, 140062038577151, ++STORE, 140062038577152, 140062038581247, ++STORE, 140062038581248, 140062038585343, ++STORE, 140736210550784, 140736210690047, ++STORE, 140736210759680, 140736210771967, ++STORE, 140736210771968, 140736210776063, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724272365568, 140737488351231, ++SNULL, 140724272373759, 140737488351231, ++STORE, 140724272365568, 140724272373759, ++STORE, 140724272234496, 140724272373759, ++STORE, 94607711965184, 94607714189311, ++SNULL, 94607712075775, 94607714189311, ++STORE, 94607711965184, 94607712075775, ++STORE, 94607712075776, 94607714189311, ++ERASE, 94607712075776, 94607714189311, ++STORE, 94607714168832, 94607714181119, ++STORE, 94607714181120, 94607714189311, ++STORE, 140054949253120, 140054951505919, ++SNULL, 140054949396479, 140054951505919, ++STORE, 140054949253120, 140054949396479, ++STORE, 140054949396480, 140054951505919, ++ERASE, 140054949396480, 140054951505919, ++STORE, 140054951493632, 140054951501823, ++STORE, 140054951501824, 140054951505919, ++STORE, 140724272992256, 140724272996351, ++STORE, 140724272979968, 140724272992255, ++STORE, 140054951464960, 140054951493631, ++STORE, 140054951456768, 140054951464959, ++STORE, 140054945456128, 140054949253119, ++SNULL, 140054945456128, 140054947115007, ++STORE, 140054947115008, 140054949253119, ++STORE, 140054945456128, 140054947115007, ++SNULL, 140054949212159, 140054949253119, ++STORE, 140054947115008, 140054949212159, ++STORE, 140054949212160, 140054949253119, ++SNULL, 140054949212160, 140054949236735, ++STORE, 140054949236736, 140054949253119, ++STORE, 140054949212160, 140054949236735, ++ERASE, 140054949212160, 140054949236735, ++STORE, 140054949212160, 140054949236735, ++ERASE, 140054949236736, 140054949253119, ++STORE, 140054949236736, 140054949253119, ++SNULL, 140054949228543, 140054949236735, ++STORE, 140054949212160, 140054949228543, ++STORE, 140054949228544, 140054949236735, ++SNULL, 94607714177023, 94607714181119, ++STORE, 94607714168832, 94607714177023, ++STORE, 94607714177024, 94607714181119, ++SNULL, 140054951497727, 140054951501823, ++STORE, 140054951493632, 140054951497727, ++STORE, 140054951497728, 140054951501823, ++ERASE, 140054951464960, 140054951493631, ++STORE, 94607733374976, 94607733510143, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733586923520, 140737488351231, ++SNULL, 140733586931711, 140737488351231, ++STORE, 140733586923520, 140733586931711, ++STORE, 140733586792448, 140733586931711, ++STORE, 93901634904064, 93901637128191, ++SNULL, 93901635014655, 93901637128191, ++STORE, 93901634904064, 93901635014655, ++STORE, 93901635014656, 93901637128191, ++ERASE, 93901635014656, 93901637128191, ++STORE, 93901637107712, 93901637119999, ++STORE, 93901637120000, 93901637128191, ++STORE, 140086104784896, 140086107037695, ++SNULL, 140086104928255, 140086107037695, ++STORE, 140086104784896, 140086104928255, ++STORE, 140086104928256, 140086107037695, ++ERASE, 140086104928256, 140086107037695, ++STORE, 140086107025408, 140086107033599, ++STORE, 140086107033600, 140086107037695, ++STORE, 140733587263488, 140733587267583, ++STORE, 140733587251200, 140733587263487, ++STORE, 140086106996736, 140086107025407, ++STORE, 140086106988544, 140086106996735, ++STORE, 140086100987904, 140086104784895, ++SNULL, 140086100987904, 140086102646783, ++STORE, 140086102646784, 140086104784895, ++STORE, 140086100987904, 140086102646783, ++SNULL, 140086104743935, 140086104784895, ++STORE, 140086102646784, 140086104743935, ++STORE, 140086104743936, 140086104784895, ++SNULL, 140086104743936, 140086104768511, ++STORE, 140086104768512, 140086104784895, ++STORE, 140086104743936, 140086104768511, ++ERASE, 140086104743936, 140086104768511, ++STORE, 140086104743936, 140086104768511, ++ERASE, 140086104768512, 140086104784895, ++STORE, 140086104768512, 140086104784895, ++SNULL, 140086104760319, 140086104768511, ++STORE, 140086104743936, 140086104760319, ++STORE, 140086104760320, 140086104768511, ++SNULL, 93901637115903, 93901637119999, ++STORE, 93901637107712, 93901637115903, ++STORE, 93901637115904, 93901637119999, ++SNULL, 140086107029503, 140086107033599, ++STORE, 140086107025408, 140086107029503, ++STORE, 140086107029504, 140086107033599, ++ERASE, 140086106996736, 140086107025407, ++STORE, 93901662715904, 93901662851071, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723365613568, 140737488351231, ++SNULL, 140723365621759, 140737488351231, ++STORE, 140723365613568, 140723365621759, ++STORE, 140723365482496, 140723365621759, ++STORE, 94759193546752, 94759195770879, ++SNULL, 94759193657343, 94759195770879, ++STORE, 94759193546752, 94759193657343, ++STORE, 94759193657344, 94759195770879, ++ERASE, 94759193657344, 94759195770879, ++STORE, 94759195750400, 94759195762687, ++STORE, 94759195762688, 94759195770879, ++STORE, 140607636246528, 140607638499327, ++SNULL, 140607636389887, 140607638499327, ++STORE, 140607636246528, 140607636389887, ++STORE, 140607636389888, 140607638499327, ++ERASE, 140607636389888, 140607638499327, ++STORE, 140607638487040, 140607638495231, ++STORE, 140607638495232, 140607638499327, ++STORE, 140723365900288, 140723365904383, ++STORE, 140723365888000, 140723365900287, ++STORE, 140607638458368, 140607638487039, ++STORE, 140607638450176, 140607638458367, ++STORE, 140607632449536, 140607636246527, ++SNULL, 140607632449536, 140607634108415, ++STORE, 140607634108416, 140607636246527, ++STORE, 140607632449536, 140607634108415, ++SNULL, 140607636205567, 140607636246527, ++STORE, 140607634108416, 140607636205567, ++STORE, 140607636205568, 140607636246527, ++SNULL, 140607636205568, 140607636230143, ++STORE, 140607636230144, 140607636246527, ++STORE, 140607636205568, 140607636230143, ++ERASE, 140607636205568, 140607636230143, ++STORE, 140607636205568, 140607636230143, ++ERASE, 140607636230144, 140607636246527, ++STORE, 140607636230144, 140607636246527, ++SNULL, 140607636221951, 140607636230143, ++STORE, 140607636205568, 140607636221951, ++STORE, 140607636221952, 140607636230143, ++SNULL, 94759195758591, 94759195762687, ++STORE, 94759195750400, 94759195758591, ++STORE, 94759195758592, 94759195762687, ++SNULL, 140607638491135, 140607638495231, ++STORE, 140607638487040, 140607638491135, ++STORE, 140607638491136, 140607638495231, ++ERASE, 140607638458368, 140607638487039, ++STORE, 94759204995072, 94759205130239, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732503789568, 140737488351231, ++SNULL, 140732503797759, 140737488351231, ++STORE, 140732503789568, 140732503797759, ++STORE, 140732503658496, 140732503797759, ++STORE, 94077792956416, 94077795180543, ++SNULL, 94077793067007, 94077795180543, ++STORE, 94077792956416, 94077793067007, ++STORE, 94077793067008, 94077795180543, ++ERASE, 94077793067008, 94077795180543, ++STORE, 94077795160064, 94077795172351, ++STORE, 94077795172352, 94077795180543, ++STORE, 140359874252800, 140359876505599, ++SNULL, 140359874396159, 140359876505599, ++STORE, 140359874252800, 140359874396159, ++STORE, 140359874396160, 140359876505599, ++ERASE, 140359874396160, 140359876505599, ++STORE, 140359876493312, 140359876501503, ++STORE, 140359876501504, 140359876505599, ++STORE, 140732504465408, 140732504469503, ++STORE, 140732504453120, 140732504465407, ++STORE, 140359876464640, 140359876493311, ++STORE, 140359876456448, 140359876464639, ++STORE, 140359870455808, 140359874252799, ++SNULL, 140359870455808, 140359872114687, ++STORE, 140359872114688, 140359874252799, ++STORE, 140359870455808, 140359872114687, ++SNULL, 140359874211839, 140359874252799, ++STORE, 140359872114688, 140359874211839, ++STORE, 140359874211840, 140359874252799, ++SNULL, 140359874211840, 140359874236415, ++STORE, 140359874236416, 140359874252799, ++STORE, 140359874211840, 140359874236415, ++ERASE, 140359874211840, 140359874236415, ++STORE, 140359874211840, 140359874236415, ++ERASE, 140359874236416, 140359874252799, ++STORE, 140359874236416, 140359874252799, ++SNULL, 140359874228223, 140359874236415, ++STORE, 140359874211840, 140359874228223, ++STORE, 140359874228224, 140359874236415, ++SNULL, 94077795168255, 94077795172351, ++STORE, 94077795160064, 94077795168255, ++STORE, 94077795168256, 94077795172351, ++SNULL, 140359876497407, 140359876501503, ++STORE, 140359876493312, 140359876497407, ++STORE, 140359876497408, 140359876501503, ++ERASE, 140359876464640, 140359876493311, ++STORE, 94077808717824, 94077808852991, ++STORE, 94549486252032, 94549486465023, ++STORE, 94549488562176, 94549488566271, ++STORE, 94549488566272, 94549488574463, ++STORE, 94549488574464, 94549488586751, ++STORE, 94549503492096, 94549506121727, ++STORE, 140085800894464, 140085802553343, ++STORE, 140085802553344, 140085804650495, ++STORE, 140085804650496, 140085804666879, ++STORE, 140085804666880, 140085804675071, ++STORE, 140085804675072, 140085804691455, ++STORE, 140085804691456, 140085804703743, ++STORE, 140085804703744, 140085806796799, ++STORE, 140085806796800, 140085806800895, ++STORE, 140085806800896, 140085806804991, ++STORE, 140085806804992, 140085806948351, ++STORE, 140085807316992, 140085809000447, ++STORE, 140085809000448, 140085809016831, ++STORE, 140085809045504, 140085809049599, ++STORE, 140085809049600, 140085809053695, ++STORE, 140085809053696, 140085809057791, ++STORE, 140731810545664, 140731810684927, ++STORE, 140731810967552, 140731810979839, ++STORE, 140731810979840, 140731810983935, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724752330752, 140737488351231, ++SNULL, 140724752338943, 140737488351231, ++STORE, 140724752330752, 140724752338943, ++STORE, 140724752199680, 140724752338943, ++STORE, 94656357539840, 94656359874559, ++SNULL, 94656357752831, 94656359874559, ++STORE, 94656357539840, 94656357752831, ++STORE, 94656357752832, 94656359874559, ++ERASE, 94656357752832, 94656359874559, ++STORE, 94656359849984, 94656359862271, ++STORE, 94656359862272, 94656359874559, ++STORE, 139632585203712, 139632587456511, ++SNULL, 139632585347071, 139632587456511, ++STORE, 139632585203712, 139632585347071, ++STORE, 139632585347072, 139632587456511, ++ERASE, 139632585347072, 139632587456511, ++STORE, 139632587444224, 139632587452415, ++STORE, 139632587452416, 139632587456511, ++STORE, 139632587440128, 139632587444223, ++STORE, 139632587427840, 139632587440127, ++STORE, 139632587399168, 139632587427839, ++STORE, 139632587390976, 139632587399167, ++STORE, 139632583090176, 139632585203711, ++SNULL, 139632583090176, 139632583102463, ++STORE, 139632583102464, 139632585203711, ++STORE, 139632583090176, 139632583102463, ++SNULL, 139632585195519, 139632585203711, ++STORE, 139632583102464, 139632585195519, ++STORE, 139632585195520, 139632585203711, ++ERASE, 139632585195520, 139632585203711, ++STORE, 139632585195520, 139632585203711, ++STORE, 139632579293184, 139632583090175, ++SNULL, 139632579293184, 139632580952063, ++STORE, 139632580952064, 139632583090175, ++STORE, 139632579293184, 139632580952063, ++SNULL, 139632583049215, 139632583090175, ++STORE, 139632580952064, 139632583049215, ++STORE, 139632583049216, 139632583090175, ++SNULL, 139632583049216, 139632583073791, ++STORE, 139632583073792, 139632583090175, ++STORE, 139632583049216, 139632583073791, ++ERASE, 139632583049216, 139632583073791, ++STORE, 139632583049216, 139632583073791, ++ERASE, 139632583073792, 139632583090175, ++STORE, 139632583073792, 139632583090175, ++STORE, 139632587382784, 139632587399167, ++SNULL, 139632583065599, 139632583073791, ++STORE, 139632583049216, 139632583065599, ++STORE, 139632583065600, 139632583073791, ++SNULL, 139632585199615, 139632585203711, ++STORE, 139632585195520, 139632585199615, ++STORE, 139632585199616, 139632585203711, ++SNULL, 94656359854079, 94656359862271, ++STORE, 94656359849984, 94656359854079, ++STORE, 94656359854080, 94656359862271, ++SNULL, 139632587448319, 139632587452415, ++STORE, 139632587444224, 139632587448319, ++STORE, 139632587448320, 139632587452415, ++ERASE, 139632587399168, 139632587427839, ++STORE, 94656378912768, 94656379047935, ++STORE, 139632585699328, 139632587382783, ++STORE, 94656378912768, 94656379183103, ++STORE, 94656378912768, 94656379318271, ++STORE, 94656378912768, 94656379494399, ++SNULL, 94656379469823, 94656379494399, ++STORE, 94656378912768, 94656379469823, ++STORE, 94656379469824, 94656379494399, ++ERASE, 94656379469824, 94656379494399, ++STORE, 94656378912768, 94656379621375, ++STORE, 94656378912768, 94656379756543, ++STORE, 94656378912768, 94656379912191, ++STORE, 94656378912768, 94656380055551, ++STORE, 94656378912768, 94656380190719, ++STORE, 94656378912768, 94656380338175, ++SNULL, 94656380313599, 94656380338175, ++STORE, 94656378912768, 94656380313599, ++STORE, 94656380313600, 94656380338175, ++ERASE, 94656380313600, 94656380338175, ++STORE, 94656378912768, 94656380448767, ++SNULL, 94656380432383, 94656380448767, ++STORE, 94656378912768, 94656380432383, ++STORE, 94656380432384, 94656380448767, ++ERASE, 94656380432384, 94656380448767, ++STORE, 94656378912768, 94656380567551, ++STORE, 94656378912768, 94656380719103, ++STORE, 94656378912768, 94656380858367, ++STORE, 94656378912768, 94656380997631, ++STORE, 94656378912768, 94656381132799, ++SNULL, 94656381124607, 94656381132799, ++STORE, 94656378912768, 94656381124607, ++STORE, 94656381124608, 94656381132799, ++ERASE, 94656381124608, 94656381132799, ++STORE, 94656378912768, 94656381276159, ++STORE, 94656378912768, 94656381427711, ++STORE, 94604087611392, 94604087824383, ++STORE, 94604089921536, 94604089925631, ++STORE, 94604089925632, 94604089933823, ++STORE, 94604089933824, 94604089946111, ++STORE, 94604105125888, 94604106424319, ++STORE, 140454937694208, 140454939353087, ++STORE, 140454939353088, 140454941450239, ++STORE, 140454941450240, 140454941466623, ++STORE, 140454941466624, 140454941474815, ++STORE, 140454941474816, 140454941491199, ++STORE, 140454941491200, 140454941503487, ++STORE, 140454941503488, 140454943596543, ++STORE, 140454943596544, 140454943600639, ++STORE, 140454943600640, 140454943604735, ++STORE, 140454943604736, 140454943748095, ++STORE, 140454944116736, 140454945800191, ++STORE, 140454945800192, 140454945816575, ++STORE, 140454945845248, 140454945849343, ++STORE, 140454945849344, 140454945853439, ++STORE, 140454945853440, 140454945857535, ++STORE, 140728438214656, 140728438353919, ++STORE, 140728439095296, 140728439107583, ++STORE, 140728439107584, 140728439111679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727821099008, 140737488351231, ++SNULL, 140727821107199, 140737488351231, ++STORE, 140727821099008, 140727821107199, ++STORE, 140727820967936, 140727821107199, ++STORE, 94088457240576, 94088459575295, ++SNULL, 94088457453567, 94088459575295, ++STORE, 94088457240576, 94088457453567, ++STORE, 94088457453568, 94088459575295, ++ERASE, 94088457453568, 94088459575295, ++STORE, 94088459550720, 94088459563007, ++STORE, 94088459563008, 94088459575295, ++STORE, 140234378989568, 140234381242367, ++SNULL, 140234379132927, 140234381242367, ++STORE, 140234378989568, 140234379132927, ++STORE, 140234379132928, 140234381242367, ++ERASE, 140234379132928, 140234381242367, ++STORE, 140234381230080, 140234381238271, ++STORE, 140234381238272, 140234381242367, ++STORE, 140727822077952, 140727822082047, ++STORE, 140727822065664, 140727822077951, ++STORE, 140234381201408, 140234381230079, ++STORE, 140234381193216, 140234381201407, ++STORE, 140234376876032, 140234378989567, ++SNULL, 140234376876032, 140234376888319, ++STORE, 140234376888320, 140234378989567, ++STORE, 140234376876032, 140234376888319, ++SNULL, 140234378981375, 140234378989567, ++STORE, 140234376888320, 140234378981375, ++STORE, 140234378981376, 140234378989567, ++ERASE, 140234378981376, 140234378989567, ++STORE, 140234378981376, 140234378989567, ++STORE, 140234373079040, 140234376876031, ++SNULL, 140234373079040, 140234374737919, ++STORE, 140234374737920, 140234376876031, ++STORE, 140234373079040, 140234374737919, ++SNULL, 140234376835071, 140234376876031, ++STORE, 140234374737920, 140234376835071, ++STORE, 140234376835072, 140234376876031, ++SNULL, 140234376835072, 140234376859647, ++STORE, 140234376859648, 140234376876031, ++STORE, 140234376835072, 140234376859647, ++ERASE, 140234376835072, 140234376859647, ++STORE, 140234376835072, 140234376859647, ++ERASE, 140234376859648, 140234376876031, ++STORE, 140234376859648, 140234376876031, ++STORE, 140234381185024, 140234381201407, ++SNULL, 140234376851455, 140234376859647, ++STORE, 140234376835072, 140234376851455, ++STORE, 140234376851456, 140234376859647, ++SNULL, 140234378985471, 140234378989567, ++STORE, 140234378981376, 140234378985471, ++STORE, 140234378985472, 140234378989567, ++SNULL, 94088459554815, 94088459563007, ++STORE, 94088459550720, 94088459554815, ++STORE, 94088459554816, 94088459563007, ++SNULL, 140234381234175, 140234381238271, ++STORE, 140234381230080, 140234381234175, ++STORE, 140234381234176, 140234381238271, ++ERASE, 140234381201408, 140234381230079, ++STORE, 94088468852736, 94088468987903, ++STORE, 140234379501568, 140234381185023, ++STORE, 94088468852736, 94088469123071, ++STORE, 94088468852736, 94088469258239, ++STORE, 94110050402304, 94110050615295, ++STORE, 94110052712448, 94110052716543, ++STORE, 94110052716544, 94110052724735, ++STORE, 94110052724736, 94110052737023, ++STORE, 94110061875200, 94110062415871, ++STORE, 140139439357952, 140139441016831, ++STORE, 140139441016832, 140139443113983, ++STORE, 140139443113984, 140139443130367, ++STORE, 140139443130368, 140139443138559, ++STORE, 140139443138560, 140139443154943, ++STORE, 140139443154944, 140139443167231, ++STORE, 140139443167232, 140139445260287, ++STORE, 140139445260288, 140139445264383, ++STORE, 140139445264384, 140139445268479, ++STORE, 140139445268480, 140139445411839, ++STORE, 140139445780480, 140139447463935, ++STORE, 140139447463936, 140139447480319, ++STORE, 140139447508992, 140139447513087, ++STORE, 140139447513088, 140139447517183, ++STORE, 140139447517184, 140139447521279, ++STORE, 140731901427712, 140731901566975, ++STORE, 140731902259200, 140731902271487, ++STORE, 140731902271488, 140731902275583, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727282622464, 140737488351231, ++SNULL, 140727282630655, 140737488351231, ++STORE, 140727282622464, 140727282630655, ++STORE, 140727282491392, 140727282630655, ++STORE, 94266649866240, 94266652200959, ++SNULL, 94266650079231, 94266652200959, ++STORE, 94266649866240, 94266650079231, ++STORE, 94266650079232, 94266652200959, ++ERASE, 94266650079232, 94266652200959, ++STORE, 94266652176384, 94266652188671, ++STORE, 94266652188672, 94266652200959, ++STORE, 139888497991680, 139888500244479, ++SNULL, 139888498135039, 139888500244479, ++STORE, 139888497991680, 139888498135039, ++STORE, 139888498135040, 139888500244479, ++ERASE, 139888498135040, 139888500244479, ++STORE, 139888500232192, 139888500240383, ++STORE, 139888500240384, 139888500244479, ++STORE, 140727283113984, 140727283118079, ++STORE, 140727283101696, 140727283113983, ++STORE, 139888500203520, 139888500232191, ++STORE, 139888500195328, 139888500203519, ++STORE, 139888495878144, 139888497991679, ++SNULL, 139888495878144, 139888495890431, ++STORE, 139888495890432, 139888497991679, ++STORE, 139888495878144, 139888495890431, ++SNULL, 139888497983487, 139888497991679, ++STORE, 139888495890432, 139888497983487, ++STORE, 139888497983488, 139888497991679, ++ERASE, 139888497983488, 139888497991679, ++STORE, 139888497983488, 139888497991679, ++STORE, 139888492081152, 139888495878143, ++SNULL, 139888492081152, 139888493740031, ++STORE, 139888493740032, 139888495878143, ++STORE, 139888492081152, 139888493740031, ++SNULL, 139888495837183, 139888495878143, ++STORE, 139888493740032, 139888495837183, ++STORE, 139888495837184, 139888495878143, ++SNULL, 139888495837184, 139888495861759, ++STORE, 139888495861760, 139888495878143, ++STORE, 139888495837184, 139888495861759, ++ERASE, 139888495837184, 139888495861759, ++STORE, 139888495837184, 139888495861759, ++ERASE, 139888495861760, 139888495878143, ++STORE, 139888495861760, 139888495878143, ++STORE, 139888500187136, 139888500203519, ++SNULL, 139888495853567, 139888495861759, ++STORE, 139888495837184, 139888495853567, ++STORE, 139888495853568, 139888495861759, ++SNULL, 139888497987583, 139888497991679, ++STORE, 139888497983488, 139888497987583, ++STORE, 139888497987584, 139888497991679, ++SNULL, 94266652180479, 94266652188671, ++STORE, 94266652176384, 94266652180479, ++STORE, 94266652180480, 94266652188671, ++SNULL, 139888500236287, 139888500240383, ++STORE, 139888500232192, 139888500236287, ++STORE, 139888500236288, 139888500240383, ++ERASE, 139888500203520, 139888500232191, ++STORE, 94266678542336, 94266678677503, ++STORE, 139888498503680, 139888500187135, ++STORE, 94266678542336, 94266678812671, ++STORE, 94266678542336, 94266678947839, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722507702272, 140737488351231, ++SNULL, 140722507710463, 140737488351231, ++STORE, 140722507702272, 140722507710463, ++STORE, 140722507571200, 140722507710463, ++STORE, 94313981394944, 94313983729663, ++SNULL, 94313981607935, 94313983729663, ++STORE, 94313981394944, 94313981607935, ++STORE, 94313981607936, 94313983729663, ++ERASE, 94313981607936, 94313983729663, ++STORE, 94313983705088, 94313983717375, ++STORE, 94313983717376, 94313983729663, ++STORE, 140456286076928, 140456288329727, ++SNULL, 140456286220287, 140456288329727, ++STORE, 140456286076928, 140456286220287, ++STORE, 140456286220288, 140456288329727, ++ERASE, 140456286220288, 140456288329727, ++STORE, 140456288317440, 140456288325631, ++STORE, 140456288325632, 140456288329727, ++STORE, 140722507997184, 140722508001279, ++STORE, 140722507984896, 140722507997183, ++STORE, 140456288288768, 140456288317439, ++STORE, 140456288280576, 140456288288767, ++STORE, 140456283963392, 140456286076927, ++SNULL, 140456283963392, 140456283975679, ++STORE, 140456283975680, 140456286076927, ++STORE, 140456283963392, 140456283975679, ++SNULL, 140456286068735, 140456286076927, ++STORE, 140456283975680, 140456286068735, ++STORE, 140456286068736, 140456286076927, ++ERASE, 140456286068736, 140456286076927, ++STORE, 140456286068736, 140456286076927, ++STORE, 140456280166400, 140456283963391, ++SNULL, 140456280166400, 140456281825279, ++STORE, 140456281825280, 140456283963391, ++STORE, 140456280166400, 140456281825279, ++SNULL, 140456283922431, 140456283963391, ++STORE, 140456281825280, 140456283922431, ++STORE, 140456283922432, 140456283963391, ++SNULL, 140456283922432, 140456283947007, ++STORE, 140456283947008, 140456283963391, ++STORE, 140456283922432, 140456283947007, ++ERASE, 140456283922432, 140456283947007, ++STORE, 140456283922432, 140456283947007, ++ERASE, 140456283947008, 140456283963391, ++STORE, 140456283947008, 140456283963391, ++STORE, 140456288272384, 140456288288767, ++SNULL, 140456283938815, 140456283947007, ++STORE, 140456283922432, 140456283938815, ++STORE, 140456283938816, 140456283947007, ++SNULL, 140456286072831, 140456286076927, ++STORE, 140456286068736, 140456286072831, ++STORE, 140456286072832, 140456286076927, ++SNULL, 94313983709183, 94313983717375, ++STORE, 94313983705088, 94313983709183, ++STORE, 94313983709184, 94313983717375, ++SNULL, 140456288321535, 140456288325631, ++STORE, 140456288317440, 140456288321535, ++STORE, 140456288321536, 140456288325631, ++ERASE, 140456288288768, 140456288317439, ++STORE, 94314006716416, 94314006851583, ++STORE, 140456286588928, 140456288272383, ++STORE, 94314006716416, 94314006986751, ++STORE, 94314006716416, 94314007121919, ++STORE, 93948644454400, 93948644667391, ++STORE, 93948646764544, 93948646768639, ++STORE, 93948646768640, 93948646776831, ++STORE, 93948646776832, 93948646789119, ++STORE, 93948664999936, 93948667142143, ++STORE, 140187350659072, 140187352317951, ++STORE, 140187352317952, 140187354415103, ++STORE, 140187354415104, 140187354431487, ++STORE, 140187354431488, 140187354439679, ++STORE, 140187354439680, 140187354456063, ++STORE, 140187354456064, 140187354468351, ++STORE, 140187354468352, 140187356561407, ++STORE, 140187356561408, 140187356565503, ++STORE, 140187356565504, 140187356569599, ++STORE, 140187356569600, 140187356712959, ++STORE, 140187357081600, 140187358765055, ++STORE, 140187358765056, 140187358781439, ++STORE, 140187358810112, 140187358814207, ++STORE, 140187358814208, 140187358818303, ++STORE, 140187358818304, 140187358822399, ++STORE, 140730484518912, 140730484658175, ++STORE, 140730485690368, 140730485702655, ++STORE, 140730485702656, 140730485706751, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721211551744, 140737488351231, ++SNULL, 140721211559935, 140737488351231, ++STORE, 140721211551744, 140721211559935, ++STORE, 140721211420672, 140721211559935, ++STORE, 94105221423104, 94105223757823, ++SNULL, 94105221636095, 94105223757823, ++STORE, 94105221423104, 94105221636095, ++STORE, 94105221636096, 94105223757823, ++ERASE, 94105221636096, 94105223757823, ++STORE, 94105223733248, 94105223745535, ++STORE, 94105223745536, 94105223757823, ++STORE, 140474453676032, 140474455928831, ++SNULL, 140474453819391, 140474455928831, ++STORE, 140474453676032, 140474453819391, ++STORE, 140474453819392, 140474455928831, ++ERASE, 140474453819392, 140474455928831, ++STORE, 140474455916544, 140474455924735, ++STORE, 140474455924736, 140474455928831, ++STORE, 140721211703296, 140721211707391, ++STORE, 140721211691008, 140721211703295, ++STORE, 140474455887872, 140474455916543, ++STORE, 140474455879680, 140474455887871, ++STORE, 140474451562496, 140474453676031, ++SNULL, 140474451562496, 140474451574783, ++STORE, 140474451574784, 140474453676031, ++STORE, 140474451562496, 140474451574783, ++SNULL, 140474453667839, 140474453676031, ++STORE, 140474451574784, 140474453667839, ++STORE, 140474453667840, 140474453676031, ++ERASE, 140474453667840, 140474453676031, ++STORE, 140474453667840, 140474453676031, ++STORE, 140474447765504, 140474451562495, ++SNULL, 140474447765504, 140474449424383, ++STORE, 140474449424384, 140474451562495, ++STORE, 140474447765504, 140474449424383, ++SNULL, 140474451521535, 140474451562495, ++STORE, 140474449424384, 140474451521535, ++STORE, 140474451521536, 140474451562495, ++SNULL, 140474451521536, 140474451546111, ++STORE, 140474451546112, 140474451562495, ++STORE, 140474451521536, 140474451546111, ++ERASE, 140474451521536, 140474451546111, ++STORE, 140474451521536, 140474451546111, ++ERASE, 140474451546112, 140474451562495, ++STORE, 140474451546112, 140474451562495, ++STORE, 140474455871488, 140474455887871, ++SNULL, 140474451537919, 140474451546111, ++STORE, 140474451521536, 140474451537919, ++STORE, 140474451537920, 140474451546111, ++SNULL, 140474453671935, 140474453676031, ++STORE, 140474453667840, 140474453671935, ++STORE, 140474453671936, 140474453676031, ++SNULL, 94105223737343, 94105223745535, ++STORE, 94105223733248, 94105223737343, ++STORE, 94105223737344, 94105223745535, ++SNULL, 140474455920639, 140474455924735, ++STORE, 140474455916544, 140474455920639, ++STORE, 140474455920640, 140474455924735, ++ERASE, 140474455887872, 140474455916543, ++STORE, 94105238712320, 94105238847487, ++STORE, 140474454188032, 140474455871487, ++STORE, 94105238712320, 94105238982655, ++STORE, 94105238712320, 94105239117823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732356354048, 140737488351231, ++SNULL, 140732356362239, 140737488351231, ++STORE, 140732356354048, 140732356362239, ++STORE, 140732356222976, 140732356362239, ++STORE, 94461165989888, 94461168324607, ++SNULL, 94461166202879, 94461168324607, ++STORE, 94461165989888, 94461166202879, ++STORE, 94461166202880, 94461168324607, ++ERASE, 94461166202880, 94461168324607, ++STORE, 94461168300032, 94461168312319, ++STORE, 94461168312320, 94461168324607, ++STORE, 140317255110656, 140317257363455, ++SNULL, 140317255254015, 140317257363455, ++STORE, 140317255110656, 140317255254015, ++STORE, 140317255254016, 140317257363455, ++ERASE, 140317255254016, 140317257363455, ++STORE, 140317257351168, 140317257359359, ++STORE, 140317257359360, 140317257363455, ++STORE, 140732356583424, 140732356587519, ++STORE, 140732356571136, 140732356583423, ++STORE, 140317257322496, 140317257351167, ++STORE, 140317257314304, 140317257322495, ++STORE, 140317252997120, 140317255110655, ++SNULL, 140317252997120, 140317253009407, ++STORE, 140317253009408, 140317255110655, ++STORE, 140317252997120, 140317253009407, ++SNULL, 140317255102463, 140317255110655, ++STORE, 140317253009408, 140317255102463, ++STORE, 140317255102464, 140317255110655, ++ERASE, 140317255102464, 140317255110655, ++STORE, 140317255102464, 140317255110655, ++STORE, 140317249200128, 140317252997119, ++SNULL, 140317249200128, 140317250859007, ++STORE, 140317250859008, 140317252997119, ++STORE, 140317249200128, 140317250859007, ++SNULL, 140317252956159, 140317252997119, ++STORE, 140317250859008, 140317252956159, ++STORE, 140317252956160, 140317252997119, ++SNULL, 140317252956160, 140317252980735, ++STORE, 140317252980736, 140317252997119, ++STORE, 140317252956160, 140317252980735, ++ERASE, 140317252956160, 140317252980735, ++STORE, 140317252956160, 140317252980735, ++ERASE, 140317252980736, 140317252997119, ++STORE, 140317252980736, 140317252997119, ++STORE, 140317257306112, 140317257322495, ++SNULL, 140317252972543, 140317252980735, ++STORE, 140317252956160, 140317252972543, ++STORE, 140317252972544, 140317252980735, ++SNULL, 140317255106559, 140317255110655, ++STORE, 140317255102464, 140317255106559, ++STORE, 140317255106560, 140317255110655, ++SNULL, 94461168304127, 94461168312319, ++STORE, 94461168300032, 94461168304127, ++STORE, 94461168304128, 94461168312319, ++SNULL, 140317257355263, 140317257359359, ++STORE, 140317257351168, 140317257355263, ++STORE, 140317257355264, 140317257359359, ++ERASE, 140317257322496, 140317257351167, ++STORE, 94461195268096, 94461195403263, ++STORE, 140317255622656, 140317257306111, ++STORE, 94461195268096, 94461195538431, ++STORE, 94461195268096, 94461195673599, ++STORE, 94110050402304, 94110050615295, ++STORE, 94110052712448, 94110052716543, ++STORE, 94110052716544, 94110052724735, ++STORE, 94110052724736, 94110052737023, ++STORE, 94110061875200, 94110062415871, ++STORE, 140139439357952, 140139441016831, ++STORE, 140139441016832, 140139443113983, ++STORE, 140139443113984, 140139443130367, ++STORE, 140139443130368, 140139443138559, ++STORE, 140139443138560, 140139443154943, ++STORE, 140139443154944, 140139443167231, ++STORE, 140139443167232, 140139445260287, ++STORE, 140139445260288, 140139445264383, ++STORE, 140139445264384, 140139445268479, ++STORE, 140139445268480, 140139445411839, ++STORE, 140139445780480, 140139447463935, ++STORE, 140139447463936, 140139447480319, ++STORE, 140139447508992, 140139447513087, ++STORE, 140139447513088, 140139447517183, ++STORE, 140139447517184, 140139447521279, ++STORE, 140731901427712, 140731901566975, ++STORE, 140731902259200, 140731902271487, ++STORE, 140731902271488, 140731902275583, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720941613056, 140737488351231, ++SNULL, 140720941621247, 140737488351231, ++STORE, 140720941613056, 140720941621247, ++STORE, 140720941481984, 140720941621247, ++STORE, 93902377721856, 93902379945983, ++SNULL, 93902377832447, 93902379945983, ++STORE, 93902377721856, 93902377832447, ++STORE, 93902377832448, 93902379945983, ++ERASE, 93902377832448, 93902379945983, ++STORE, 93902379925504, 93902379937791, ++STORE, 93902379937792, 93902379945983, ++STORE, 139836543635456, 139836545888255, ++SNULL, 139836543778815, 139836545888255, ++STORE, 139836543635456, 139836543778815, ++STORE, 139836543778816, 139836545888255, ++ERASE, 139836543778816, 139836545888255, ++STORE, 139836545875968, 139836545884159, ++STORE, 139836545884160, 139836545888255, ++STORE, 140720941711360, 140720941715455, ++STORE, 140720941699072, 140720941711359, ++STORE, 139836545847296, 139836545875967, ++STORE, 139836545839104, 139836545847295, ++STORE, 139836539838464, 139836543635455, ++SNULL, 139836539838464, 139836541497343, ++STORE, 139836541497344, 139836543635455, ++STORE, 139836539838464, 139836541497343, ++SNULL, 139836543594495, 139836543635455, ++STORE, 139836541497344, 139836543594495, ++STORE, 139836543594496, 139836543635455, ++SNULL, 139836543594496, 139836543619071, ++STORE, 139836543619072, 139836543635455, ++STORE, 139836543594496, 139836543619071, ++ERASE, 139836543594496, 139836543619071, ++STORE, 139836543594496, 139836543619071, ++ERASE, 139836543619072, 139836543635455, ++STORE, 139836543619072, 139836543635455, ++SNULL, 139836543610879, 139836543619071, ++STORE, 139836543594496, 139836543610879, ++STORE, 139836543610880, 139836543619071, ++SNULL, 93902379933695, 93902379937791, ++STORE, 93902379925504, 93902379933695, ++STORE, 93902379933696, 93902379937791, ++SNULL, 139836545880063, 139836545884159, ++STORE, 139836545875968, 139836545880063, ++STORE, 139836545880064, 139836545884159, ++ERASE, 139836545847296, 139836545875967, ++STORE, 93902396891136, 93902397026303, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736538206208, 140737488351231, ++SNULL, 140736538214399, 140737488351231, ++STORE, 140736538206208, 140736538214399, ++STORE, 140736538075136, 140736538214399, ++STORE, 94173471399936, 94173473734655, ++SNULL, 94173471612927, 94173473734655, ++STORE, 94173471399936, 94173471612927, ++STORE, 94173471612928, 94173473734655, ++ERASE, 94173471612928, 94173473734655, ++STORE, 94173473710080, 94173473722367, ++STORE, 94173473722368, 94173473734655, ++STORE, 140035513556992, 140035515809791, ++SNULL, 140035513700351, 140035515809791, ++STORE, 140035513556992, 140035513700351, ++STORE, 140035513700352, 140035515809791, ++ERASE, 140035513700352, 140035515809791, ++STORE, 140035515797504, 140035515805695, ++STORE, 140035515805696, 140035515809791, ++STORE, 140736538329088, 140736538333183, ++STORE, 140736538316800, 140736538329087, ++STORE, 140035515768832, 140035515797503, ++STORE, 140035515760640, 140035515768831, ++STORE, 140035511443456, 140035513556991, ++SNULL, 140035511443456, 140035511455743, ++STORE, 140035511455744, 140035513556991, ++STORE, 140035511443456, 140035511455743, ++SNULL, 140035513548799, 140035513556991, ++STORE, 140035511455744, 140035513548799, ++STORE, 140035513548800, 140035513556991, ++ERASE, 140035513548800, 140035513556991, ++STORE, 140035513548800, 140035513556991, ++STORE, 140035507646464, 140035511443455, ++SNULL, 140035507646464, 140035509305343, ++STORE, 140035509305344, 140035511443455, ++STORE, 140035507646464, 140035509305343, ++SNULL, 140035511402495, 140035511443455, ++STORE, 140035509305344, 140035511402495, ++STORE, 140035511402496, 140035511443455, ++SNULL, 140035511402496, 140035511427071, ++STORE, 140035511427072, 140035511443455, ++STORE, 140035511402496, 140035511427071, ++ERASE, 140035511402496, 140035511427071, ++STORE, 140035511402496, 140035511427071, ++ERASE, 140035511427072, 140035511443455, ++STORE, 140035511427072, 140035511443455, ++STORE, 140035515752448, 140035515768831, ++SNULL, 140035511418879, 140035511427071, ++STORE, 140035511402496, 140035511418879, ++STORE, 140035511418880, 140035511427071, ++SNULL, 140035513552895, 140035513556991, ++STORE, 140035513548800, 140035513552895, ++STORE, 140035513552896, 140035513556991, ++SNULL, 94173473714175, 94173473722367, ++STORE, 94173473710080, 94173473714175, ++STORE, 94173473714176, 94173473722367, ++SNULL, 140035515801599, 140035515805695, ++STORE, 140035515797504, 140035515801599, ++STORE, 140035515801600, 140035515805695, ++ERASE, 140035515768832, 140035515797503, ++STORE, 94173478645760, 94173478780927, ++STORE, 140035514068992, 140035515752447, ++STORE, 94173478645760, 94173478916095, ++STORE, 94173478645760, 94173479051263, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724216176640, 140737488351231, ++SNULL, 140724216184831, 140737488351231, ++STORE, 140724216176640, 140724216184831, ++STORE, 140724216045568, 140724216184831, ++STORE, 94870930628608, 94870932963327, ++SNULL, 94870930841599, 94870932963327, ++STORE, 94870930628608, 94870930841599, ++STORE, 94870930841600, 94870932963327, ++ERASE, 94870930841600, 94870932963327, ++STORE, 94870932938752, 94870932951039, ++STORE, 94870932951040, 94870932963327, ++STORE, 140453683736576, 140453685989375, ++SNULL, 140453683879935, 140453685989375, ++STORE, 140453683736576, 140453683879935, ++STORE, 140453683879936, 140453685989375, ++ERASE, 140453683879936, 140453685989375, ++STORE, 140453685977088, 140453685985279, ++STORE, 140453685985280, 140453685989375, ++STORE, 140724216832000, 140724216836095, ++STORE, 140724216819712, 140724216831999, ++STORE, 140453685948416, 140453685977087, ++STORE, 140453685940224, 140453685948415, ++STORE, 140453681623040, 140453683736575, ++SNULL, 140453681623040, 140453681635327, ++STORE, 140453681635328, 140453683736575, ++STORE, 140453681623040, 140453681635327, ++SNULL, 140453683728383, 140453683736575, ++STORE, 140453681635328, 140453683728383, ++STORE, 140453683728384, 140453683736575, ++ERASE, 140453683728384, 140453683736575, ++STORE, 140453683728384, 140453683736575, ++STORE, 140453677826048, 140453681623039, ++SNULL, 140453677826048, 140453679484927, ++STORE, 140453679484928, 140453681623039, ++STORE, 140453677826048, 140453679484927, ++SNULL, 140453681582079, 140453681623039, ++STORE, 140453679484928, 140453681582079, ++STORE, 140453681582080, 140453681623039, ++SNULL, 140453681582080, 140453681606655, ++STORE, 140453681606656, 140453681623039, ++STORE, 140453681582080, 140453681606655, ++ERASE, 140453681582080, 140453681606655, ++STORE, 140453681582080, 140453681606655, ++ERASE, 140453681606656, 140453681623039, ++STORE, 140453681606656, 140453681623039, ++STORE, 140453685932032, 140453685948415, ++SNULL, 140453681598463, 140453681606655, ++STORE, 140453681582080, 140453681598463, ++STORE, 140453681598464, 140453681606655, ++SNULL, 140453683732479, 140453683736575, ++STORE, 140453683728384, 140453683732479, ++STORE, 140453683732480, 140453683736575, ++SNULL, 94870932942847, 94870932951039, ++STORE, 94870932938752, 94870932942847, ++STORE, 94870932942848, 94870932951039, ++SNULL, 140453685981183, 140453685985279, ++STORE, 140453685977088, 140453685981183, ++STORE, 140453685981184, 140453685985279, ++ERASE, 140453685948416, 140453685977087, ++STORE, 94870940565504, 94870940700671, ++STORE, 140453684248576, 140453685932031, ++STORE, 94870940565504, 94870940835839, ++STORE, 94870940565504, 94870940971007, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731275661312, 140737488351231, ++SNULL, 140731275669503, 140737488351231, ++STORE, 140731275661312, 140731275669503, ++STORE, 140731275530240, 140731275669503, ++STORE, 94642788548608, 94642790883327, ++SNULL, 94642788761599, 94642790883327, ++STORE, 94642788548608, 94642788761599, ++STORE, 94642788761600, 94642790883327, ++ERASE, 94642788761600, 94642790883327, ++STORE, 94642790858752, 94642790871039, ++STORE, 94642790871040, 94642790883327, ++STORE, 140228458749952, 140228461002751, ++SNULL, 140228458893311, 140228461002751, ++STORE, 140228458749952, 140228458893311, ++STORE, 140228458893312, 140228461002751, ++ERASE, 140228458893312, 140228461002751, ++STORE, 140228460990464, 140228460998655, ++STORE, 140228460998656, 140228461002751, ++STORE, 140731276349440, 140731276353535, ++STORE, 140731276337152, 140731276349439, ++STORE, 140228460961792, 140228460990463, ++STORE, 140228460953600, 140228460961791, ++STORE, 140228456636416, 140228458749951, ++SNULL, 140228456636416, 140228456648703, ++STORE, 140228456648704, 140228458749951, ++STORE, 140228456636416, 140228456648703, ++SNULL, 140228458741759, 140228458749951, ++STORE, 140228456648704, 140228458741759, ++STORE, 140228458741760, 140228458749951, ++ERASE, 140228458741760, 140228458749951, ++STORE, 140228458741760, 140228458749951, ++STORE, 140228452839424, 140228456636415, ++SNULL, 140228452839424, 140228454498303, ++STORE, 140228454498304, 140228456636415, ++STORE, 140228452839424, 140228454498303, ++SNULL, 140228456595455, 140228456636415, ++STORE, 140228454498304, 140228456595455, ++STORE, 140228456595456, 140228456636415, ++SNULL, 140228456595456, 140228456620031, ++STORE, 140228456620032, 140228456636415, ++STORE, 140228456595456, 140228456620031, ++ERASE, 140228456595456, 140228456620031, ++STORE, 140228456595456, 140228456620031, ++ERASE, 140228456620032, 140228456636415, ++STORE, 140228456620032, 140228456636415, ++STORE, 140228460945408, 140228460961791, ++SNULL, 140228456611839, 140228456620031, ++STORE, 140228456595456, 140228456611839, ++STORE, 140228456611840, 140228456620031, ++SNULL, 140228458745855, 140228458749951, ++STORE, 140228458741760, 140228458745855, ++STORE, 140228458745856, 140228458749951, ++SNULL, 94642790862847, 94642790871039, ++STORE, 94642790858752, 94642790862847, ++STORE, 94642790862848, 94642790871039, ++SNULL, 140228460994559, 140228460998655, ++STORE, 140228460990464, 140228460994559, ++STORE, 140228460994560, 140228460998655, ++ERASE, 140228460961792, 140228460990463, ++STORE, 94642801549312, 94642801684479, ++STORE, 140228459261952, 140228460945407, ++STORE, 94642801549312, 94642801819647, ++STORE, 94642801549312, 94642801954815, ++STORE, 94604087611392, 94604087824383, ++STORE, 94604089921536, 94604089925631, ++STORE, 94604089925632, 94604089933823, ++STORE, 94604089933824, 94604089946111, ++STORE, 94604105125888, 94604106424319, ++STORE, 140454937694208, 140454939353087, ++STORE, 140454939353088, 140454941450239, ++STORE, 140454941450240, 140454941466623, ++STORE, 140454941466624, 140454941474815, ++STORE, 140454941474816, 140454941491199, ++STORE, 140454941491200, 140454941503487, ++STORE, 140454941503488, 140454943596543, ++STORE, 140454943596544, 140454943600639, ++STORE, 140454943600640, 140454943604735, ++STORE, 140454943604736, 140454943748095, ++STORE, 140454944116736, 140454945800191, ++STORE, 140454945800192, 140454945816575, ++STORE, 140454945845248, 140454945849343, ++STORE, 140454945849344, 140454945853439, ++STORE, 140454945853440, 140454945857535, ++STORE, 140728438214656, 140728438353919, ++STORE, 140728439095296, 140728439107583, ++STORE, 140728439107584, 140728439111679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721843453952, 140737488351231, ++SNULL, 140721843462143, 140737488351231, ++STORE, 140721843453952, 140721843462143, ++STORE, 140721843322880, 140721843462143, ++STORE, 94465962455040, 94465964789759, ++SNULL, 94465962668031, 94465964789759, ++STORE, 94465962455040, 94465962668031, ++STORE, 94465962668032, 94465964789759, ++ERASE, 94465962668032, 94465964789759, ++STORE, 94465964765184, 94465964777471, ++STORE, 94465964777472, 94465964789759, ++STORE, 139913488314368, 139913490567167, ++SNULL, 139913488457727, 139913490567167, ++STORE, 139913488314368, 139913488457727, ++STORE, 139913488457728, 139913490567167, ++ERASE, 139913488457728, 139913490567167, ++STORE, 139913490554880, 139913490563071, ++STORE, 139913490563072, 139913490567167, ++STORE, 140721843503104, 140721843507199, ++STORE, 140721843490816, 140721843503103, ++STORE, 139913490526208, 139913490554879, ++STORE, 139913490518016, 139913490526207, ++STORE, 139913486200832, 139913488314367, ++SNULL, 139913486200832, 139913486213119, ++STORE, 139913486213120, 139913488314367, ++STORE, 139913486200832, 139913486213119, ++SNULL, 139913488306175, 139913488314367, ++STORE, 139913486213120, 139913488306175, ++STORE, 139913488306176, 139913488314367, ++ERASE, 139913488306176, 139913488314367, ++STORE, 139913488306176, 139913488314367, ++STORE, 139913482403840, 139913486200831, ++SNULL, 139913482403840, 139913484062719, ++STORE, 139913484062720, 139913486200831, ++STORE, 139913482403840, 139913484062719, ++SNULL, 139913486159871, 139913486200831, ++STORE, 139913484062720, 139913486159871, ++STORE, 139913486159872, 139913486200831, ++SNULL, 139913486159872, 139913486184447, ++STORE, 139913486184448, 139913486200831, ++STORE, 139913486159872, 139913486184447, ++ERASE, 139913486159872, 139913486184447, ++STORE, 139913486159872, 139913486184447, ++ERASE, 139913486184448, 139913486200831, ++STORE, 139913486184448, 139913486200831, ++STORE, 139913490509824, 139913490526207, ++SNULL, 139913486176255, 139913486184447, ++STORE, 139913486159872, 139913486176255, ++STORE, 139913486176256, 139913486184447, ++SNULL, 139913488310271, 139913488314367, ++STORE, 139913488306176, 139913488310271, ++STORE, 139913488310272, 139913488314367, ++SNULL, 94465964769279, 94465964777471, ++STORE, 94465964765184, 94465964769279, ++STORE, 94465964769280, 94465964777471, ++SNULL, 139913490558975, 139913490563071, ++STORE, 139913490554880, 139913490558975, ++STORE, 139913490558976, 139913490563071, ++ERASE, 139913490526208, 139913490554879, ++STORE, 94465970024448, 94465970159615, ++STORE, 139913488826368, 139913490509823, ++STORE, 94465970024448, 94465970294783, ++STORE, 94465970024448, 94465970429951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720583307264, 140737488351231, ++SNULL, 140720583315455, 140737488351231, ++STORE, 140720583307264, 140720583315455, ++STORE, 140720583176192, 140720583315455, ++STORE, 94212322082816, 94212324417535, ++SNULL, 94212322295807, 94212324417535, ++STORE, 94212322082816, 94212322295807, ++STORE, 94212322295808, 94212324417535, ++ERASE, 94212322295808, 94212324417535, ++STORE, 94212324392960, 94212324405247, ++STORE, 94212324405248, 94212324417535, ++STORE, 139659688538112, 139659690790911, ++SNULL, 139659688681471, 139659690790911, ++STORE, 139659688538112, 139659688681471, ++STORE, 139659688681472, 139659690790911, ++ERASE, 139659688681472, 139659690790911, ++STORE, 139659690778624, 139659690786815, ++STORE, 139659690786816, 139659690790911, ++STORE, 140720584781824, 140720584785919, ++STORE, 140720584769536, 140720584781823, ++STORE, 139659690749952, 139659690778623, ++STORE, 139659690741760, 139659690749951, ++STORE, 139659686424576, 139659688538111, ++SNULL, 139659686424576, 139659686436863, ++STORE, 139659686436864, 139659688538111, ++STORE, 139659686424576, 139659686436863, ++SNULL, 139659688529919, 139659688538111, ++STORE, 139659686436864, 139659688529919, ++STORE, 139659688529920, 139659688538111, ++ERASE, 139659688529920, 139659688538111, ++STORE, 139659688529920, 139659688538111, ++STORE, 139659682627584, 139659686424575, ++SNULL, 139659682627584, 139659684286463, ++STORE, 139659684286464, 139659686424575, ++STORE, 139659682627584, 139659684286463, ++SNULL, 139659686383615, 139659686424575, ++STORE, 139659684286464, 139659686383615, ++STORE, 139659686383616, 139659686424575, ++SNULL, 139659686383616, 139659686408191, ++STORE, 139659686408192, 139659686424575, ++STORE, 139659686383616, 139659686408191, ++ERASE, 139659686383616, 139659686408191, ++STORE, 139659686383616, 139659686408191, ++ERASE, 139659686408192, 139659686424575, ++STORE, 139659686408192, 139659686424575, ++STORE, 139659690733568, 139659690749951, ++SNULL, 139659686399999, 139659686408191, ++STORE, 139659686383616, 139659686399999, ++STORE, 139659686400000, 139659686408191, ++SNULL, 139659688534015, 139659688538111, ++STORE, 139659688529920, 139659688534015, ++STORE, 139659688534016, 139659688538111, ++SNULL, 94212324397055, 94212324405247, ++STORE, 94212324392960, 94212324397055, ++STORE, 94212324397056, 94212324405247, ++SNULL, 139659690782719, 139659690786815, ++STORE, 139659690778624, 139659690782719, ++STORE, 139659690782720, 139659690786815, ++ERASE, 139659690749952, 139659690778623, ++STORE, 94212355014656, 94212355149823, ++STORE, 139659689050112, 139659690733567, ++STORE, 94212355014656, 94212355284991, ++STORE, 94212355014656, 94212355420159, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727689830400, 140737488351231, ++SNULL, 140727689838591, 140737488351231, ++STORE, 140727689830400, 140727689838591, ++STORE, 140727689699328, 140727689838591, ++STORE, 94572390281216, 94572392615935, ++SNULL, 94572390494207, 94572392615935, ++STORE, 94572390281216, 94572390494207, ++STORE, 94572390494208, 94572392615935, ++ERASE, 94572390494208, 94572392615935, ++STORE, 94572392591360, 94572392603647, ++STORE, 94572392603648, 94572392615935, ++STORE, 140575923769344, 140575926022143, ++SNULL, 140575923912703, 140575926022143, ++STORE, 140575923769344, 140575923912703, ++STORE, 140575923912704, 140575926022143, ++ERASE, 140575923912704, 140575926022143, ++STORE, 140575926009856, 140575926018047, ++STORE, 140575926018048, 140575926022143, ++STORE, 140727689871360, 140727689875455, ++STORE, 140727689859072, 140727689871359, ++STORE, 140575925981184, 140575926009855, ++STORE, 140575925972992, 140575925981183, ++STORE, 140575921655808, 140575923769343, ++SNULL, 140575921655808, 140575921668095, ++STORE, 140575921668096, 140575923769343, ++STORE, 140575921655808, 140575921668095, ++SNULL, 140575923761151, 140575923769343, ++STORE, 140575921668096, 140575923761151, ++STORE, 140575923761152, 140575923769343, ++ERASE, 140575923761152, 140575923769343, ++STORE, 140575923761152, 140575923769343, ++STORE, 140575917858816, 140575921655807, ++SNULL, 140575917858816, 140575919517695, ++STORE, 140575919517696, 140575921655807, ++STORE, 140575917858816, 140575919517695, ++SNULL, 140575921614847, 140575921655807, ++STORE, 140575919517696, 140575921614847, ++STORE, 140575921614848, 140575921655807, ++SNULL, 140575921614848, 140575921639423, ++STORE, 140575921639424, 140575921655807, ++STORE, 140575921614848, 140575921639423, ++ERASE, 140575921614848, 140575921639423, ++STORE, 140575921614848, 140575921639423, ++ERASE, 140575921639424, 140575921655807, ++STORE, 140575921639424, 140575921655807, ++STORE, 140575925964800, 140575925981183, ++SNULL, 140575921631231, 140575921639423, ++STORE, 140575921614848, 140575921631231, ++STORE, 140575921631232, 140575921639423, ++SNULL, 140575923765247, 140575923769343, ++STORE, 140575923761152, 140575923765247, ++STORE, 140575923765248, 140575923769343, ++SNULL, 94572392595455, 94572392603647, ++STORE, 94572392591360, 94572392595455, ++STORE, 94572392595456, 94572392603647, ++SNULL, 140575926013951, 140575926018047, ++STORE, 140575926009856, 140575926013951, ++STORE, 140575926013952, 140575926018047, ++ERASE, 140575925981184, 140575926009855, ++STORE, 94572402278400, 94572402413567, ++STORE, 140575924281344, 140575925964799, ++STORE, 94572402278400, 94572402548735, ++STORE, 94572402278400, 94572402683903, ++STORE, 94572402278400, 94572402851839, ++SNULL, 94572402827263, 94572402851839, ++STORE, 94572402278400, 94572402827263, ++STORE, 94572402827264, 94572402851839, ++ERASE, 94572402827264, 94572402851839, ++STORE, 94572402278400, 94572402966527, ++STORE, 94572402278400, 94572403109887, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725520506880, 140737488351231, ++SNULL, 140725520515071, 140737488351231, ++STORE, 140725520506880, 140725520515071, ++STORE, 140725520375808, 140725520515071, ++STORE, 93829948788736, 93829951012863, ++SNULL, 93829948899327, 93829951012863, ++STORE, 93829948788736, 93829948899327, ++STORE, 93829948899328, 93829951012863, ++ERASE, 93829948899328, 93829951012863, ++STORE, 93829950992384, 93829951004671, ++STORE, 93829951004672, 93829951012863, ++STORE, 140133696794624, 140133699047423, ++SNULL, 140133696937983, 140133699047423, ++STORE, 140133696794624, 140133696937983, ++STORE, 140133696937984, 140133699047423, ++ERASE, 140133696937984, 140133699047423, ++STORE, 140133699035136, 140133699043327, ++STORE, 140133699043328, 140133699047423, ++STORE, 140725520875520, 140725520879615, ++STORE, 140725520863232, 140725520875519, ++STORE, 140133699006464, 140133699035135, ++STORE, 140133698998272, 140133699006463, ++STORE, 140133692997632, 140133696794623, ++SNULL, 140133692997632, 140133694656511, ++STORE, 140133694656512, 140133696794623, ++STORE, 140133692997632, 140133694656511, ++SNULL, 140133696753663, 140133696794623, ++STORE, 140133694656512, 140133696753663, ++STORE, 140133696753664, 140133696794623, ++SNULL, 140133696753664, 140133696778239, ++STORE, 140133696778240, 140133696794623, ++STORE, 140133696753664, 140133696778239, ++ERASE, 140133696753664, 140133696778239, ++STORE, 140133696753664, 140133696778239, ++ERASE, 140133696778240, 140133696794623, ++STORE, 140133696778240, 140133696794623, ++SNULL, 140133696770047, 140133696778239, ++STORE, 140133696753664, 140133696770047, ++STORE, 140133696770048, 140133696778239, ++SNULL, 93829951000575, 93829951004671, ++STORE, 93829950992384, 93829951000575, ++STORE, 93829951000576, 93829951004671, ++SNULL, 140133699039231, 140133699043327, ++STORE, 140133699035136, 140133699039231, ++STORE, 140133699039232, 140133699043327, ++ERASE, 140133699006464, 140133699035135, ++STORE, 93829978693632, 93829978828799, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736118022144, 140737488351231, ++SNULL, 140736118030335, 140737488351231, ++STORE, 140736118022144, 140736118030335, ++STORE, 140736117891072, 140736118030335, ++STORE, 94467663982592, 94467666206719, ++SNULL, 94467664093183, 94467666206719, ++STORE, 94467663982592, 94467664093183, ++STORE, 94467664093184, 94467666206719, ++ERASE, 94467664093184, 94467666206719, ++STORE, 94467666186240, 94467666198527, ++STORE, 94467666198528, 94467666206719, ++STORE, 140525377327104, 140525379579903, ++SNULL, 140525377470463, 140525379579903, ++STORE, 140525377327104, 140525377470463, ++STORE, 140525377470464, 140525379579903, ++ERASE, 140525377470464, 140525379579903, ++STORE, 140525379567616, 140525379575807, ++STORE, 140525379575808, 140525379579903, ++STORE, 140736118771712, 140736118775807, ++STORE, 140736118759424, 140736118771711, ++STORE, 140525379538944, 140525379567615, ++STORE, 140525379530752, 140525379538943, ++STORE, 140525373530112, 140525377327103, ++SNULL, 140525373530112, 140525375188991, ++STORE, 140525375188992, 140525377327103, ++STORE, 140525373530112, 140525375188991, ++SNULL, 140525377286143, 140525377327103, ++STORE, 140525375188992, 140525377286143, ++STORE, 140525377286144, 140525377327103, ++SNULL, 140525377286144, 140525377310719, ++STORE, 140525377310720, 140525377327103, ++STORE, 140525377286144, 140525377310719, ++ERASE, 140525377286144, 140525377310719, ++STORE, 140525377286144, 140525377310719, ++ERASE, 140525377310720, 140525377327103, ++STORE, 140525377310720, 140525377327103, ++SNULL, 140525377302527, 140525377310719, ++STORE, 140525377286144, 140525377302527, ++STORE, 140525377302528, 140525377310719, ++SNULL, 94467666194431, 94467666198527, ++STORE, 94467666186240, 94467666194431, ++STORE, 94467666194432, 94467666198527, ++SNULL, 140525379571711, 140525379575807, ++STORE, 140525379567616, 140525379571711, ++STORE, 140525379571712, 140525379575807, ++ERASE, 140525379538944, 140525379567615, ++STORE, 94467693379584, 94467693514751, ++STORE, 94200172744704, 94200172957695, ++STORE, 94200175054848, 94200175058943, ++STORE, 94200175058944, 94200175067135, ++STORE, 94200175067136, 94200175079423, ++STORE, 94200196673536, 94200198905855, ++STORE, 140053867720704, 140053869379583, ++STORE, 140053869379584, 140053871476735, ++STORE, 140053871476736, 140053871493119, ++STORE, 140053871493120, 140053871501311, ++STORE, 140053871501312, 140053871517695, ++STORE, 140053871517696, 140053871529983, ++STORE, 140053871529984, 140053873623039, ++STORE, 140053873623040, 140053873627135, ++STORE, 140053873627136, 140053873631231, ++STORE, 140053873631232, 140053873774591, ++STORE, 140053874143232, 140053875826687, ++STORE, 140053875826688, 140053875843071, ++STORE, 140053875871744, 140053875875839, ++STORE, 140053875875840, 140053875879935, ++STORE, 140053875879936, 140053875884031, ++STORE, 140728538484736, 140728538623999, ++STORE, 140728538652672, 140728538664959, ++STORE, 140728538664960, 140728538669055, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732307775488, 140737488351231, ++SNULL, 140732307783679, 140737488351231, ++STORE, 140732307775488, 140732307783679, ++STORE, 140732307644416, 140732307783679, ++STORE, 93831417630720, 93831419965439, ++SNULL, 93831417843711, 93831419965439, ++STORE, 93831417630720, 93831417843711, ++STORE, 93831417843712, 93831419965439, ++ERASE, 93831417843712, 93831419965439, ++STORE, 93831419940864, 93831419953151, ++STORE, 93831419953152, 93831419965439, ++STORE, 140241062088704, 140241064341503, ++SNULL, 140241062232063, 140241064341503, ++STORE, 140241062088704, 140241062232063, ++STORE, 140241062232064, 140241064341503, ++ERASE, 140241062232064, 140241064341503, ++STORE, 140241064329216, 140241064337407, ++STORE, 140241064337408, 140241064341503, ++STORE, 140732308140032, 140732308144127, ++STORE, 140732308127744, 140732308140031, ++STORE, 140241064300544, 140241064329215, ++STORE, 140241064292352, 140241064300543, ++STORE, 140241059975168, 140241062088703, ++SNULL, 140241059975168, 140241059987455, ++STORE, 140241059987456, 140241062088703, ++STORE, 140241059975168, 140241059987455, ++SNULL, 140241062080511, 140241062088703, ++STORE, 140241059987456, 140241062080511, ++STORE, 140241062080512, 140241062088703, ++ERASE, 140241062080512, 140241062088703, ++STORE, 140241062080512, 140241062088703, ++STORE, 140241056178176, 140241059975167, ++SNULL, 140241056178176, 140241057837055, ++STORE, 140241057837056, 140241059975167, ++STORE, 140241056178176, 140241057837055, ++SNULL, 140241059934207, 140241059975167, ++STORE, 140241057837056, 140241059934207, ++STORE, 140241059934208, 140241059975167, ++SNULL, 140241059934208, 140241059958783, ++STORE, 140241059958784, 140241059975167, ++STORE, 140241059934208, 140241059958783, ++ERASE, 140241059934208, 140241059958783, ++STORE, 140241059934208, 140241059958783, ++ERASE, 140241059958784, 140241059975167, ++STORE, 140241059958784, 140241059975167, ++STORE, 140241064284160, 140241064300543, ++SNULL, 140241059950591, 140241059958783, ++STORE, 140241059934208, 140241059950591, ++STORE, 140241059950592, 140241059958783, ++SNULL, 140241062084607, 140241062088703, ++STORE, 140241062080512, 140241062084607, ++STORE, 140241062084608, 140241062088703, ++SNULL, 93831419944959, 93831419953151, ++STORE, 93831419940864, 93831419944959, ++STORE, 93831419944960, 93831419953151, ++SNULL, 140241064333311, 140241064337407, ++STORE, 140241064329216, 140241064333311, ++STORE, 140241064333312, 140241064337407, ++ERASE, 140241064300544, 140241064329215, ++STORE, 93831435284480, 93831435419647, ++STORE, 140241062600704, 140241064284159, ++STORE, 93831435284480, 93831435554815, ++STORE, 93831435284480, 93831435689983, ++STORE, 93831435284480, 93831435862015, ++SNULL, 93831435837439, 93831435862015, ++STORE, 93831435284480, 93831435837439, ++STORE, 93831435837440, 93831435862015, ++ERASE, 93831435837440, 93831435862015, ++STORE, 93831435284480, 93831435972607, ++STORE, 93831435284480, 93831436107775, ++SNULL, 93831436091391, 93831436107775, ++STORE, 93831435284480, 93831436091391, ++STORE, 93831436091392, 93831436107775, ++ERASE, 93831436091392, 93831436107775, ++STORE, 93831435284480, 93831436226559, ++STORE, 93831435284480, 93831436361727, ++STORE, 93831435284480, 93831436505087, ++STORE, 93831435284480, 93831436652543, ++STORE, 93831435284480, 93831436787711, ++STORE, 93831435284480, 93831436926975, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728546775040, 140737488351231, ++SNULL, 140728546783231, 140737488351231, ++STORE, 140728546775040, 140728546783231, ++STORE, 140728546643968, 140728546783231, ++STORE, 94456178786304, 94456181010431, ++SNULL, 94456178896895, 94456181010431, ++STORE, 94456178786304, 94456178896895, ++STORE, 94456178896896, 94456181010431, ++ERASE, 94456178896896, 94456181010431, ++STORE, 94456180989952, 94456181002239, ++STORE, 94456181002240, 94456181010431, ++STORE, 140221893091328, 140221895344127, ++SNULL, 140221893234687, 140221895344127, ++STORE, 140221893091328, 140221893234687, ++STORE, 140221893234688, 140221895344127, ++ERASE, 140221893234688, 140221895344127, ++STORE, 140221895331840, 140221895340031, ++STORE, 140221895340032, 140221895344127, ++STORE, 140728547803136, 140728547807231, ++STORE, 140728547790848, 140728547803135, ++STORE, 140221895303168, 140221895331839, ++STORE, 140221895294976, 140221895303167, ++STORE, 140221889294336, 140221893091327, ++SNULL, 140221889294336, 140221890953215, ++STORE, 140221890953216, 140221893091327, ++STORE, 140221889294336, 140221890953215, ++SNULL, 140221893050367, 140221893091327, ++STORE, 140221890953216, 140221893050367, ++STORE, 140221893050368, 140221893091327, ++SNULL, 140221893050368, 140221893074943, ++STORE, 140221893074944, 140221893091327, ++STORE, 140221893050368, 140221893074943, ++ERASE, 140221893050368, 140221893074943, ++STORE, 140221893050368, 140221893074943, ++ERASE, 140221893074944, 140221893091327, ++STORE, 140221893074944, 140221893091327, ++SNULL, 140221893066751, 140221893074943, ++STORE, 140221893050368, 140221893066751, ++STORE, 140221893066752, 140221893074943, ++SNULL, 94456180998143, 94456181002239, ++STORE, 94456180989952, 94456180998143, ++STORE, 94456180998144, 94456181002239, ++SNULL, 140221895335935, 140221895340031, ++STORE, 140221895331840, 140221895335935, ++STORE, 140221895335936, 140221895340031, ++ERASE, 140221895303168, 140221895331839, ++STORE, 94456203730944, 94456203866111, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140734438637568, 140737488351231, ++SNULL, 140734438645759, 140737488351231, ++STORE, 140734438637568, 140734438645759, ++STORE, 140734438506496, 140734438645759, ++STORE, 94652233351168, 94652235575295, ++SNULL, 94652233461759, 94652235575295, ++STORE, 94652233351168, 94652233461759, ++STORE, 94652233461760, 94652235575295, ++ERASE, 94652233461760, 94652235575295, ++STORE, 94652235554816, 94652235567103, ++STORE, 94652235567104, 94652235575295, ++STORE, 140536493195264, 140536495448063, ++SNULL, 140536493338623, 140536495448063, ++STORE, 140536493195264, 140536493338623, ++STORE, 140536493338624, 140536495448063, ++ERASE, 140536493338624, 140536495448063, ++STORE, 140536495435776, 140536495443967, ++STORE, 140536495443968, 140536495448063, ++STORE, 140734439002112, 140734439006207, ++STORE, 140734438989824, 140734439002111, ++STORE, 140536495407104, 140536495435775, ++STORE, 140536495398912, 140536495407103, ++STORE, 140536489398272, 140536493195263, ++SNULL, 140536489398272, 140536491057151, ++STORE, 140536491057152, 140536493195263, ++STORE, 140536489398272, 140536491057151, ++SNULL, 140536493154303, 140536493195263, ++STORE, 140536491057152, 140536493154303, ++STORE, 140536493154304, 140536493195263, ++SNULL, 140536493154304, 140536493178879, ++STORE, 140536493178880, 140536493195263, ++STORE, 140536493154304, 140536493178879, ++ERASE, 140536493154304, 140536493178879, ++STORE, 140536493154304, 140536493178879, ++ERASE, 140536493178880, 140536493195263, ++STORE, 140536493178880, 140536493195263, ++SNULL, 140536493170687, 140536493178879, ++STORE, 140536493154304, 140536493170687, ++STORE, 140536493170688, 140536493178879, ++SNULL, 94652235563007, 94652235567103, ++STORE, 94652235554816, 94652235563007, ++STORE, 94652235563008, 94652235567103, ++SNULL, 140536495439871, 140536495443967, ++STORE, 140536495435776, 140536495439871, ++STORE, 140536495439872, 140536495443967, ++ERASE, 140536495407104, 140536495435775, ++STORE, 94652265619456, 94652265754623, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721814200320, 140737488351231, ++SNULL, 140721814208511, 140737488351231, ++STORE, 140721814200320, 140721814208511, ++STORE, 140721814069248, 140721814208511, ++STORE, 94062800691200, 94062802915327, ++SNULL, 94062800801791, 94062802915327, ++STORE, 94062800691200, 94062800801791, ++STORE, 94062800801792, 94062802915327, ++ERASE, 94062800801792, 94062802915327, ++STORE, 94062802894848, 94062802907135, ++STORE, 94062802907136, 94062802915327, ++STORE, 139717739700224, 139717741953023, ++SNULL, 139717739843583, 139717741953023, ++STORE, 139717739700224, 139717739843583, ++STORE, 139717739843584, 139717741953023, ++ERASE, 139717739843584, 139717741953023, ++STORE, 139717741940736, 139717741948927, ++STORE, 139717741948928, 139717741953023, ++STORE, 140721814224896, 140721814228991, ++STORE, 140721814212608, 140721814224895, ++STORE, 139717741912064, 139717741940735, ++STORE, 139717741903872, 139717741912063, ++STORE, 139717735903232, 139717739700223, ++SNULL, 139717735903232, 139717737562111, ++STORE, 139717737562112, 139717739700223, ++STORE, 139717735903232, 139717737562111, ++SNULL, 139717739659263, 139717739700223, ++STORE, 139717737562112, 139717739659263, ++STORE, 139717739659264, 139717739700223, ++SNULL, 139717739659264, 139717739683839, ++STORE, 139717739683840, 139717739700223, ++STORE, 139717739659264, 139717739683839, ++ERASE, 139717739659264, 139717739683839, ++STORE, 139717739659264, 139717739683839, ++ERASE, 139717739683840, 139717739700223, ++STORE, 139717739683840, 139717739700223, ++SNULL, 139717739675647, 139717739683839, ++STORE, 139717739659264, 139717739675647, ++STORE, 139717739675648, 139717739683839, ++SNULL, 94062802903039, 94062802907135, ++STORE, 94062802894848, 94062802903039, ++STORE, 94062802903040, 94062802907135, ++SNULL, 139717741944831, 139717741948927, ++STORE, 139717741940736, 139717741944831, ++STORE, 139717741944832, 139717741948927, ++ERASE, 139717741912064, 139717741940735, ++STORE, 94062814060544, 94062814195711, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723945754624, 140737488351231, ++SNULL, 140723945762815, 140737488351231, ++STORE, 140723945754624, 140723945762815, ++STORE, 140723945623552, 140723945762815, ++STORE, 94886119305216, 94886121639935, ++SNULL, 94886119518207, 94886121639935, ++STORE, 94886119305216, 94886119518207, ++STORE, 94886119518208, 94886121639935, ++ERASE, 94886119518208, 94886121639935, ++STORE, 94886121615360, 94886121627647, ++STORE, 94886121627648, 94886121639935, ++STORE, 140152532131840, 140152534384639, ++SNULL, 140152532275199, 140152534384639, ++STORE, 140152532131840, 140152532275199, ++STORE, 140152532275200, 140152534384639, ++ERASE, 140152532275200, 140152534384639, ++STORE, 140152534372352, 140152534380543, ++STORE, 140152534380544, 140152534384639, ++STORE, 140723946213376, 140723946217471, ++STORE, 140723946201088, 140723946213375, ++STORE, 140152534343680, 140152534372351, ++STORE, 140152534335488, 140152534343679, ++STORE, 140152530018304, 140152532131839, ++SNULL, 140152530018304, 140152530030591, ++STORE, 140152530030592, 140152532131839, ++STORE, 140152530018304, 140152530030591, ++SNULL, 140152532123647, 140152532131839, ++STORE, 140152530030592, 140152532123647, ++STORE, 140152532123648, 140152532131839, ++ERASE, 140152532123648, 140152532131839, ++STORE, 140152532123648, 140152532131839, ++STORE, 140152526221312, 140152530018303, ++SNULL, 140152526221312, 140152527880191, ++STORE, 140152527880192, 140152530018303, ++STORE, 140152526221312, 140152527880191, ++SNULL, 140152529977343, 140152530018303, ++STORE, 140152527880192, 140152529977343, ++STORE, 140152529977344, 140152530018303, ++SNULL, 140152529977344, 140152530001919, ++STORE, 140152530001920, 140152530018303, ++STORE, 140152529977344, 140152530001919, ++ERASE, 140152529977344, 140152530001919, ++STORE, 140152529977344, 140152530001919, ++ERASE, 140152530001920, 140152530018303, ++STORE, 140152530001920, 140152530018303, ++STORE, 140152534327296, 140152534343679, ++SNULL, 140152529993727, 140152530001919, ++STORE, 140152529977344, 140152529993727, ++STORE, 140152529993728, 140152530001919, ++SNULL, 140152532127743, 140152532131839, ++STORE, 140152532123648, 140152532127743, ++STORE, 140152532127744, 140152532131839, ++SNULL, 94886121619455, 94886121627647, ++STORE, 94886121615360, 94886121619455, ++STORE, 94886121619456, 94886121627647, ++SNULL, 140152534376447, 140152534380543, ++STORE, 140152534372352, 140152534376447, ++STORE, 140152534376448, 140152534380543, ++ERASE, 140152534343680, 140152534372351, ++STORE, 94886129770496, 94886129905663, ++STORE, 140152532643840, 140152534327295, ++STORE, 94886129770496, 94886130040831, ++STORE, 94886129770496, 94886130175999, ++STORE, 94886129770496, 94886130348031, ++SNULL, 94886130323455, 94886130348031, ++STORE, 94886129770496, 94886130323455, ++STORE, 94886130323456, 94886130348031, ++ERASE, 94886130323456, 94886130348031, ++STORE, 94886129770496, 94886130458623, ++STORE, 94886129770496, 94886130606079, ++SNULL, 94886130573311, 94886130606079, ++STORE, 94886129770496, 94886130573311, ++STORE, 94886130573312, 94886130606079, ++ERASE, 94886130573312, 94886130606079, ++STORE, 94886129770496, 94886130724863, ++STORE, 94886129770496, 94886130876415, ++STORE, 94886129770496, 94886131023871, ++STORE, 94886129770496, 94886131175423, ++STORE, 94886129770496, 94886131318783, ++STORE, 94886129770496, 94886131453951, ++SNULL, 94886131449855, 94886131453951, ++STORE, 94886129770496, 94886131449855, ++STORE, 94886131449856, 94886131453951, ++ERASE, 94886131449856, 94886131453951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735450779648, 140737488351231, ++SNULL, 140735450787839, 140737488351231, ++STORE, 140735450779648, 140735450787839, ++STORE, 140735450648576, 140735450787839, ++STORE, 93947794079744, 93947796414463, ++SNULL, 93947794292735, 93947796414463, ++STORE, 93947794079744, 93947794292735, ++STORE, 93947794292736, 93947796414463, ++ERASE, 93947794292736, 93947796414463, ++STORE, 93947796389888, 93947796402175, ++STORE, 93947796402176, 93947796414463, ++STORE, 139841993433088, 139841995685887, ++SNULL, 139841993576447, 139841995685887, ++STORE, 139841993433088, 139841993576447, ++STORE, 139841993576448, 139841995685887, ++ERASE, 139841993576448, 139841995685887, ++STORE, 139841995673600, 139841995681791, ++STORE, 139841995681792, 139841995685887, ++STORE, 140735451308032, 140735451312127, ++STORE, 140735451295744, 140735451308031, ++STORE, 139841995644928, 139841995673599, ++STORE, 139841995636736, 139841995644927, ++STORE, 139841991319552, 139841993433087, ++SNULL, 139841991319552, 139841991331839, ++STORE, 139841991331840, 139841993433087, ++STORE, 139841991319552, 139841991331839, ++SNULL, 139841993424895, 139841993433087, ++STORE, 139841991331840, 139841993424895, ++STORE, 139841993424896, 139841993433087, ++ERASE, 139841993424896, 139841993433087, ++STORE, 139841993424896, 139841993433087, ++STORE, 139841987522560, 139841991319551, ++SNULL, 139841987522560, 139841989181439, ++STORE, 139841989181440, 139841991319551, ++STORE, 139841987522560, 139841989181439, ++SNULL, 139841991278591, 139841991319551, ++STORE, 139841989181440, 139841991278591, ++STORE, 139841991278592, 139841991319551, ++SNULL, 139841991278592, 139841991303167, ++STORE, 139841991303168, 139841991319551, ++STORE, 139841991278592, 139841991303167, ++ERASE, 139841991278592, 139841991303167, ++STORE, 139841991278592, 139841991303167, ++ERASE, 139841991303168, 139841991319551, ++STORE, 139841991303168, 139841991319551, ++STORE, 139841995628544, 139841995644927, ++SNULL, 139841991294975, 139841991303167, ++STORE, 139841991278592, 139841991294975, ++STORE, 139841991294976, 139841991303167, ++SNULL, 139841993428991, 139841993433087, ++STORE, 139841993424896, 139841993428991, ++STORE, 139841993428992, 139841993433087, ++SNULL, 93947796393983, 93947796402175, ++STORE, 93947796389888, 93947796393983, ++STORE, 93947796393984, 93947796402175, ++SNULL, 139841995677695, 139841995681791, ++STORE, 139841995673600, 139841995677695, ++STORE, 139841995677696, 139841995681791, ++ERASE, 139841995644928, 139841995673599, ++STORE, 93947829739520, 93947829874687, ++STORE, 139841993945088, 139841995628543, ++STORE, 93947829739520, 93947830009855, ++STORE, 93947829739520, 93947830145023, ++STORE, 94659351814144, 94659352027135, ++STORE, 94659354124288, 94659354128383, ++STORE, 94659354128384, 94659354136575, ++STORE, 94659354136576, 94659354148863, ++STORE, 94659383476224, 94659385057279, ++STORE, 139959054557184, 139959056216063, ++STORE, 139959056216064, 139959058313215, ++STORE, 139959058313216, 139959058329599, ++STORE, 139959058329600, 139959058337791, ++STORE, 139959058337792, 139959058354175, ++STORE, 139959058354176, 139959058366463, ++STORE, 139959058366464, 139959060459519, ++STORE, 139959060459520, 139959060463615, ++STORE, 139959060463616, 139959060467711, ++STORE, 139959060467712, 139959060611071, ++STORE, 139959060979712, 139959062663167, ++STORE, 139959062663168, 139959062679551, ++STORE, 139959062708224, 139959062712319, ++STORE, 139959062712320, 139959062716415, ++STORE, 139959062716416, 139959062720511, ++STORE, 140735532539904, 140735532679167, ++STORE, 140735532830720, 140735532843007, ++STORE, 140735532843008, 140735532847103, ++STORE, 93894361829376, 93894362042367, ++STORE, 93894364139520, 93894364143615, ++STORE, 93894364143616, 93894364151807, ++STORE, 93894364151808, 93894364164095, ++STORE, 93894396944384, 93894397624319, ++STORE, 140075612573696, 140075614232575, ++STORE, 140075614232576, 140075616329727, ++STORE, 140075616329728, 140075616346111, ++STORE, 140075616346112, 140075616354303, ++STORE, 140075616354304, 140075616370687, ++STORE, 140075616370688, 140075616382975, ++STORE, 140075616382976, 140075618476031, ++STORE, 140075618476032, 140075618480127, ++STORE, 140075618480128, 140075618484223, ++STORE, 140075618484224, 140075618627583, ++STORE, 140075618996224, 140075620679679, ++STORE, 140075620679680, 140075620696063, ++STORE, 140075620724736, 140075620728831, ++STORE, 140075620728832, 140075620732927, ++STORE, 140075620732928, 140075620737023, ++STORE, 140720830312448, 140720830451711, ++STORE, 140720830631936, 140720830644223, ++STORE, 140720830644224, 140720830648319, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735116226560, 140737488351231, ++SNULL, 140735116234751, 140737488351231, ++STORE, 140735116226560, 140735116234751, ++STORE, 140735116095488, 140735116234751, ++STORE, 94873398054912, 94873400279039, ++SNULL, 94873398165503, 94873400279039, ++STORE, 94873398054912, 94873398165503, ++STORE, 94873398165504, 94873400279039, ++ERASE, 94873398165504, 94873400279039, ++STORE, 94873400258560, 94873400270847, ++STORE, 94873400270848, 94873400279039, ++STORE, 140303828606976, 140303830859775, ++SNULL, 140303828750335, 140303830859775, ++STORE, 140303828606976, 140303828750335, ++STORE, 140303828750336, 140303830859775, ++ERASE, 140303828750336, 140303830859775, ++STORE, 140303830847488, 140303830855679, ++STORE, 140303830855680, 140303830859775, ++STORE, 140735116251136, 140735116255231, ++STORE, 140735116238848, 140735116251135, ++STORE, 140303830818816, 140303830847487, ++STORE, 140303830810624, 140303830818815, ++STORE, 140303824809984, 140303828606975, ++SNULL, 140303824809984, 140303826468863, ++STORE, 140303826468864, 140303828606975, ++STORE, 140303824809984, 140303826468863, ++SNULL, 140303828566015, 140303828606975, ++STORE, 140303826468864, 140303828566015, ++STORE, 140303828566016, 140303828606975, ++SNULL, 140303828566016, 140303828590591, ++STORE, 140303828590592, 140303828606975, ++STORE, 140303828566016, 140303828590591, ++ERASE, 140303828566016, 140303828590591, ++STORE, 140303828566016, 140303828590591, ++ERASE, 140303828590592, 140303828606975, ++STORE, 140303828590592, 140303828606975, ++SNULL, 140303828582399, 140303828590591, ++STORE, 140303828566016, 140303828582399, ++STORE, 140303828582400, 140303828590591, ++SNULL, 94873400266751, 94873400270847, ++STORE, 94873400258560, 94873400266751, ++STORE, 94873400266752, 94873400270847, ++SNULL, 140303830851583, 140303830855679, ++STORE, 140303830847488, 140303830851583, ++STORE, 140303830851584, 140303830855679, ++ERASE, 140303830818816, 140303830847487, ++STORE, 94873413713920, 94873413849087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732349956096, 140737488351231, ++SNULL, 140732349964287, 140737488351231, ++STORE, 140732349956096, 140732349964287, ++STORE, 140732349825024, 140732349964287, ++STORE, 94009652736000, 94009655070719, ++SNULL, 94009652948991, 94009655070719, ++STORE, 94009652736000, 94009652948991, ++STORE, 94009652948992, 94009655070719, ++ERASE, 94009652948992, 94009655070719, ++STORE, 94009655046144, 94009655058431, ++STORE, 94009655058432, 94009655070719, ++STORE, 140295688531968, 140295690784767, ++SNULL, 140295688675327, 140295690784767, ++STORE, 140295688531968, 140295688675327, ++STORE, 140295688675328, 140295690784767, ++ERASE, 140295688675328, 140295690784767, ++STORE, 140295690772480, 140295690780671, ++STORE, 140295690780672, 140295690784767, ++STORE, 140732350005248, 140732350009343, ++STORE, 140732349992960, 140732350005247, ++STORE, 140295690743808, 140295690772479, ++STORE, 140295690735616, 140295690743807, ++STORE, 140295686418432, 140295688531967, ++SNULL, 140295686418432, 140295686430719, ++STORE, 140295686430720, 140295688531967, ++STORE, 140295686418432, 140295686430719, ++SNULL, 140295688523775, 140295688531967, ++STORE, 140295686430720, 140295688523775, ++STORE, 140295688523776, 140295688531967, ++ERASE, 140295688523776, 140295688531967, ++STORE, 140295688523776, 140295688531967, ++STORE, 140295682621440, 140295686418431, ++SNULL, 140295682621440, 140295684280319, ++STORE, 140295684280320, 140295686418431, ++STORE, 140295682621440, 140295684280319, ++SNULL, 140295686377471, 140295686418431, ++STORE, 140295684280320, 140295686377471, ++STORE, 140295686377472, 140295686418431, ++SNULL, 140295686377472, 140295686402047, ++STORE, 140295686402048, 140295686418431, ++STORE, 140295686377472, 140295686402047, ++ERASE, 140295686377472, 140295686402047, ++STORE, 140295686377472, 140295686402047, ++ERASE, 140295686402048, 140295686418431, ++STORE, 140295686402048, 140295686418431, ++STORE, 140295690727424, 140295690743807, ++SNULL, 140295686393855, 140295686402047, ++STORE, 140295686377472, 140295686393855, ++STORE, 140295686393856, 140295686402047, ++SNULL, 140295688527871, 140295688531967, ++STORE, 140295688523776, 140295688527871, ++STORE, 140295688527872, 140295688531967, ++SNULL, 94009655050239, 94009655058431, ++STORE, 94009655046144, 94009655050239, ++STORE, 94009655050240, 94009655058431, ++SNULL, 140295690776575, 140295690780671, ++STORE, 140295690772480, 140295690776575, ++STORE, 140295690776576, 140295690780671, ++ERASE, 140295690743808, 140295690772479, ++STORE, 94009672114176, 94009672249343, ++STORE, 140295689043968, 140295690727423, ++STORE, 94009672114176, 94009672384511, ++STORE, 94009672114176, 94009672519679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722376515584, 140737488351231, ++SNULL, 140722376523775, 140737488351231, ++STORE, 140722376515584, 140722376523775, ++STORE, 140722376384512, 140722376523775, ++STORE, 94089815773184, 94089818107903, ++SNULL, 94089815986175, 94089818107903, ++STORE, 94089815773184, 94089815986175, ++STORE, 94089815986176, 94089818107903, ++ERASE, 94089815986176, 94089818107903, ++STORE, 94089818083328, 94089818095615, ++STORE, 94089818095616, 94089818107903, ++STORE, 140265595711488, 140265597964287, ++SNULL, 140265595854847, 140265597964287, ++STORE, 140265595711488, 140265595854847, ++STORE, 140265595854848, 140265597964287, ++ERASE, 140265595854848, 140265597964287, ++STORE, 140265597952000, 140265597960191, ++STORE, 140265597960192, 140265597964287, ++STORE, 140722378297344, 140722378301439, ++STORE, 140722378285056, 140722378297343, ++STORE, 140265597923328, 140265597951999, ++STORE, 140265597915136, 140265597923327, ++STORE, 140265593597952, 140265595711487, ++SNULL, 140265593597952, 140265593610239, ++STORE, 140265593610240, 140265595711487, ++STORE, 140265593597952, 140265593610239, ++SNULL, 140265595703295, 140265595711487, ++STORE, 140265593610240, 140265595703295, ++STORE, 140265595703296, 140265595711487, ++ERASE, 140265595703296, 140265595711487, ++STORE, 140265595703296, 140265595711487, ++STORE, 140265589800960, 140265593597951, ++SNULL, 140265589800960, 140265591459839, ++STORE, 140265591459840, 140265593597951, ++STORE, 140265589800960, 140265591459839, ++SNULL, 140265593556991, 140265593597951, ++STORE, 140265591459840, 140265593556991, ++STORE, 140265593556992, 140265593597951, ++SNULL, 140265593556992, 140265593581567, ++STORE, 140265593581568, 140265593597951, ++STORE, 140265593556992, 140265593581567, ++ERASE, 140265593556992, 140265593581567, ++STORE, 140265593556992, 140265593581567, ++ERASE, 140265593581568, 140265593597951, ++STORE, 140265593581568, 140265593597951, ++STORE, 140265597906944, 140265597923327, ++SNULL, 140265593573375, 140265593581567, ++STORE, 140265593556992, 140265593573375, ++STORE, 140265593573376, 140265593581567, ++SNULL, 140265595707391, 140265595711487, ++STORE, 140265595703296, 140265595707391, ++STORE, 140265595707392, 140265595711487, ++SNULL, 94089818087423, 94089818095615, ++STORE, 94089818083328, 94089818087423, ++STORE, 94089818087424, 94089818095615, ++SNULL, 140265597956095, 140265597960191, ++STORE, 140265597952000, 140265597956095, ++STORE, 140265597956096, 140265597960191, ++ERASE, 140265597923328, 140265597951999, ++STORE, 94089837146112, 94089837281279, ++STORE, 140265596223488, 140265597906943, ++STORE, 94089837146112, 94089837416447, ++STORE, 94089837146112, 94089837551615, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735265218560, 140737488351231, ++SNULL, 140735265226751, 140737488351231, ++STORE, 140735265218560, 140735265226751, ++STORE, 140735265087488, 140735265226751, ++STORE, 94250422370304, 94250424705023, ++SNULL, 94250422583295, 94250424705023, ++STORE, 94250422370304, 94250422583295, ++STORE, 94250422583296, 94250424705023, ++ERASE, 94250422583296, 94250424705023, ++STORE, 94250424680448, 94250424692735, ++STORE, 94250424692736, 94250424705023, ++STORE, 140344442474496, 140344444727295, ++SNULL, 140344442617855, 140344444727295, ++STORE, 140344442474496, 140344442617855, ++STORE, 140344442617856, 140344444727295, ++ERASE, 140344442617856, 140344444727295, ++STORE, 140344444715008, 140344444723199, ++STORE, 140344444723200, 140344444727295, ++STORE, 140735265341440, 140735265345535, ++STORE, 140735265329152, 140735265341439, ++STORE, 140344444686336, 140344444715007, ++STORE, 140344444678144, 140344444686335, ++STORE, 140344440360960, 140344442474495, ++SNULL, 140344440360960, 140344440373247, ++STORE, 140344440373248, 140344442474495, ++STORE, 140344440360960, 140344440373247, ++SNULL, 140344442466303, 140344442474495, ++STORE, 140344440373248, 140344442466303, ++STORE, 140344442466304, 140344442474495, ++ERASE, 140344442466304, 140344442474495, ++STORE, 140344442466304, 140344442474495, ++STORE, 140344436563968, 140344440360959, ++SNULL, 140344436563968, 140344438222847, ++STORE, 140344438222848, 140344440360959, ++STORE, 140344436563968, 140344438222847, ++SNULL, 140344440319999, 140344440360959, ++STORE, 140344438222848, 140344440319999, ++STORE, 140344440320000, 140344440360959, ++SNULL, 140344440320000, 140344440344575, ++STORE, 140344440344576, 140344440360959, ++STORE, 140344440320000, 140344440344575, ++ERASE, 140344440320000, 140344440344575, ++STORE, 140344440320000, 140344440344575, ++ERASE, 140344440344576, 140344440360959, ++STORE, 140344440344576, 140344440360959, ++STORE, 140344444669952, 140344444686335, ++SNULL, 140344440336383, 140344440344575, ++STORE, 140344440320000, 140344440336383, ++STORE, 140344440336384, 140344440344575, ++SNULL, 140344442470399, 140344442474495, ++STORE, 140344442466304, 140344442470399, ++STORE, 140344442470400, 140344442474495, ++SNULL, 94250424684543, 94250424692735, ++STORE, 94250424680448, 94250424684543, ++STORE, 94250424684544, 94250424692735, ++SNULL, 140344444719103, 140344444723199, ++STORE, 140344444715008, 140344444719103, ++STORE, 140344444719104, 140344444723199, ++ERASE, 140344444686336, 140344444715007, ++STORE, 94250445512704, 94250445647871, ++STORE, 140344442986496, 140344444669951, ++STORE, 94250445512704, 94250445783039, ++STORE, 94250445512704, 94250445918207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725762719744, 140737488351231, ++SNULL, 140725762727935, 140737488351231, ++STORE, 140725762719744, 140725762727935, ++STORE, 140725762588672, 140725762727935, ++STORE, 94819009097728, 94819011432447, ++SNULL, 94819009310719, 94819011432447, ++STORE, 94819009097728, 94819009310719, ++STORE, 94819009310720, 94819011432447, ++ERASE, 94819009310720, 94819011432447, ++STORE, 94819011407872, 94819011420159, ++STORE, 94819011420160, 94819011432447, ++STORE, 139987985596416, 139987987849215, ++SNULL, 139987985739775, 139987987849215, ++STORE, 139987985596416, 139987985739775, ++STORE, 139987985739776, 139987987849215, ++ERASE, 139987985739776, 139987987849215, ++STORE, 139987987836928, 139987987845119, ++STORE, 139987987845120, 139987987849215, ++STORE, 140725763072000, 140725763076095, ++STORE, 140725763059712, 140725763071999, ++STORE, 139987987808256, 139987987836927, ++STORE, 139987987800064, 139987987808255, ++STORE, 139987983482880, 139987985596415, ++SNULL, 139987983482880, 139987983495167, ++STORE, 139987983495168, 139987985596415, ++STORE, 139987983482880, 139987983495167, ++SNULL, 139987985588223, 139987985596415, ++STORE, 139987983495168, 139987985588223, ++STORE, 139987985588224, 139987985596415, ++ERASE, 139987985588224, 139987985596415, ++STORE, 139987985588224, 139987985596415, ++STORE, 139987979685888, 139987983482879, ++SNULL, 139987979685888, 139987981344767, ++STORE, 139987981344768, 139987983482879, ++STORE, 139987979685888, 139987981344767, ++SNULL, 139987983441919, 139987983482879, ++STORE, 139987981344768, 139987983441919, ++STORE, 139987983441920, 139987983482879, ++SNULL, 139987983441920, 139987983466495, ++STORE, 139987983466496, 139987983482879, ++STORE, 139987983441920, 139987983466495, ++ERASE, 139987983441920, 139987983466495, ++STORE, 139987983441920, 139987983466495, ++ERASE, 139987983466496, 139987983482879, ++STORE, 139987983466496, 139987983482879, ++STORE, 139987987791872, 139987987808255, ++SNULL, 139987983458303, 139987983466495, ++STORE, 139987983441920, 139987983458303, ++STORE, 139987983458304, 139987983466495, ++SNULL, 139987985592319, 139987985596415, ++STORE, 139987985588224, 139987985592319, ++STORE, 139987985592320, 139987985596415, ++SNULL, 94819011411967, 94819011420159, ++STORE, 94819011407872, 94819011411967, ++STORE, 94819011411968, 94819011420159, ++SNULL, 139987987841023, 139987987845119, ++STORE, 139987987836928, 139987987841023, ++STORE, 139987987841024, 139987987845119, ++ERASE, 139987987808256, 139987987836927, ++STORE, 94819028176896, 94819028312063, ++STORE, 139987986108416, 139987987791871, ++STORE, 94819028176896, 94819028447231, ++STORE, 94819028176896, 94819028582399, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722475413504, 140737488351231, ++SNULL, 140722475421695, 140737488351231, ++STORE, 140722475413504, 140722475421695, ++STORE, 140722475282432, 140722475421695, ++STORE, 94620599119872, 94620601343999, ++SNULL, 94620599230463, 94620601343999, ++STORE, 94620599119872, 94620599230463, ++STORE, 94620599230464, 94620601343999, ++ERASE, 94620599230464, 94620601343999, ++STORE, 94620601323520, 94620601335807, ++STORE, 94620601335808, 94620601343999, ++STORE, 139891763060736, 139891765313535, ++SNULL, 139891763204095, 139891765313535, ++STORE, 139891763060736, 139891763204095, ++STORE, 139891763204096, 139891765313535, ++ERASE, 139891763204096, 139891765313535, ++STORE, 139891765301248, 139891765309439, ++STORE, 139891765309440, 139891765313535, ++STORE, 140722475700224, 140722475704319, ++STORE, 140722475687936, 140722475700223, ++STORE, 139891765272576, 139891765301247, ++STORE, 139891765264384, 139891765272575, ++STORE, 139891759263744, 139891763060735, ++SNULL, 139891759263744, 139891760922623, ++STORE, 139891760922624, 139891763060735, ++STORE, 139891759263744, 139891760922623, ++SNULL, 139891763019775, 139891763060735, ++STORE, 139891760922624, 139891763019775, ++STORE, 139891763019776, 139891763060735, ++SNULL, 139891763019776, 139891763044351, ++STORE, 139891763044352, 139891763060735, ++STORE, 139891763019776, 139891763044351, ++ERASE, 139891763019776, 139891763044351, ++STORE, 139891763019776, 139891763044351, ++ERASE, 139891763044352, 139891763060735, ++STORE, 139891763044352, 139891763060735, ++SNULL, 139891763036159, 139891763044351, ++STORE, 139891763019776, 139891763036159, ++STORE, 139891763036160, 139891763044351, ++SNULL, 94620601331711, 94620601335807, ++STORE, 94620601323520, 94620601331711, ++STORE, 94620601331712, 94620601335807, ++SNULL, 139891765305343, 139891765309439, ++STORE, 139891765301248, 139891765305343, ++STORE, 139891765305344, 139891765309439, ++ERASE, 139891765272576, 139891765301247, ++STORE, 94620610027520, 94620610162687, ++STORE, 94031976210432, 94031976423423, ++STORE, 94031978520576, 94031978524671, ++STORE, 94031978524672, 94031978532863, ++STORE, 94031978532864, 94031978545151, ++STORE, 94031990398976, 94031992565759, ++STORE, 140336240640000, 140336242298879, ++STORE, 140336242298880, 140336244396031, ++STORE, 140336244396032, 140336244412415, ++STORE, 140336244412416, 140336244420607, ++STORE, 140336244420608, 140336244436991, ++STORE, 140336244436992, 140336244449279, ++STORE, 140336244449280, 140336246542335, ++STORE, 140336246542336, 140336246546431, ++STORE, 140336246546432, 140336246550527, ++STORE, 140336246550528, 140336246693887, ++STORE, 140336247062528, 140336248745983, ++STORE, 140336248745984, 140336248762367, ++STORE, 140336248791040, 140336248795135, ++STORE, 140336248795136, 140336248799231, ++STORE, 140336248799232, 140336248803327, ++STORE, 140728500064256, 140728500203519, ++STORE, 140728501501952, 140728501514239, ++STORE, 140728501514240, 140728501518335, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730503987200, 140737488351231, ++SNULL, 140730503995391, 140737488351231, ++STORE, 140730503987200, 140730503995391, ++STORE, 140730503856128, 140730503995391, ++STORE, 93866544205824, 93866546429951, ++SNULL, 93866544316415, 93866546429951, ++STORE, 93866544205824, 93866544316415, ++STORE, 93866544316416, 93866546429951, ++ERASE, 93866544316416, 93866546429951, ++STORE, 93866546409472, 93866546421759, ++STORE, 93866546421760, 93866546429951, ++STORE, 140216311959552, 140216314212351, ++SNULL, 140216312102911, 140216314212351, ++STORE, 140216311959552, 140216312102911, ++STORE, 140216312102912, 140216314212351, ++ERASE, 140216312102912, 140216314212351, ++STORE, 140216314200064, 140216314208255, ++STORE, 140216314208256, 140216314212351, ++STORE, 140730504626176, 140730504630271, ++STORE, 140730504613888, 140730504626175, ++STORE, 140216314171392, 140216314200063, ++STORE, 140216314163200, 140216314171391, ++STORE, 140216308162560, 140216311959551, ++SNULL, 140216308162560, 140216309821439, ++STORE, 140216309821440, 140216311959551, ++STORE, 140216308162560, 140216309821439, ++SNULL, 140216311918591, 140216311959551, ++STORE, 140216309821440, 140216311918591, ++STORE, 140216311918592, 140216311959551, ++SNULL, 140216311918592, 140216311943167, ++STORE, 140216311943168, 140216311959551, ++STORE, 140216311918592, 140216311943167, ++ERASE, 140216311918592, 140216311943167, ++STORE, 140216311918592, 140216311943167, ++ERASE, 140216311943168, 140216311959551, ++STORE, 140216311943168, 140216311959551, ++SNULL, 140216311934975, 140216311943167, ++STORE, 140216311918592, 140216311934975, ++STORE, 140216311934976, 140216311943167, ++SNULL, 93866546417663, 93866546421759, ++STORE, 93866546409472, 93866546417663, ++STORE, 93866546417664, 93866546421759, ++SNULL, 140216314204159, 140216314208255, ++STORE, 140216314200064, 140216314204159, ++STORE, 140216314204160, 140216314208255, ++ERASE, 140216314171392, 140216314200063, ++STORE, 93866550386688, 93866550521855, ++STORE, 94074292674560, 94074292887551, ++STORE, 94074294984704, 94074294988799, ++STORE, 94074294988800, 94074294996991, ++STORE, 94074294996992, 94074295009279, ++STORE, 94074300219392, 94074301378559, ++STORE, 139781563256832, 139781564915711, ++STORE, 139781564915712, 139781567012863, ++STORE, 139781567012864, 139781567029247, ++STORE, 139781567029248, 139781567037439, ++STORE, 139781567037440, 139781567053823, ++STORE, 139781567053824, 139781567066111, ++STORE, 139781567066112, 139781569159167, ++STORE, 139781569159168, 139781569163263, ++STORE, 139781569163264, 139781569167359, ++STORE, 139781569167360, 139781569310719, ++STORE, 139781569679360, 139781571362815, ++STORE, 139781571362816, 139781571379199, ++STORE, 139781571407872, 139781571411967, ++STORE, 139781571411968, 139781571416063, ++STORE, 139781571416064, 139781571420159, ++STORE, 140723688488960, 140723688628223, ++STORE, 140723689005056, 140723689017343, ++STORE, 140723689017344, 140723689021439, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735189745664, 140737488351231, ++SNULL, 140735189753855, 140737488351231, ++STORE, 140735189745664, 140735189753855, ++STORE, 140735189614592, 140735189753855, ++STORE, 94172072177664, 94172074512383, ++SNULL, 94172072390655, 94172074512383, ++STORE, 94172072177664, 94172072390655, ++STORE, 94172072390656, 94172074512383, ++ERASE, 94172072390656, 94172074512383, ++STORE, 94172074487808, 94172074500095, ++STORE, 94172074500096, 94172074512383, ++STORE, 140687827263488, 140687829516287, ++SNULL, 140687827406847, 140687829516287, ++STORE, 140687827263488, 140687827406847, ++STORE, 140687827406848, 140687829516287, ++ERASE, 140687827406848, 140687829516287, ++STORE, 140687829504000, 140687829512191, ++STORE, 140687829512192, 140687829516287, ++STORE, 140735189766144, 140735189770239, ++STORE, 140735189753856, 140735189766143, ++STORE, 140687829475328, 140687829503999, ++STORE, 140687829467136, 140687829475327, ++STORE, 140687825149952, 140687827263487, ++SNULL, 140687825149952, 140687825162239, ++STORE, 140687825162240, 140687827263487, ++STORE, 140687825149952, 140687825162239, ++SNULL, 140687827255295, 140687827263487, ++STORE, 140687825162240, 140687827255295, ++STORE, 140687827255296, 140687827263487, ++ERASE, 140687827255296, 140687827263487, ++STORE, 140687827255296, 140687827263487, ++STORE, 140687821352960, 140687825149951, ++SNULL, 140687821352960, 140687823011839, ++STORE, 140687823011840, 140687825149951, ++STORE, 140687821352960, 140687823011839, ++SNULL, 140687825108991, 140687825149951, ++STORE, 140687823011840, 140687825108991, ++STORE, 140687825108992, 140687825149951, ++SNULL, 140687825108992, 140687825133567, ++STORE, 140687825133568, 140687825149951, ++STORE, 140687825108992, 140687825133567, ++ERASE, 140687825108992, 140687825133567, ++STORE, 140687825108992, 140687825133567, ++ERASE, 140687825133568, 140687825149951, ++STORE, 140687825133568, 140687825149951, ++STORE, 140687829458944, 140687829475327, ++SNULL, 140687825125375, 140687825133567, ++STORE, 140687825108992, 140687825125375, ++STORE, 140687825125376, 140687825133567, ++SNULL, 140687827259391, 140687827263487, ++STORE, 140687827255296, 140687827259391, ++STORE, 140687827259392, 140687827263487, ++SNULL, 94172074491903, 94172074500095, ++STORE, 94172074487808, 94172074491903, ++STORE, 94172074491904, 94172074500095, ++SNULL, 140687829508095, 140687829512191, ++STORE, 140687829504000, 140687829508095, ++STORE, 140687829508096, 140687829512191, ++ERASE, 140687829475328, 140687829503999, ++STORE, 94172092432384, 94172092567551, ++STORE, 140687827775488, 140687829458943, ++STORE, 94172092432384, 94172092702719, ++STORE, 94172092432384, 94172092837887, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737229504512, 140737488351231, ++SNULL, 140737229512703, 140737488351231, ++STORE, 140737229504512, 140737229512703, ++STORE, 140737229373440, 140737229512703, ++STORE, 94155246866432, 94155249090559, ++SNULL, 94155246977023, 94155249090559, ++STORE, 94155246866432, 94155246977023, ++STORE, 94155246977024, 94155249090559, ++ERASE, 94155246977024, 94155249090559, ++STORE, 94155249070080, 94155249082367, ++STORE, 94155249082368, 94155249090559, ++STORE, 140640993693696, 140640995946495, ++SNULL, 140640993837055, 140640995946495, ++STORE, 140640993693696, 140640993837055, ++STORE, 140640993837056, 140640995946495, ++ERASE, 140640993837056, 140640995946495, ++STORE, 140640995934208, 140640995942399, ++STORE, 140640995942400, 140640995946495, ++STORE, 140737230004224, 140737230008319, ++STORE, 140737229991936, 140737230004223, ++STORE, 140640995905536, 140640995934207, ++STORE, 140640995897344, 140640995905535, ++STORE, 140640989896704, 140640993693695, ++SNULL, 140640989896704, 140640991555583, ++STORE, 140640991555584, 140640993693695, ++STORE, 140640989896704, 140640991555583, ++SNULL, 140640993652735, 140640993693695, ++STORE, 140640991555584, 140640993652735, ++STORE, 140640993652736, 140640993693695, ++SNULL, 140640993652736, 140640993677311, ++STORE, 140640993677312, 140640993693695, ++STORE, 140640993652736, 140640993677311, ++ERASE, 140640993652736, 140640993677311, ++STORE, 140640993652736, 140640993677311, ++ERASE, 140640993677312, 140640993693695, ++STORE, 140640993677312, 140640993693695, ++SNULL, 140640993669119, 140640993677311, ++STORE, 140640993652736, 140640993669119, ++STORE, 140640993669120, 140640993677311, ++SNULL, 94155249078271, 94155249082367, ++STORE, 94155249070080, 94155249078271, ++STORE, 94155249078272, 94155249082367, ++SNULL, 140640995938303, 140640995942399, ++STORE, 140640995934208, 140640995938303, ++STORE, 140640995938304, 140640995942399, ++ERASE, 140640995905536, 140640995934207, ++STORE, 94155281035264, 94155281170431, ++STORE, 94088066453504, 94088066564095, ++STORE, 94088068657152, 94088068665343, ++STORE, 94088068665344, 94088068669439, ++STORE, 94088068669440, 94088068677631, ++STORE, 94088090214400, 94088090349567, ++STORE, 140503024627712, 140503026286591, ++STORE, 140503026286592, 140503028383743, ++STORE, 140503028383744, 140503028400127, ++STORE, 140503028400128, 140503028408319, ++STORE, 140503028408320, 140503028424703, ++STORE, 140503028424704, 140503028568063, ++STORE, 140503030628352, 140503030636543, ++STORE, 140503030665216, 140503030669311, ++STORE, 140503030669312, 140503030673407, ++STORE, 140503030673408, 140503030677503, ++STORE, 140730894725120, 140730894864383, ++STORE, 140730894880768, 140730894893055, ++STORE, 140730894893056, 140730894897151, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730434342912, 140737488351231, ++SNULL, 140730434351103, 140737488351231, ++STORE, 140730434342912, 140730434351103, ++STORE, 140730434211840, 140730434351103, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140109041938432, 140109044191231, ++SNULL, 140109042081791, 140109044191231, ++STORE, 140109041938432, 140109042081791, ++STORE, 140109042081792, 140109044191231, ++ERASE, 140109042081792, 140109044191231, ++STORE, 140109044178944, 140109044187135, ++STORE, 140109044187136, 140109044191231, ++STORE, 140730434850816, 140730434854911, ++STORE, 140730434838528, 140730434850815, ++STORE, 140109044150272, 140109044178943, ++STORE, 140109044142080, 140109044150271, ++STORE, 140109038776320, 140109041938431, ++SNULL, 140109038776320, 140109039837183, ++STORE, 140109039837184, 140109041938431, ++STORE, 140109038776320, 140109039837183, ++SNULL, 140109041930239, 140109041938431, ++STORE, 140109039837184, 140109041930239, ++STORE, 140109041930240, 140109041938431, ++ERASE, 140109041930240, 140109041938431, ++STORE, 140109041930240, 140109041938431, ++STORE, 140109034979328, 140109038776319, ++SNULL, 140109034979328, 140109036638207, ++STORE, 140109036638208, 140109038776319, ++STORE, 140109034979328, 140109036638207, ++SNULL, 140109038735359, 140109038776319, ++STORE, 140109036638208, 140109038735359, ++STORE, 140109038735360, 140109038776319, ++SNULL, 140109038735360, 140109038759935, ++STORE, 140109038759936, 140109038776319, ++STORE, 140109038735360, 140109038759935, ++ERASE, 140109038735360, 140109038759935, ++STORE, 140109038735360, 140109038759935, ++ERASE, 140109038759936, 140109038776319, ++STORE, 140109038759936, 140109038776319, ++STORE, 140109044129792, 140109044150271, ++SNULL, 140109038751743, 140109038759935, ++STORE, 140109038735360, 140109038751743, ++STORE, 140109038751744, 140109038759935, ++SNULL, 140109041934335, 140109041938431, ++STORE, 140109041930240, 140109041934335, ++STORE, 140109041934336, 140109041938431, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140109044183039, 140109044187135, ++STORE, 140109044178944, 140109044183039, ++STORE, 140109044183040, 140109044187135, ++ERASE, 140109044150272, 140109044178943, ++STORE, 20000768, 20135935, ++STORE, 20000768, 20283391, ++STORE, 140109042446336, 140109044129791, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730853408768, 140737488351231, ++SNULL, 140730853416959, 140737488351231, ++STORE, 140730853408768, 140730853416959, ++STORE, 140730853277696, 140730853416959, ++STORE, 94865902977024, 94865905311743, ++SNULL, 94865903190015, 94865905311743, ++STORE, 94865902977024, 94865903190015, ++STORE, 94865903190016, 94865905311743, ++ERASE, 94865903190016, 94865905311743, ++STORE, 94865905287168, 94865905299455, ++STORE, 94865905299456, 94865905311743, ++STORE, 139768865738752, 139768867991551, ++SNULL, 139768865882111, 139768867991551, ++STORE, 139768865738752, 139768865882111, ++STORE, 139768865882112, 139768867991551, ++ERASE, 139768865882112, 139768867991551, ++STORE, 139768867979264, 139768867987455, ++STORE, 139768867987456, 139768867991551, ++STORE, 140730853957632, 140730853961727, ++STORE, 140730853945344, 140730853957631, ++STORE, 139768867950592, 139768867979263, ++STORE, 139768867942400, 139768867950591, ++STORE, 139768863625216, 139768865738751, ++SNULL, 139768863625216, 139768863637503, ++STORE, 139768863637504, 139768865738751, ++STORE, 139768863625216, 139768863637503, ++SNULL, 139768865730559, 139768865738751, ++STORE, 139768863637504, 139768865730559, ++STORE, 139768865730560, 139768865738751, ++ERASE, 139768865730560, 139768865738751, ++STORE, 139768865730560, 139768865738751, ++STORE, 139768859828224, 139768863625215, ++SNULL, 139768859828224, 139768861487103, ++STORE, 139768861487104, 139768863625215, ++STORE, 139768859828224, 139768861487103, ++SNULL, 139768863584255, 139768863625215, ++STORE, 139768861487104, 139768863584255, ++STORE, 139768863584256, 139768863625215, ++SNULL, 139768863584256, 139768863608831, ++STORE, 139768863608832, 139768863625215, ++STORE, 139768863584256, 139768863608831, ++ERASE, 139768863584256, 139768863608831, ++STORE, 139768863584256, 139768863608831, ++ERASE, 139768863608832, 139768863625215, ++STORE, 139768863608832, 139768863625215, ++STORE, 139768867934208, 139768867950591, ++SNULL, 139768863600639, 139768863608831, ++STORE, 139768863584256, 139768863600639, ++STORE, 139768863600640, 139768863608831, ++SNULL, 139768865734655, 139768865738751, ++STORE, 139768865730560, 139768865734655, ++STORE, 139768865734656, 139768865738751, ++SNULL, 94865905291263, 94865905299455, ++STORE, 94865905287168, 94865905291263, ++STORE, 94865905291264, 94865905299455, ++SNULL, 139768867983359, 139768867987455, ++STORE, 139768867979264, 139768867983359, ++STORE, 139768867983360, 139768867987455, ++ERASE, 139768867950592, 139768867979263, ++STORE, 94865923670016, 94865923805183, ++STORE, 139768866250752, 139768867934207, ++STORE, 94865923670016, 94865923940351, ++STORE, 94865923670016, 94865924075519, ++STORE, 94865923670016, 94865924222975, ++SNULL, 94865924210687, 94865924222975, ++STORE, 94865923670016, 94865924210687, ++STORE, 94865924210688, 94865924222975, ++ERASE, 94865924210688, 94865924222975, ++STORE, 94865923670016, 94865924349951, ++STORE, 94865923670016, 94865924493311, ++STORE, 94865923670016, 94865924640767, ++SNULL, 94865924603903, 94865924640767, ++STORE, 94865923670016, 94865924603903, ++STORE, 94865924603904, 94865924640767, ++ERASE, 94865924603904, 94865924640767, ++STORE, 94865923670016, 94865924747263, ++STORE, 94865923670016, 94865924898815, ++SNULL, 94865924874239, 94865924898815, ++STORE, 94865923670016, 94865924874239, ++STORE, 94865924874240, 94865924898815, ++ERASE, 94865924874240, 94865924898815, ++STORE, 94865923670016, 94865925025791, ++SNULL, 94865925013503, 94865925025791, ++STORE, 94865923670016, 94865925013503, ++STORE, 94865925013504, 94865925025791, ++ERASE, 94865925013504, 94865925025791, ++SNULL, 94865924988927, 94865925013503, ++STORE, 94865923670016, 94865924988927, ++STORE, 94865924988928, 94865925013503, ++ERASE, 94865924988928, 94865925013503, ++STORE, 94865923670016, 94865925152767, ++SNULL, 94865925136383, 94865925152767, ++STORE, 94865923670016, 94865925136383, ++STORE, 94865925136384, 94865925152767, ++ERASE, 94865925136384, 94865925152767, ++STORE, 94865923670016, 94865925292031, ++SNULL, 94865925279743, 94865925292031, ++STORE, 94865923670016, 94865925279743, ++STORE, 94865925279744, 94865925292031, ++ERASE, 94865925279744, 94865925292031, ++SNULL, 94865925255167, 94865925279743, ++STORE, 94865923670016, 94865925255167, ++STORE, 94865925255168, 94865925279743, ++ERASE, 94865925255168, 94865925279743, ++STORE, 94865923670016, 94865925406719, ++SNULL, 94865925394431, 94865925406719, ++STORE, 94865923670016, 94865925394431, ++STORE, 94865925394432, 94865925406719, ++ERASE, 94865925394432, 94865925406719, ++STORE, 94865923670016, 94865925545983, ++SNULL, 94865925533695, 94865925545983, ++STORE, 94865923670016, 94865925533695, ++STORE, 94865925533696, 94865925545983, ++ERASE, 94865925533696, 94865925545983, ++SNULL, 94865925492735, 94865925533695, ++STORE, 94865923670016, 94865925492735, ++STORE, 94865925492736, 94865925533695, ++ERASE, 94865925492736, 94865925533695, ++STORE, 94865923670016, 94865925627903, ++SNULL, 94865925599231, 94865925627903, ++STORE, 94865923670016, 94865925599231, ++STORE, 94865925599232, 94865925627903, ++ERASE, 94865925599232, 94865925627903, ++STORE, 94865923670016, 94865925738495, ++SNULL, 94865925726207, 94865925738495, ++STORE, 94865923670016, 94865925726207, ++STORE, 94865925726208, 94865925738495, ++ERASE, 94865925726208, 94865925738495, ++STORE, 94865923670016, 94865925877759, ++SNULL, 94865925865471, 94865925877759, ++STORE, 94865923670016, 94865925865471, ++STORE, 94865925865472, 94865925877759, ++ERASE, 94865925865472, 94865925877759, ++STORE, 94865923670016, 94865926021119, ++SNULL, 94865926008831, 94865926021119, ++STORE, 94865923670016, 94865926008831, ++STORE, 94865926008832, 94865926021119, ++ERASE, 94865926008832, 94865926021119, ++SNULL, 94865925971967, 94865926008831, ++STORE, 94865923670016, 94865925971967, ++STORE, 94865925971968, 94865926008831, ++ERASE, 94865925971968, 94865926008831, ++STORE, 94865923670016, 94865926115327, ++STORE, 94865923670016, 94865926254591, ++SNULL, 94865926246399, 94865926254591, ++STORE, 94865923670016, 94865926246399, ++STORE, 94865926246400, 94865926254591, ++ERASE, 94865926246400, 94865926254591, ++STORE, 94865923670016, 94865926385663, ++STORE, 94865923670016, 94865926537215, ++STORE, 94865923670016, 94865926672383, ++STORE, 94865923670016, 94865926815743, ++STORE, 94865923670016, 94865926955007, ++STORE, 94865923670016, 94865927094271, ++STORE, 94865923670016, 94865927233535, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731148435456, 140737488351231, ++SNULL, 140731148443647, 140737488351231, ++STORE, 140731148435456, 140731148443647, ++STORE, 140731148304384, 140731148443647, ++STORE, 94090775400448, 94090777735167, ++SNULL, 94090775613439, 94090777735167, ++STORE, 94090775400448, 94090775613439, ++STORE, 94090775613440, 94090777735167, ++ERASE, 94090775613440, 94090777735167, ++STORE, 94090777710592, 94090777722879, ++STORE, 94090777722880, 94090777735167, ++STORE, 140301090283520, 140301092536319, ++SNULL, 140301090426879, 140301092536319, ++STORE, 140301090283520, 140301090426879, ++STORE, 140301090426880, 140301092536319, ++ERASE, 140301090426880, 140301092536319, ++STORE, 140301092524032, 140301092532223, ++STORE, 140301092532224, 140301092536319, ++STORE, 140731148570624, 140731148574719, ++STORE, 140731148558336, 140731148570623, ++STORE, 140301092495360, 140301092524031, ++STORE, 140301092487168, 140301092495359, ++STORE, 140301088169984, 140301090283519, ++SNULL, 140301088169984, 140301088182271, ++STORE, 140301088182272, 140301090283519, ++STORE, 140301088169984, 140301088182271, ++SNULL, 140301090275327, 140301090283519, ++STORE, 140301088182272, 140301090275327, ++STORE, 140301090275328, 140301090283519, ++ERASE, 140301090275328, 140301090283519, ++STORE, 140301090275328, 140301090283519, ++STORE, 140301084372992, 140301088169983, ++SNULL, 140301084372992, 140301086031871, ++STORE, 140301086031872, 140301088169983, ++STORE, 140301084372992, 140301086031871, ++SNULL, 140301088129023, 140301088169983, ++STORE, 140301086031872, 140301088129023, ++STORE, 140301088129024, 140301088169983, ++SNULL, 140301088129024, 140301088153599, ++STORE, 140301088153600, 140301088169983, ++STORE, 140301088129024, 140301088153599, ++ERASE, 140301088129024, 140301088153599, ++STORE, 140301088129024, 140301088153599, ++ERASE, 140301088153600, 140301088169983, ++STORE, 140301088153600, 140301088169983, ++STORE, 140301092478976, 140301092495359, ++SNULL, 140301088145407, 140301088153599, ++STORE, 140301088129024, 140301088145407, ++STORE, 140301088145408, 140301088153599, ++SNULL, 140301090279423, 140301090283519, ++STORE, 140301090275328, 140301090279423, ++STORE, 140301090279424, 140301090283519, ++SNULL, 94090777714687, 94090777722879, ++STORE, 94090777710592, 94090777714687, ++STORE, 94090777714688, 94090777722879, ++SNULL, 140301092528127, 140301092532223, ++STORE, 140301092524032, 140301092528127, ++STORE, 140301092528128, 140301092532223, ++ERASE, 140301092495360, 140301092524031, ++STORE, 94090794590208, 94090794725375, ++STORE, 140301090795520, 140301092478975, ++STORE, 94090794590208, 94090794860543, ++STORE, 94090794590208, 94090794995711, ++STORE, 94090794590208, 94090795163647, ++SNULL, 94090795139071, 94090795163647, ++STORE, 94090794590208, 94090795139071, ++STORE, 94090795139072, 94090795163647, ++ERASE, 94090795139072, 94090795163647, ++STORE, 94090794590208, 94090795278335, ++STORE, 94090794590208, 94090795425791, ++SNULL, 94090795388927, 94090795425791, ++STORE, 94090794590208, 94090795388927, ++STORE, 94090795388928, 94090795425791, ++ERASE, 94090795388928, 94090795425791, ++STORE, 94090794590208, 94090795528191, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733084430336, 140737488351231, ++SNULL, 140733084438527, 140737488351231, ++STORE, 140733084430336, 140733084438527, ++STORE, 140733084299264, 140733084438527, ++STORE, 94116169183232, 94116171517951, ++SNULL, 94116169396223, 94116171517951, ++STORE, 94116169183232, 94116169396223, ++STORE, 94116169396224, 94116171517951, ++ERASE, 94116169396224, 94116171517951, ++STORE, 94116171493376, 94116171505663, ++STORE, 94116171505664, 94116171517951, ++STORE, 139772214128640, 139772216381439, ++SNULL, 139772214271999, 139772216381439, ++STORE, 139772214128640, 139772214271999, ++STORE, 139772214272000, 139772216381439, ++ERASE, 139772214272000, 139772216381439, ++STORE, 139772216369152, 139772216377343, ++STORE, 139772216377344, 139772216381439, ++STORE, 140733085270016, 140733085274111, ++STORE, 140733085257728, 140733085270015, ++STORE, 139772216340480, 139772216369151, ++STORE, 139772216332288, 139772216340479, ++STORE, 139772212015104, 139772214128639, ++SNULL, 139772212015104, 139772212027391, ++STORE, 139772212027392, 139772214128639, ++STORE, 139772212015104, 139772212027391, ++SNULL, 139772214120447, 139772214128639, ++STORE, 139772212027392, 139772214120447, ++STORE, 139772214120448, 139772214128639, ++ERASE, 139772214120448, 139772214128639, ++STORE, 139772214120448, 139772214128639, ++STORE, 139772208218112, 139772212015103, ++SNULL, 139772208218112, 139772209876991, ++STORE, 139772209876992, 139772212015103, ++STORE, 139772208218112, 139772209876991, ++SNULL, 139772211974143, 139772212015103, ++STORE, 139772209876992, 139772211974143, ++STORE, 139772211974144, 139772212015103, ++SNULL, 139772211974144, 139772211998719, ++STORE, 139772211998720, 139772212015103, ++STORE, 139772211974144, 139772211998719, ++ERASE, 139772211974144, 139772211998719, ++STORE, 139772211974144, 139772211998719, ++ERASE, 139772211998720, 139772212015103, ++STORE, 139772211998720, 139772212015103, ++STORE, 139772216324096, 139772216340479, ++SNULL, 139772211990527, 139772211998719, ++STORE, 139772211974144, 139772211990527, ++STORE, 139772211990528, 139772211998719, ++SNULL, 139772214124543, 139772214128639, ++STORE, 139772214120448, 139772214124543, ++STORE, 139772214124544, 139772214128639, ++SNULL, 94116171497471, 94116171505663, ++STORE, 94116171493376, 94116171497471, ++STORE, 94116171497472, 94116171505663, ++SNULL, 139772216373247, 139772216377343, ++STORE, 139772216369152, 139772216373247, ++STORE, 139772216373248, 139772216377343, ++ERASE, 139772216340480, 139772216369151, ++STORE, 94116199383040, 94116199518207, ++STORE, 139772214640640, 139772216324095, ++STORE, 94116199383040, 94116199653375, ++STORE, 94116199383040, 94116199788543, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726067826688, 140737488351231, ++SNULL, 140726067830783, 140737488351231, ++STORE, 140726067826688, 140726067830783, ++STORE, 140726067695616, 140726067830783, ++STORE, 94535150673920, 94535152898047, ++SNULL, 94535150784511, 94535152898047, ++STORE, 94535150673920, 94535150784511, ++STORE, 94535150784512, 94535152898047, ++ERASE, 94535150784512, 94535152898047, ++STORE, 94535152877568, 94535152889855, ++STORE, 94535152889856, 94535152898047, ++STORE, 140381257314304, 140381259567103, ++SNULL, 140381257457663, 140381259567103, ++STORE, 140381257314304, 140381257457663, ++STORE, 140381257457664, 140381259567103, ++ERASE, 140381257457664, 140381259567103, ++STORE, 140381259554816, 140381259563007, ++STORE, 140381259563008, 140381259567103, ++STORE, 140726068060160, 140726068064255, ++STORE, 140726068047872, 140726068060159, ++STORE, 140381259526144, 140381259554815, ++STORE, 140381259517952, 140381259526143, ++STORE, 140381253517312, 140381257314303, ++SNULL, 140381253517312, 140381255176191, ++STORE, 140381255176192, 140381257314303, ++STORE, 140381253517312, 140381255176191, ++SNULL, 140381257273343, 140381257314303, ++STORE, 140381255176192, 140381257273343, ++STORE, 140381257273344, 140381257314303, ++SNULL, 140381257273344, 140381257297919, ++STORE, 140381257297920, 140381257314303, ++STORE, 140381257273344, 140381257297919, ++ERASE, 140381257273344, 140381257297919, ++STORE, 140381257273344, 140381257297919, ++ERASE, 140381257297920, 140381257314303, ++STORE, 140381257297920, 140381257314303, ++SNULL, 140381257289727, 140381257297919, ++STORE, 140381257273344, 140381257289727, ++STORE, 140381257289728, 140381257297919, ++SNULL, 94535152885759, 94535152889855, ++STORE, 94535152877568, 94535152885759, ++STORE, 94535152885760, 94535152889855, ++SNULL, 140381259558911, 140381259563007, ++STORE, 140381259554816, 140381259558911, ++STORE, 140381259558912, 140381259563007, ++ERASE, 140381259526144, 140381259554815, ++STORE, 94535186296832, 94535186431999, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140729189425152, 140737488351231, ++SNULL, 140729189433343, 140737488351231, ++STORE, 140729189425152, 140729189433343, ++STORE, 140729189294080, 140729189433343, ++STORE, 94428200128512, 94428202352639, ++SNULL, 94428200239103, 94428202352639, ++STORE, 94428200128512, 94428200239103, ++STORE, 94428200239104, 94428202352639, ++ERASE, 94428200239104, 94428202352639, ++STORE, 94428202332160, 94428202344447, ++STORE, 94428202344448, 94428202352639, ++STORE, 139707216986112, 139707219238911, ++SNULL, 139707217129471, 139707219238911, ++STORE, 139707216986112, 139707217129471, ++STORE, 139707217129472, 139707219238911, ++ERASE, 139707217129472, 139707219238911, ++STORE, 139707219226624, 139707219234815, ++STORE, 139707219234816, 139707219238911, ++STORE, 140729189785600, 140729189789695, ++STORE, 140729189773312, 140729189785599, ++STORE, 139707219197952, 139707219226623, ++STORE, 139707219189760, 139707219197951, ++STORE, 139707213189120, 139707216986111, ++SNULL, 139707213189120, 139707214847999, ++STORE, 139707214848000, 139707216986111, ++STORE, 139707213189120, 139707214847999, ++SNULL, 139707216945151, 139707216986111, ++STORE, 139707214848000, 139707216945151, ++STORE, 139707216945152, 139707216986111, ++SNULL, 139707216945152, 139707216969727, ++STORE, 139707216969728, 139707216986111, ++STORE, 139707216945152, 139707216969727, ++ERASE, 139707216945152, 139707216969727, ++STORE, 139707216945152, 139707216969727, ++ERASE, 139707216969728, 139707216986111, ++STORE, 139707216969728, 139707216986111, ++SNULL, 139707216961535, 139707216969727, ++STORE, 139707216945152, 139707216961535, ++STORE, 139707216961536, 139707216969727, ++SNULL, 94428202340351, 94428202344447, ++STORE, 94428202332160, 94428202340351, ++STORE, 94428202340352, 94428202344447, ++SNULL, 139707219230719, 139707219234815, ++STORE, 139707219226624, 139707219230719, ++STORE, 139707219230720, 139707219234815, ++ERASE, 139707219197952, 139707219226623, ++STORE, 94428208599040, 94428208734207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722000953344, 140737488351231, ++SNULL, 140722000961535, 140737488351231, ++STORE, 140722000953344, 140722000961535, ++STORE, 140722000822272, 140722000961535, ++STORE, 94636494757888, 94636496982015, ++SNULL, 94636494868479, 94636496982015, ++STORE, 94636494757888, 94636494868479, ++STORE, 94636494868480, 94636496982015, ++ERASE, 94636494868480, 94636496982015, ++STORE, 94636496961536, 94636496973823, ++STORE, 94636496973824, 94636496982015, ++STORE, 140142275100672, 140142277353471, ++SNULL, 140142275244031, 140142277353471, ++STORE, 140142275100672, 140142275244031, ++STORE, 140142275244032, 140142277353471, ++ERASE, 140142275244032, 140142277353471, ++STORE, 140142277341184, 140142277349375, ++STORE, 140142277349376, 140142277353471, ++STORE, 140722002747392, 140722002751487, ++STORE, 140722002735104, 140722002747391, ++STORE, 140142277312512, 140142277341183, ++STORE, 140142277304320, 140142277312511, ++STORE, 140142271303680, 140142275100671, ++SNULL, 140142271303680, 140142272962559, ++STORE, 140142272962560, 140142275100671, ++STORE, 140142271303680, 140142272962559, ++SNULL, 140142275059711, 140142275100671, ++STORE, 140142272962560, 140142275059711, ++STORE, 140142275059712, 140142275100671, ++SNULL, 140142275059712, 140142275084287, ++STORE, 140142275084288, 140142275100671, ++STORE, 140142275059712, 140142275084287, ++ERASE, 140142275059712, 140142275084287, ++STORE, 140142275059712, 140142275084287, ++ERASE, 140142275084288, 140142275100671, ++STORE, 140142275084288, 140142275100671, ++SNULL, 140142275076095, 140142275084287, ++STORE, 140142275059712, 140142275076095, ++STORE, 140142275076096, 140142275084287, ++SNULL, 94636496969727, 94636496973823, ++STORE, 94636496961536, 94636496969727, ++STORE, 94636496969728, 94636496973823, ++SNULL, 140142277345279, 140142277349375, ++STORE, 140142277341184, 140142277345279, ++STORE, 140142277345280, 140142277349375, ++ERASE, 140142277312512, 140142277341183, ++STORE, 94636516286464, 94636516421631, ++STORE, 94071103692800, 94071103905791, ++STORE, 94071106002944, 94071106007039, ++STORE, 94071106007040, 94071106015231, ++STORE, 94071106015232, 94071106027519, ++STORE, 94071138521088, 94071140368383, ++STORE, 140145668190208, 140145669849087, ++STORE, 140145669849088, 140145671946239, ++STORE, 140145671946240, 140145671962623, ++STORE, 140145671962624, 140145671970815, ++STORE, 140145671970816, 140145671987199, ++STORE, 140145671987200, 140145671999487, ++STORE, 140145671999488, 140145674092543, ++STORE, 140145674092544, 140145674096639, ++STORE, 140145674096640, 140145674100735, ++STORE, 140145674100736, 140145674244095, ++STORE, 140145674612736, 140145676296191, ++STORE, 140145676296192, 140145676312575, ++STORE, 140145676341248, 140145676345343, ++STORE, 140145676345344, 140145676349439, ++STORE, 140145676349440, 140145676353535, ++STORE, 140734927740928, 140734927880191, ++STORE, 140734928842752, 140734928855039, ++STORE, 140734928855040, 140734928859135, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722342535168, 140737488351231, ++SNULL, 140722342543359, 140737488351231, ++STORE, 140722342535168, 140722342543359, ++STORE, 140722342404096, 140722342543359, ++STORE, 94399699714048, 94399702048767, ++SNULL, 94399699927039, 94399702048767, ++STORE, 94399699714048, 94399699927039, ++STORE, 94399699927040, 94399702048767, ++ERASE, 94399699927040, 94399702048767, ++STORE, 94399702024192, 94399702036479, ++STORE, 94399702036480, 94399702048767, ++STORE, 139811024748544, 139811027001343, ++SNULL, 139811024891903, 139811027001343, ++STORE, 139811024748544, 139811024891903, ++STORE, 139811024891904, 139811027001343, ++ERASE, 139811024891904, 139811027001343, ++STORE, 139811026989056, 139811026997247, ++STORE, 139811026997248, 139811027001343, ++STORE, 140722342707200, 140722342711295, ++STORE, 140722342694912, 140722342707199, ++STORE, 139811026960384, 139811026989055, ++STORE, 139811026952192, 139811026960383, ++STORE, 139811022635008, 139811024748543, ++SNULL, 139811022635008, 139811022647295, ++STORE, 139811022647296, 139811024748543, ++STORE, 139811022635008, 139811022647295, ++SNULL, 139811024740351, 139811024748543, ++STORE, 139811022647296, 139811024740351, ++STORE, 139811024740352, 139811024748543, ++ERASE, 139811024740352, 139811024748543, ++STORE, 139811024740352, 139811024748543, ++STORE, 139811018838016, 139811022635007, ++SNULL, 139811018838016, 139811020496895, ++STORE, 139811020496896, 139811022635007, ++STORE, 139811018838016, 139811020496895, ++SNULL, 139811022594047, 139811022635007, ++STORE, 139811020496896, 139811022594047, ++STORE, 139811022594048, 139811022635007, ++SNULL, 139811022594048, 139811022618623, ++STORE, 139811022618624, 139811022635007, ++STORE, 139811022594048, 139811022618623, ++ERASE, 139811022594048, 139811022618623, ++STORE, 139811022594048, 139811022618623, ++ERASE, 139811022618624, 139811022635007, ++STORE, 139811022618624, 139811022635007, ++STORE, 139811026944000, 139811026960383, ++SNULL, 139811022610431, 139811022618623, ++STORE, 139811022594048, 139811022610431, ++STORE, 139811022610432, 139811022618623, ++SNULL, 139811024744447, 139811024748543, ++STORE, 139811024740352, 139811024744447, ++STORE, 139811024744448, 139811024748543, ++SNULL, 94399702028287, 94399702036479, ++STORE, 94399702024192, 94399702028287, ++STORE, 94399702028288, 94399702036479, ++SNULL, 139811026993151, 139811026997247, ++STORE, 139811026989056, 139811026993151, ++STORE, 139811026993152, 139811026997247, ++ERASE, 139811026960384, 139811026989055, ++STORE, 94399723880448, 94399724015615, ++STORE, 139811025260544, 139811026943999, ++STORE, 94399723880448, 94399724150783, ++STORE, 94399723880448, 94399724285951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735364939776, 140737488351231, ++SNULL, 140735364947967, 140737488351231, ++STORE, 140735364939776, 140735364947967, ++STORE, 140735364808704, 140735364947967, ++STORE, 94421528674304, 94421531009023, ++SNULL, 94421528887295, 94421531009023, ++STORE, 94421528674304, 94421528887295, ++STORE, 94421528887296, 94421531009023, ++ERASE, 94421528887296, 94421531009023, ++STORE, 94421530984448, 94421530996735, ++STORE, 94421530996736, 94421531009023, ++STORE, 140162004742144, 140162006994943, ++SNULL, 140162004885503, 140162006994943, ++STORE, 140162004742144, 140162004885503, ++STORE, 140162004885504, 140162006994943, ++ERASE, 140162004885504, 140162006994943, ++STORE, 140162006982656, 140162006990847, ++STORE, 140162006990848, 140162006994943, ++STORE, 140735365402624, 140735365406719, ++STORE, 140735365390336, 140735365402623, ++STORE, 140162006953984, 140162006982655, ++STORE, 140162006945792, 140162006953983, ++STORE, 140162002628608, 140162004742143, ++SNULL, 140162002628608, 140162002640895, ++STORE, 140162002640896, 140162004742143, ++STORE, 140162002628608, 140162002640895, ++SNULL, 140162004733951, 140162004742143, ++STORE, 140162002640896, 140162004733951, ++STORE, 140162004733952, 140162004742143, ++ERASE, 140162004733952, 140162004742143, ++STORE, 140162004733952, 140162004742143, ++STORE, 140161998831616, 140162002628607, ++SNULL, 140161998831616, 140162000490495, ++STORE, 140162000490496, 140162002628607, ++STORE, 140161998831616, 140162000490495, ++SNULL, 140162002587647, 140162002628607, ++STORE, 140162000490496, 140162002587647, ++STORE, 140162002587648, 140162002628607, ++SNULL, 140162002587648, 140162002612223, ++STORE, 140162002612224, 140162002628607, ++STORE, 140162002587648, 140162002612223, ++ERASE, 140162002587648, 140162002612223, ++STORE, 140162002587648, 140162002612223, ++ERASE, 140162002612224, 140162002628607, ++STORE, 140162002612224, 140162002628607, ++STORE, 140162006937600, 140162006953983, ++SNULL, 140162002604031, 140162002612223, ++STORE, 140162002587648, 140162002604031, ++STORE, 140162002604032, 140162002612223, ++SNULL, 140162004738047, 140162004742143, ++STORE, 140162004733952, 140162004738047, ++STORE, 140162004738048, 140162004742143, ++SNULL, 94421530988543, 94421530996735, ++STORE, 94421530984448, 94421530988543, ++STORE, 94421530988544, 94421530996735, ++SNULL, 140162006986751, 140162006990847, ++STORE, 140162006982656, 140162006986751, ++STORE, 140162006986752, 140162006990847, ++ERASE, 140162006953984, 140162006982655, ++STORE, 94421551697920, 94421551833087, ++STORE, 140162005254144, 140162006937599, ++STORE, 94421551697920, 94421551968255, ++STORE, 94421551697920, 94421552103423, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733498486784, 140737488351231, ++SNULL, 140733498494975, 140737488351231, ++STORE, 140733498486784, 140733498494975, ++STORE, 140733498355712, 140733498494975, ++STORE, 94567985836032, 94567988170751, ++SNULL, 94567986049023, 94567988170751, ++STORE, 94567985836032, 94567986049023, ++STORE, 94567986049024, 94567988170751, ++ERASE, 94567986049024, 94567988170751, ++STORE, 94567988146176, 94567988158463, ++STORE, 94567988158464, 94567988170751, ++STORE, 139634278572032, 139634280824831, ++SNULL, 139634278715391, 139634280824831, ++STORE, 139634278572032, 139634278715391, ++STORE, 139634278715392, 139634280824831, ++ERASE, 139634278715392, 139634280824831, ++STORE, 139634280812544, 139634280820735, ++STORE, 139634280820736, 139634280824831, ++STORE, 140733498544128, 140733498548223, ++STORE, 140733498531840, 140733498544127, ++STORE, 139634280783872, 139634280812543, ++STORE, 139634280775680, 139634280783871, ++STORE, 139634276458496, 139634278572031, ++SNULL, 139634276458496, 139634276470783, ++STORE, 139634276470784, 139634278572031, ++STORE, 139634276458496, 139634276470783, ++SNULL, 139634278563839, 139634278572031, ++STORE, 139634276470784, 139634278563839, ++STORE, 139634278563840, 139634278572031, ++ERASE, 139634278563840, 139634278572031, ++STORE, 139634278563840, 139634278572031, ++STORE, 139634272661504, 139634276458495, ++SNULL, 139634272661504, 139634274320383, ++STORE, 139634274320384, 139634276458495, ++STORE, 139634272661504, 139634274320383, ++SNULL, 139634276417535, 139634276458495, ++STORE, 139634274320384, 139634276417535, ++STORE, 139634276417536, 139634276458495, ++SNULL, 139634276417536, 139634276442111, ++STORE, 139634276442112, 139634276458495, ++STORE, 139634276417536, 139634276442111, ++ERASE, 139634276417536, 139634276442111, ++STORE, 139634276417536, 139634276442111, ++ERASE, 139634276442112, 139634276458495, ++STORE, 139634276442112, 139634276458495, ++STORE, 139634280767488, 139634280783871, ++SNULL, 139634276433919, 139634276442111, ++STORE, 139634276417536, 139634276433919, ++STORE, 139634276433920, 139634276442111, ++SNULL, 139634278567935, 139634278572031, ++STORE, 139634278563840, 139634278567935, ++STORE, 139634278567936, 139634278572031, ++SNULL, 94567988150271, 94567988158463, ++STORE, 94567988146176, 94567988150271, ++STORE, 94567988150272, 94567988158463, ++SNULL, 139634280816639, 139634280820735, ++STORE, 139634280812544, 139634280816639, ++STORE, 139634280816640, 139634280820735, ++ERASE, 139634280783872, 139634280812543, ++STORE, 94567996379136, 94567996514303, ++STORE, 139634279084032, 139634280767487, ++STORE, 94567996379136, 94567996649471, ++STORE, 94567996379136, 94567996784639, ++STORE, 94567996379136, 94567996960767, ++SNULL, 94567996932095, 94567996960767, ++STORE, 94567996379136, 94567996932095, ++STORE, 94567996932096, 94567996960767, ++ERASE, 94567996932096, 94567996960767, ++STORE, 94567996379136, 94567997071359, ++STORE, 94567996379136, 94567997206527, ++SNULL, 94567997186047, 94567997206527, ++STORE, 94567996379136, 94567997186047, ++STORE, 94567997186048, 94567997206527, ++ERASE, 94567997186048, 94567997206527, ++STORE, 94567996379136, 94567997358079, ++STORE, 94567996379136, 94567997493247, ++SNULL, 94567997476863, 94567997493247, ++STORE, 94567996379136, 94567997476863, ++STORE, 94567997476864, 94567997493247, ++ERASE, 94567997476864, 94567997493247, ++STORE, 94567996379136, 94567997612031, ++STORE, 94567996379136, 94567997767679, ++SNULL, 94567997739007, 94567997767679, ++STORE, 94567996379136, 94567997739007, ++STORE, 94567997739008, 94567997767679, ++ERASE, 94567997739008, 94567997767679, ++SNULL, 94567997698047, 94567997739007, ++STORE, 94567996379136, 94567997698047, ++STORE, 94567997698048, 94567997739007, ++ERASE, 94567997698048, 94567997739007, ++STORE, 94567996379136, 94567997853695, ++STORE, 94567996379136, 94567997988863, ++STORE, 94567996379136, 94567998132223, ++STORE, 94567996379136, 94567998275583, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723667759104, 140737488351231, ++SNULL, 140723667767295, 140737488351231, ++STORE, 140723667759104, 140723667767295, ++STORE, 140723667628032, 140723667767295, ++STORE, 94231598800896, 94231601135615, ++SNULL, 94231599013887, 94231601135615, ++STORE, 94231598800896, 94231599013887, ++STORE, 94231599013888, 94231601135615, ++ERASE, 94231599013888, 94231601135615, ++STORE, 94231601111040, 94231601123327, ++STORE, 94231601123328, 94231601135615, ++STORE, 140269472649216, 140269474902015, ++SNULL, 140269472792575, 140269474902015, ++STORE, 140269472649216, 140269472792575, ++STORE, 140269472792576, 140269474902015, ++ERASE, 140269472792576, 140269474902015, ++STORE, 140269474889728, 140269474897919, ++STORE, 140269474897920, 140269474902015, ++STORE, 140723667836928, 140723667841023, ++STORE, 140723667824640, 140723667836927, ++STORE, 140269474861056, 140269474889727, ++STORE, 140269474852864, 140269474861055, ++STORE, 140269470535680, 140269472649215, ++SNULL, 140269470535680, 140269470547967, ++STORE, 140269470547968, 140269472649215, ++STORE, 140269470535680, 140269470547967, ++SNULL, 140269472641023, 140269472649215, ++STORE, 140269470547968, 140269472641023, ++STORE, 140269472641024, 140269472649215, ++ERASE, 140269472641024, 140269472649215, ++STORE, 140269472641024, 140269472649215, ++STORE, 140269466738688, 140269470535679, ++SNULL, 140269466738688, 140269468397567, ++STORE, 140269468397568, 140269470535679, ++STORE, 140269466738688, 140269468397567, ++SNULL, 140269470494719, 140269470535679, ++STORE, 140269468397568, 140269470494719, ++STORE, 140269470494720, 140269470535679, ++SNULL, 140269470494720, 140269470519295, ++STORE, 140269470519296, 140269470535679, ++STORE, 140269470494720, 140269470519295, ++ERASE, 140269470494720, 140269470519295, ++STORE, 140269470494720, 140269470519295, ++ERASE, 140269470519296, 140269470535679, ++STORE, 140269470519296, 140269470535679, ++STORE, 140269474844672, 140269474861055, ++SNULL, 140269470511103, 140269470519295, ++STORE, 140269470494720, 140269470511103, ++STORE, 140269470511104, 140269470519295, ++SNULL, 140269472645119, 140269472649215, ++STORE, 140269472641024, 140269472645119, ++STORE, 140269472645120, 140269472649215, ++SNULL, 94231601115135, 94231601123327, ++STORE, 94231601111040, 94231601115135, ++STORE, 94231601115136, 94231601123327, ++SNULL, 140269474893823, 140269474897919, ++STORE, 140269474889728, 140269474893823, ++STORE, 140269474893824, 140269474897919, ++ERASE, 140269474861056, 140269474889727, ++STORE, 94231626592256, 94231626727423, ++STORE, 140269473161216, 140269474844671, ++STORE, 94231626592256, 94231626862591, ++STORE, 94231626592256, 94231626997759, ++STORE, 94327178862592, 94327179075583, ++STORE, 94327181172736, 94327181176831, ++STORE, 94327181176832, 94327181185023, ++STORE, 94327181185024, 94327181197311, ++STORE, 94327185715200, 94327186685951, ++STORE, 140172071755776, 140172073414655, ++STORE, 140172073414656, 140172075511807, ++STORE, 140172075511808, 140172075528191, ++STORE, 140172075528192, 140172075536383, ++STORE, 140172075536384, 140172075552767, ++STORE, 140172075552768, 140172075565055, ++STORE, 140172075565056, 140172077658111, ++STORE, 140172077658112, 140172077662207, ++STORE, 140172077662208, 140172077666303, ++STORE, 140172077666304, 140172077809663, ++STORE, 140172078178304, 140172079861759, ++STORE, 140172079861760, 140172079878143, ++STORE, 140172079878144, 140172079906815, ++STORE, 140172079906816, 140172079910911, ++STORE, 140172079910912, 140172079915007, ++STORE, 140172079915008, 140172079919103, ++STORE, 140720358359040, 140720358494207, ++STORE, 140720358498304, 140720358510591, ++STORE, 140720358510592, 140720358514687, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722548621312, 140737488351231, ++SNULL, 140722548629503, 140737488351231, ++STORE, 140722548621312, 140722548629503, ++STORE, 140722548490240, 140722548629503, ++STORE, 93949289504768, 93949291728895, ++SNULL, 93949289615359, 93949291728895, ++STORE, 93949289504768, 93949289615359, ++STORE, 93949289615360, 93949291728895, ++ERASE, 93949289615360, 93949291728895, ++STORE, 93949291708416, 93949291720703, ++STORE, 93949291720704, 93949291728895, ++STORE, 140305861902336, 140305864155135, ++SNULL, 140305862045695, 140305864155135, ++STORE, 140305861902336, 140305862045695, ++STORE, 140305862045696, 140305864155135, ++ERASE, 140305862045696, 140305864155135, ++STORE, 140305864142848, 140305864151039, ++STORE, 140305864151040, 140305864155135, ++STORE, 140722549821440, 140722549825535, ++STORE, 140722549809152, 140722549821439, ++STORE, 140305864114176, 140305864142847, ++STORE, 140305864105984, 140305864114175, ++STORE, 140305858105344, 140305861902335, ++SNULL, 140305858105344, 140305859764223, ++STORE, 140305859764224, 140305861902335, ++STORE, 140305858105344, 140305859764223, ++SNULL, 140305861861375, 140305861902335, ++STORE, 140305859764224, 140305861861375, ++STORE, 140305861861376, 140305861902335, ++SNULL, 140305861861376, 140305861885951, ++STORE, 140305861885952, 140305861902335, ++STORE, 140305861861376, 140305861885951, ++ERASE, 140305861861376, 140305861885951, ++STORE, 140305861861376, 140305861885951, ++ERASE, 140305861885952, 140305861902335, ++STORE, 140305861885952, 140305861902335, ++SNULL, 140305861877759, 140305861885951, ++STORE, 140305861861376, 140305861877759, ++STORE, 140305861877760, 140305861885951, ++SNULL, 93949291716607, 93949291720703, ++STORE, 93949291708416, 93949291716607, ++STORE, 93949291716608, 93949291720703, ++SNULL, 140305864146943, 140305864151039, ++STORE, 140305864142848, 140305864146943, ++STORE, 140305864146944, 140305864151039, ++ERASE, 140305864114176, 140305864142847, ++STORE, 93949324136448, 93949324271615, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725754908672, 140737488351231, ++SNULL, 140725754916863, 140737488351231, ++STORE, 140725754908672, 140725754916863, ++STORE, 140725754777600, 140725754916863, ++STORE, 94831184375808, 94831186599935, ++SNULL, 94831184486399, 94831186599935, ++STORE, 94831184375808, 94831184486399, ++STORE, 94831184486400, 94831186599935, ++ERASE, 94831184486400, 94831186599935, ++STORE, 94831186579456, 94831186591743, ++STORE, 94831186591744, 94831186599935, ++STORE, 140605482479616, 140605484732415, ++SNULL, 140605482622975, 140605484732415, ++STORE, 140605482479616, 140605482622975, ++STORE, 140605482622976, 140605484732415, ++ERASE, 140605482622976, 140605484732415, ++STORE, 140605484720128, 140605484728319, ++STORE, 140605484728320, 140605484732415, ++STORE, 140725755670528, 140725755674623, ++STORE, 140725755658240, 140725755670527, ++STORE, 140605484691456, 140605484720127, ++STORE, 140605484683264, 140605484691455, ++STORE, 140605478682624, 140605482479615, ++SNULL, 140605478682624, 140605480341503, ++STORE, 140605480341504, 140605482479615, ++STORE, 140605478682624, 140605480341503, ++SNULL, 140605482438655, 140605482479615, ++STORE, 140605480341504, 140605482438655, ++STORE, 140605482438656, 140605482479615, ++SNULL, 140605482438656, 140605482463231, ++STORE, 140605482463232, 140605482479615, ++STORE, 140605482438656, 140605482463231, ++ERASE, 140605482438656, 140605482463231, ++STORE, 140605482438656, 140605482463231, ++ERASE, 140605482463232, 140605482479615, ++STORE, 140605482463232, 140605482479615, ++SNULL, 140605482455039, 140605482463231, ++STORE, 140605482438656, 140605482455039, ++STORE, 140605482455040, 140605482463231, ++SNULL, 94831186587647, 94831186591743, ++STORE, 94831186579456, 94831186587647, ++STORE, 94831186587648, 94831186591743, ++SNULL, 140605484724223, 140605484728319, ++STORE, 140605484720128, 140605484724223, ++STORE, 140605484724224, 140605484728319, ++ERASE, 140605484691456, 140605484720127, ++STORE, 94831217156096, 94831217291263, ++STORE, 94327178862592, 94327179075583, ++STORE, 94327181172736, 94327181176831, ++STORE, 94327181176832, 94327181185023, ++STORE, 94327181185024, 94327181197311, ++STORE, 94327185715200, 94327186685951, ++STORE, 140172071755776, 140172073414655, ++STORE, 140172073414656, 140172075511807, ++STORE, 140172075511808, 140172075528191, ++STORE, 140172075528192, 140172075536383, ++STORE, 140172075536384, 140172075552767, ++STORE, 140172075552768, 140172075565055, ++STORE, 140172075565056, 140172077658111, ++STORE, 140172077658112, 140172077662207, ++STORE, 140172077662208, 140172077666303, ++STORE, 140172077666304, 140172077809663, ++STORE, 140172078178304, 140172079861759, ++STORE, 140172079861760, 140172079878143, ++STORE, 140172079878144, 140172079906815, ++STORE, 140172079906816, 140172079910911, ++STORE, 140172079910912, 140172079915007, ++STORE, 140172079915008, 140172079919103, ++STORE, 140720358359040, 140720358494207, ++STORE, 140720358498304, 140720358510591, ++STORE, 140720358510592, 140720358514687, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140734529933312, 140737488351231, ++SNULL, 140734529945599, 140737488351231, ++STORE, 140734529933312, 140734529945599, ++STORE, 140734529802240, 140734529945599, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 140249744060416, 140249746313215, ++SNULL, 140249744203775, 140249746313215, ++STORE, 140249744060416, 140249744203775, ++STORE, 140249744203776, 140249746313215, ++ERASE, 140249744203776, 140249746313215, ++STORE, 140249746300928, 140249746309119, ++STORE, 140249746309120, 140249746313215, ++STORE, 140734530174976, 140734530179071, ++STORE, 140734530162688, 140734530174975, ++STORE, 140249746272256, 140249746300927, ++STORE, 140249746264064, 140249746272255, ++STORE, 140249740226560, 140249744060415, ++SNULL, 140249740226560, 140249741934591, ++STORE, 140249741934592, 140249744060415, ++STORE, 140249740226560, 140249741934591, ++SNULL, 140249744027647, 140249744060415, ++STORE, 140249741934592, 140249744027647, ++STORE, 140249744027648, 140249744060415, ++ERASE, 140249744027648, 140249744060415, ++STORE, 140249744027648, 140249744060415, ++STORE, 140249738031104, 140249740226559, ++SNULL, 140249738031104, 140249738125311, ++STORE, 140249738125312, 140249740226559, ++STORE, 140249738031104, 140249738125311, ++SNULL, 140249740218367, 140249740226559, ++STORE, 140249738125312, 140249740218367, ++STORE, 140249740218368, 140249740226559, ++ERASE, 140249740218368, 140249740226559, ++STORE, 140249740218368, 140249740226559, ++STORE, 140249735512064, 140249738031103, ++SNULL, 140249735512064, 140249735925759, ++STORE, 140249735925760, 140249738031103, ++STORE, 140249735512064, 140249735925759, ++SNULL, 140249738018815, 140249738031103, ++STORE, 140249735925760, 140249738018815, ++STORE, 140249738018816, 140249738031103, ++ERASE, 140249738018816, 140249738031103, ++STORE, 140249738018816, 140249738031103, ++STORE, 140249732878336, 140249735512063, ++SNULL, 140249732878336, 140249733406719, ++STORE, 140249733406720, 140249735512063, ++STORE, 140249732878336, 140249733406719, ++SNULL, 140249735503871, 140249735512063, ++STORE, 140249733406720, 140249735503871, ++STORE, 140249735503872, 140249735512063, ++ERASE, 140249735503872, 140249735512063, ++STORE, 140249735503872, 140249735512063, ++STORE, 140249730764800, 140249732878335, ++SNULL, 140249730764800, 140249730777087, ++STORE, 140249730777088, 140249732878335, ++STORE, 140249730764800, 140249730777087, ++SNULL, 140249732870143, 140249732878335, ++STORE, 140249730777088, 140249732870143, ++STORE, 140249732870144, 140249732878335, ++ERASE, 140249732870144, 140249732878335, ++STORE, 140249732870144, 140249732878335, ++STORE, 140249728561152, 140249730764799, ++SNULL, 140249728561152, 140249728663551, ++STORE, 140249728663552, 140249730764799, ++STORE, 140249728561152, 140249728663551, ++SNULL, 140249730756607, 140249730764799, ++STORE, 140249728663552, 140249730756607, ++STORE, 140249730756608, 140249730764799, ++ERASE, 140249730756608, 140249730764799, ++STORE, 140249730756608, 140249730764799, ++STORE, 140249746255872, 140249746272255, ++STORE, 140249725399040, 140249728561151, ++SNULL, 140249725399040, 140249726459903, ++STORE, 140249726459904, 140249728561151, ++STORE, 140249725399040, 140249726459903, ++SNULL, 140249728552959, 140249728561151, ++STORE, 140249726459904, 140249728552959, ++STORE, 140249728552960, 140249728561151, ++ERASE, 140249728552960, 140249728561151, ++STORE, 140249728552960, 140249728561151, ++STORE, 140249721602048, 140249725399039, ++SNULL, 140249721602048, 140249723260927, ++STORE, 140249723260928, 140249725399039, ++STORE, 140249721602048, 140249723260927, ++SNULL, 140249725358079, 140249725399039, ++STORE, 140249723260928, 140249725358079, ++STORE, 140249725358080, 140249725399039, ++SNULL, 140249725358080, 140249725382655, ++STORE, 140249725382656, 140249725399039, ++STORE, 140249725358080, 140249725382655, ++ERASE, 140249725358080, 140249725382655, ++STORE, 140249725358080, 140249725382655, ++ERASE, 140249725382656, 140249725399039, ++STORE, 140249725382656, 140249725399039, ++STORE, 140249746243584, 140249746272255, ++SNULL, 140249725374463, 140249725382655, ++STORE, 140249725358080, 140249725374463, ++STORE, 140249725374464, 140249725382655, ++SNULL, 140249728557055, 140249728561151, ++STORE, 140249728552960, 140249728557055, ++STORE, 140249728557056, 140249728561151, ++SNULL, 140249730760703, 140249730764799, ++STORE, 140249730756608, 140249730760703, ++STORE, 140249730760704, 140249730764799, ++SNULL, 140249732874239, 140249732878335, ++STORE, 140249732870144, 140249732874239, ++STORE, 140249732874240, 140249732878335, ++SNULL, 140249735507967, 140249735512063, ++STORE, 140249735503872, 140249735507967, ++STORE, 140249735507968, 140249735512063, ++SNULL, 140249738027007, 140249738031103, ++STORE, 140249738018816, 140249738027007, ++STORE, 140249738027008, 140249738031103, ++SNULL, 140249740222463, 140249740226559, ++STORE, 140249740218368, 140249740222463, ++STORE, 140249740222464, 140249740226559, ++SNULL, 140249744031743, 140249744060415, ++STORE, 140249744027648, 140249744031743, ++STORE, 140249744031744, 140249744060415, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 140249746305023, 140249746309119, ++STORE, 140249746300928, 140249746305023, ++STORE, 140249746305024, 140249746309119, ++ERASE, 140249746272256, 140249746300927, ++STORE, 33853440, 33988607, ++STORE, 140249744560128, 140249746243583, ++STORE, 140249746296832, 140249746300927, ++STORE, 140249744424960, 140249744560127, ++STORE, 33853440, 34131967, ++STORE, 140249719504896, 140249721602047, ++STORE, 140249746288640, 140249746300927, ++STORE, 140249746280448, 140249746300927, ++STORE, 140249746243584, 140249746280447, ++STORE, 140249744408576, 140249744560127, ++STORE, 33853440, 34267135, ++STORE, 33853440, 34422783, ++STORE, 140249744400384, 140249744560127, ++STORE, 140249744392192, 140249744560127, ++STORE, 33853440, 34557951, ++STORE, 33853440, 34693119, ++STORE, 140249744375808, 140249744560127, ++STORE, 140249744367616, 140249744560127, ++STORE, 33853440, 34832383, ++STORE, 140249719230464, 140249721602047, ++STORE, 140249744207872, 140249744560127, ++STORE, 33853440, 34971647, ++SNULL, 34963455, 34971647, ++STORE, 33853440, 34963455, ++STORE, 34963456, 34971647, ++ERASE, 34963456, 34971647, ++SNULL, 34955263, 34963455, ++STORE, 33853440, 34955263, ++STORE, 34955264, 34963455, ++ERASE, 34955264, 34963455, ++SNULL, 34947071, 34955263, ++STORE, 33853440, 34947071, ++STORE, 34947072, 34955263, ++ERASE, 34947072, 34955263, ++SNULL, 34938879, 34947071, ++STORE, 33853440, 34938879, ++STORE, 34938880, 34947071, ++ERASE, 34938880, 34947071, ++STORE, 140249719214080, 140249721602047, ++STORE, 140249719148544, 140249721602047, ++STORE, 140249719115776, 140249721602047, ++STORE, 140249717018624, 140249721602047, ++STORE, 140249716953088, 140249721602047, ++STORE, 33853440, 35086335, ++STORE, 140249716822016, 140249721602047, ++STORE, 140249716559872, 140249721602047, ++STORE, 140249716551680, 140249721602047, ++STORE, 140249716535296, 140249721602047, ++STORE, 140249716527104, 140249721602047, ++STORE, 140249716518912, 140249721602047, ++STORE, 33853440, 35221503, ++SNULL, 35213311, 35221503, ++STORE, 33853440, 35213311, ++STORE, 35213312, 35221503, ++ERASE, 35213312, 35221503, ++SNULL, 35205119, 35213311, ++STORE, 33853440, 35205119, ++STORE, 35205120, 35213311, ++ERASE, 35205120, 35213311, ++SNULL, 35192831, 35205119, ++STORE, 33853440, 35192831, ++STORE, 35192832, 35205119, ++ERASE, 35192832, 35205119, ++SNULL, 35176447, 35192831, ++STORE, 33853440, 35176447, ++STORE, 35176448, 35192831, ++ERASE, 35176448, 35192831, ++STORE, 140249716502528, 140249721602047, ++STORE, 33853440, 35311615, ++SNULL, 35307519, 35311615, ++STORE, 33853440, 35307519, ++STORE, 35307520, 35311615, ++ERASE, 35307520, 35311615, ++SNULL, 35303423, 35307519, ++STORE, 33853440, 35303423, ++STORE, 35303424, 35307519, ++ERASE, 35303424, 35307519, ++SNULL, 35299327, 35303423, ++STORE, 33853440, 35299327, ++STORE, 35299328, 35303423, ++ERASE, 35299328, 35303423, ++SNULL, 35295231, 35299327, ++STORE, 33853440, 35295231, ++STORE, 35295232, 35299327, ++ERASE, 35295232, 35299327, ++SNULL, 35291135, 35295231, ++STORE, 33853440, 35291135, ++STORE, 35291136, 35295231, ++ERASE, 35291136, 35295231, ++SNULL, 35287039, 35291135, ++STORE, 33853440, 35287039, ++STORE, 35287040, 35291135, ++ERASE, 35287040, 35291135, ++SNULL, 35282943, 35287039, ++STORE, 33853440, 35282943, ++STORE, 35282944, 35287039, ++ERASE, 35282944, 35287039, ++STORE, 140249716486144, 140249721602047, ++STORE, 140249716453376, 140249721602047, ++STORE, 33853440, 35418111, ++SNULL, 35401727, 35418111, ++STORE, 33853440, 35401727, ++STORE, 35401728, 35418111, ++ERASE, 35401728, 35418111, ++SNULL, 35389439, 35401727, ++STORE, 33853440, 35389439, ++STORE, 35389440, 35401727, ++ERASE, 35389440, 35401727, ++STORE, 140249714356224, 140249721602047, ++STORE, 33853440, 35540991, ++STORE, 140249714339840, 140249721602047, ++STORE, 140249714077696, 140249721602047, ++STORE, 140249714069504, 140249721602047, ++STORE, 140249714061312, 140249721602047, ++STORE, 33853440, 35680255, ++SNULL, 35672063, 35680255, ++STORE, 33853440, 35672063, ++STORE, 35672064, 35680255, ++ERASE, 35672064, 35680255, ++SNULL, 35627007, 35672063, ++STORE, 33853440, 35627007, ++STORE, 35627008, 35672063, ++ERASE, 35627008, 35672063, ++STORE, 140249711964160, 140249721602047, ++STORE, 33853440, 35762175, ++SNULL, 35753983, 35762175, ++STORE, 33853440, 35753983, ++STORE, 35753984, 35762175, ++ERASE, 35753984, 35762175, ++SNULL, 35745791, 35753983, ++STORE, 33853440, 35745791, ++STORE, 35745792, 35753983, ++ERASE, 35745792, 35753983, ++STORE, 140249711955968, 140249721602047, ++STORE, 140249711947776, 140249721602047, ++STORE, 140249710899200, 140249721602047, ++STORE, 140249710866432, 140249721602047, ++STORE, 140249710600192, 140249721602047, ++SNULL, 140249744424959, 140249744560127, ++STORE, 140249744207872, 140249744424959, ++STORE, 140249744424960, 140249744560127, ++ERASE, 140249744424960, 140249744560127, ++STORE, 140249708503040, 140249721602047, ++STORE, 33853440, 35885055, ++STORE, 140249707978752, 140249721602047, ++STORE, 140249705881600, 140249721602047, ++STORE, 33853440, 36036607, ++STORE, 33853440, 36175871, ++STORE, 140249744551936, 140249744560127, ++STORE, 140249744543744, 140249744560127, ++STORE, 140249744535552, 140249744560127, ++STORE, 140249744527360, 140249744560127, ++STORE, 140249744519168, 140249744560127, ++STORE, 140249705619456, 140249721602047, ++STORE, 140249744510976, 140249744560127, ++STORE, 140249744502784, 140249744560127, ++STORE, 140249744494592, 140249744560127, ++STORE, 140249744486400, 140249744560127, ++STORE, 140249744478208, 140249744560127, ++STORE, 140249744470016, 140249744560127, ++STORE, 140249744461824, 140249744560127, ++STORE, 140249744453632, 140249744560127, ++STORE, 140249744445440, 140249744560127, ++STORE, 140249744437248, 140249744560127, ++STORE, 140249744429056, 140249744560127, ++STORE, 140249703522304, 140249721602047, ++STORE, 33853440, 36311039, ++STORE, 140249703489536, 140249721602047, ++STORE, 33853440, 36474879, ++STORE, 140249703456768, 140249721602047, ++STORE, 33853440, 36622335, ++STORE, 140249703424000, 140249721602047, ++STORE, 140249703391232, 140249721602047, ++STORE, 33853440, 36810751, ++STORE, 140249703358464, 140249721602047, ++STORE, 140249703325696, 140249721602047, ++SNULL, 36655103, 36810751, ++STORE, 33853440, 36655103, ++STORE, 36655104, 36810751, ++ERASE, 36655104, 36810751, ++SNULL, 36438015, 36655103, ++STORE, 33853440, 36438015, ++STORE, 36438016, 36655103, ++ERASE, 36438016, 36655103, ++STORE, 140249703317504, 140249721602047, ++STORE, 140249701220352, 140249721602047, ++STORE, 33853440, 36585471, ++STORE, 33853440, 36782079, ++STORE, 140249701212160, 140249721602047, ++STORE, 140249701203968, 140249721602047, ++STORE, 140249701195776, 140249721602047, ++STORE, 140249701187584, 140249721602047, ++STORE, 140249701179392, 140249721602047, ++STORE, 140249701171200, 140249721602047, ++STORE, 140249701163008, 140249721602047, ++STORE, 140249701154816, 140249721602047, ++STORE, 140249701146624, 140249721602047, ++STORE, 140249701138432, 140249721602047, ++STORE, 140249701130240, 140249721602047, ++STORE, 140249700081664, 140249721602047, ++STORE, 140249700073472, 140249721602047, ++STORE, 33853440, 36978687, ++STORE, 140249697976320, 140249721602047, ++STORE, 33853440, 37240831, ++STORE, 140249695879168, 140249721602047, ++STORE, 140249695870976, 140249721602047, ++STORE, 140249695862784, 140249721602047, ++STORE, 140249695854592, 140249721602047, ++STORE, 140249695326208, 140249721602047, ++SNULL, 140249710600191, 140249721602047, ++STORE, 140249695326208, 140249710600191, ++STORE, 140249710600192, 140249721602047, ++SNULL, 140249710600192, 140249710866431, ++STORE, 140249710866432, 140249721602047, ++STORE, 140249710600192, 140249710866431, ++ERASE, 140249710600192, 140249710866431, ++STORE, 140249691131904, 140249710600191, ++STORE, 33853440, 37474303, ++STORE, 140249710858240, 140249721602047, ++STORE, 140249710850048, 140249721602047, ++STORE, 140249710841856, 140249721602047, ++STORE, 140249710833664, 140249721602047, ++STORE, 140249710825472, 140249721602047, ++STORE, 140249710817280, 140249721602047, ++STORE, 140249710809088, 140249721602047, ++STORE, 140249710800896, 140249721602047, ++STORE, 140249710792704, 140249721602047, ++STORE, 140249710784512, 140249721602047, ++STORE, 140249710776320, 140249721602047, ++STORE, 140249710768128, 140249721602047, ++STORE, 140249710759936, 140249721602047, ++STORE, 140249710751744, 140249721602047, ++STORE, 140249710743552, 140249721602047, ++STORE, 140249710735360, 140249721602047, ++STORE, 140249689034752, 140249710600191, ++STORE, 140249710727168, 140249721602047, ++STORE, 140249686937600, 140249710600191, ++STORE, 33853440, 37867519, ++STORE, 140249684840448, 140249710600191, ++STORE, 140249710718976, 140249721602047, ++STORE, 140249682743296, 140249710600191, ++STORE, 140249710710784, 140249721602047, ++STORE, 140249710702592, 140249721602047, ++STORE, 140249710694400, 140249721602047, ++STORE, 140249710686208, 140249721602047, ++STORE, 140249710678016, 140249721602047, ++STORE, 140249682612224, 140249710600191, ++STORE, 140249682087936, 140249710600191, ++SNULL, 140249705619455, 140249710600191, ++STORE, 140249682087936, 140249705619455, ++STORE, 140249705619456, 140249710600191, ++SNULL, 140249705619456, 140249705881599, ++STORE, 140249705881600, 140249710600191, ++STORE, 140249705619456, 140249705881599, ++ERASE, 140249705619456, 140249705881599, ++STORE, 140249679990784, 140249705619455, ++STORE, 140249710669824, 140249721602047, ++STORE, 140249677893632, 140249705619455, ++STORE, 140249710653440, 140249721602047, ++STORE, 140249710645248, 140249721602047, ++STORE, 140249710637056, 140249721602047, ++STORE, 140249710628864, 140249721602047, ++STORE, 140249710620672, 140249721602047, ++STORE, 140249710612480, 140249721602047, ++STORE, 140249710604288, 140249721602047, ++STORE, 140249705873408, 140249710600191, ++STORE, 140249705865216, 140249710600191, ++STORE, 140249705857024, 140249710600191, ++STORE, 140249705848832, 140249710600191, ++STORE, 140249705840640, 140249710600191, ++STORE, 140249705832448, 140249710600191, ++STORE, 140249705824256, 140249710600191, ++STORE, 140249705816064, 140249710600191, ++STORE, 140249705807872, 140249710600191, ++STORE, 140249705799680, 140249710600191, ++STORE, 33853440, 38129663, ++SNULL, 140249744207872, 140249744367615, ++STORE, 140249744367616, 140249744424959, ++STORE, 140249744207872, 140249744367615, ++ERASE, 140249744207872, 140249744367615, ++STORE, 140249677606912, 140249705619455, ++STORE, 140249675509760, 140249705619455, ++SNULL, 140249677606911, 140249705619455, ++STORE, 140249675509760, 140249677606911, ++STORE, 140249677606912, 140249705619455, ++SNULL, 140249677606912, 140249677893631, ++STORE, 140249677893632, 140249705619455, ++STORE, 140249677606912, 140249677893631, ++ERASE, 140249677606912, 140249677893631, ++STORE, 140249744359424, 140249744424959, ++STORE, 33853440, 38391807, ++STORE, 140249674981376, 140249677606911, ++STORE, 140249672884224, 140249677606911, ++SNULL, 140249719230463, 140249721602047, ++STORE, 140249710604288, 140249719230463, ++STORE, 140249719230464, 140249721602047, ++SNULL, 140249719230464, 140249719504895, ++STORE, 140249719504896, 140249721602047, ++STORE, 140249719230464, 140249719504895, ++ERASE, 140249719230464, 140249719504895, ++STORE, 140249744351232, 140249744424959, ++STORE, 140249744343040, 140249744424959, ++STORE, 140249744334848, 140249744424959, ++STORE, 140249744326656, 140249744424959, ++STORE, 140249744310272, 140249744424959, ++STORE, 140249744302080, 140249744424959, ++STORE, 140249744285696, 140249744424959, ++STORE, 140249744277504, 140249744424959, ++STORE, 140249744261120, 140249744424959, ++STORE, 140249744252928, 140249744424959, ++STORE, 140249744220160, 140249744424959, ++STORE, 140249744211968, 140249744424959, ++STORE, 140249719488512, 140249721602047, ++STORE, 140249744203776, 140249744424959, ++STORE, 140249719472128, 140249721602047, ++STORE, 140249719463936, 140249721602047, ++STORE, 140249719447552, 140249721602047, ++STORE, 140249719439360, 140249721602047, ++STORE, 140249719406592, 140249721602047, ++STORE, 140249719398400, 140249721602047, ++STORE, 140249719382016, 140249721602047, ++STORE, 140249719373824, 140249721602047, ++STORE, 140249719357440, 140249721602047, ++STORE, 140249719349248, 140249721602047, ++STORE, 140249719332864, 140249721602047, ++STORE, 140249719324672, 140249721602047, ++STORE, 140249719291904, 140249721602047, ++STORE, 140249719283712, 140249721602047, ++STORE, 140249719267328, 140249721602047, ++STORE, 140249719259136, 140249721602047, ++STORE, 140249719242752, 140249721602047, ++STORE, 140249719234560, 140249721602047, ++STORE, 140249705783296, 140249710600191, ++STORE, 140249705775104, 140249710600191, ++STORE, 140249705742336, 140249710600191, ++STORE, 140249705734144, 140249710600191, ++STORE, 140249705717760, 140249710600191, ++STORE, 140249670787072, 140249677606911, ++STORE, 140249705709568, 140249710600191, ++STORE, 140249705693184, 140249710600191, ++STORE, 140249705684992, 140249710600191, ++STORE, 140249705668608, 140249710600191, ++STORE, 140249705660416, 140249710600191, ++STORE, 140249705627648, 140249710600191, ++STORE, 140249677893632, 140249710600191, ++STORE, 140249677877248, 140249710600191, ++STORE, 140249677869056, 140249710600191, ++STORE, 140249677852672, 140249710600191, ++STORE, 140249677844480, 140249710600191, ++STORE, 140249677828096, 140249710600191, ++STORE, 140249668689920, 140249677606911, ++STORE, 140249677819904, 140249710600191, ++STORE, 140249677787136, 140249710600191, ++STORE, 140249677778944, 140249710600191, ++STORE, 140249677762560, 140249710600191, ++STORE, 140249677754368, 140249710600191, ++STORE, 140249677737984, 140249710600191, ++STORE, 140249677729792, 140249710600191, ++STORE, 140249677713408, 140249710600191, ++STORE, 140249677705216, 140249710600191, ++STORE, 140249677672448, 140249710600191, ++STORE, 140249677664256, 140249710600191, ++STORE, 140249677647872, 140249710600191, ++STORE, 140249677639680, 140249710600191, ++STORE, 140249677623296, 140249710600191, ++STORE, 140249677615104, 140249710600191, ++STORE, 140249668673536, 140249677606911, ++STORE, 140249668673536, 140249710600191, ++STORE, 140249668640768, 140249710600191, ++STORE, 140249668632576, 140249710600191, ++STORE, 140249668616192, 140249710600191, ++STORE, 140249668608000, 140249710600191, ++STORE, 140249668591616, 140249710600191, ++STORE, 140249668583424, 140249710600191, ++STORE, 140249668567040, 140249710600191, ++STORE, 140249668558848, 140249710600191, ++STORE, 140249668526080, 140249710600191, ++STORE, 140249668517888, 140249710600191, ++STORE, 140249668501504, 140249710600191, ++STORE, 140249668493312, 140249710600191, ++STORE, 140249668476928, 140249710600191, ++STORE, 140249668468736, 140249710600191, ++STORE, 140249668452352, 140249710600191, ++STORE, 140249668444160, 140249710600191, ++STORE, 140249668411392, 140249710600191, ++STORE, 140249668403200, 140249710600191, ++STORE, 140249668386816, 140249710600191, ++STORE, 140249668378624, 140249710600191, ++STORE, 140249668362240, 140249710600191, ++STORE, 140249668354048, 140249710600191, ++STORE, 140249668337664, 140249710600191, ++STORE, 140249668329472, 140249710600191, ++STORE, 140249668296704, 140249710600191, ++STORE, 140249668288512, 140249710600191, ++STORE, 140249668272128, 140249710600191, ++STORE, 140249668263936, 140249710600191, ++STORE, 140249668247552, 140249710600191, ++STORE, 140249668239360, 140249710600191, ++STORE, 140249668222976, 140249710600191, ++STORE, 140249668214784, 140249710600191, ++STORE, 140249668182016, 140249710600191, ++STORE, 140249668173824, 140249710600191, ++STORE, 140249668157440, 140249710600191, ++STORE, 140249668149248, 140249710600191, ++STORE, 140249668132864, 140249710600191, ++STORE, 140249668124672, 140249710600191, ++STORE, 140249668108288, 140249710600191, ++STORE, 140249668100096, 140249710600191, ++STORE, 140249668067328, 140249710600191, ++STORE, 140249668059136, 140249710600191, ++STORE, 140249668042752, 140249710600191, ++STORE, 140249668034560, 140249710600191, ++STORE, 140249668018176, 140249710600191, ++STORE, 140249668009984, 140249710600191, ++STORE, 140249667993600, 140249710600191, ++STORE, 140249667985408, 140249710600191, ++STORE, 140249667952640, 140249710600191, ++STORE, 140249667944448, 140249710600191, ++STORE, 140249667928064, 140249710600191, ++STORE, 140249667919872, 140249710600191, ++STORE, 140249667903488, 140249710600191, ++STORE, 140249667895296, 140249710600191, ++STORE, 140249667878912, 140249710600191, ++STORE, 140249667870720, 140249710600191, ++STORE, 140249667837952, 140249710600191, ++STORE, 140249667829760, 140249710600191, ++STORE, 140249667813376, 140249710600191, ++STORE, 140249667805184, 140249710600191, ++STORE, 140249667788800, 140249710600191, ++STORE, 140249667780608, 140249710600191, ++STORE, 140249667764224, 140249710600191, ++STORE, 140249667756032, 140249710600191, ++STORE, 140249667723264, 140249710600191, ++STORE, 140249667715072, 140249710600191, ++STORE, 140249667698688, 140249710600191, ++STORE, 140249667690496, 140249710600191, ++STORE, 140249667674112, 140249710600191, ++STORE, 140249667665920, 140249710600191, ++STORE, 140249667649536, 140249710600191, ++STORE, 140249667641344, 140249710600191, ++STORE, 140249667608576, 140249710600191, ++STORE, 140249667600384, 140249710600191, ++STORE, 140249667584000, 140249710600191, ++STORE, 140249667575808, 140249710600191, ++STORE, 140249667559424, 140249710600191, ++STORE, 140249667551232, 140249710600191, ++STORE, 140249667534848, 140249710600191, ++STORE, 140249667526656, 140249710600191, ++STORE, 140249667493888, 140249710600191, ++STORE, 140249667485696, 140249710600191, ++STORE, 140249667469312, 140249710600191, ++STORE, 140249667461120, 140249710600191, ++STORE, 140249667444736, 140249710600191, ++STORE, 140249667436544, 140249710600191, ++STORE, 140249667420160, 140249710600191, ++STORE, 140249665323008, 140249710600191, ++STORE, 140249665314816, 140249710600191, ++STORE, 140249665282048, 140249710600191, ++STORE, 140249665273856, 140249710600191, ++STORE, 140249665257472, 140249710600191, ++STORE, 140249665249280, 140249710600191, ++STORE, 140249665232896, 140249710600191, ++STORE, 140249665224704, 140249710600191, ++STORE, 140249665208320, 140249710600191, ++STORE, 140249665200128, 140249710600191, ++STORE, 140249665167360, 140249710600191, ++STORE, 140249665159168, 140249710600191, ++STORE, 140249665142784, 140249710600191, ++STORE, 140249665134592, 140249710600191, ++STORE, 140249665118208, 140249710600191, ++STORE, 140249665110016, 140249710600191, ++STORE, 140249665093632, 140249710600191, ++STORE, 140249665085440, 140249710600191, ++STORE, 140249665052672, 140249710600191, ++STORE, 140249665044480, 140249710600191, ++STORE, 140249665028096, 140249710600191, ++STORE, 140249665019904, 140249710600191, ++STORE, 140249665003520, 140249710600191, ++STORE, 140249664995328, 140249710600191, ++STORE, 140249664978944, 140249710600191, ++STORE, 140249664970752, 140249710600191, ++STORE, 140249664937984, 140249710600191, ++STORE, 140249664929792, 140249710600191, ++STORE, 140249664913408, 140249710600191, ++STORE, 140249664905216, 140249710600191, ++STORE, 140249664888832, 140249710600191, ++STORE, 140249664880640, 140249710600191, ++STORE, 140249664864256, 140249710600191, ++STORE, 140249664856064, 140249710600191, ++STORE, 140249664823296, 140249710600191, ++STORE, 140249664815104, 140249710600191, ++STORE, 140249664798720, 140249710600191, ++STORE, 140249664790528, 140249710600191, ++STORE, 140249664774144, 140249710600191, ++STORE, 140249664765952, 140249710600191, ++STORE, 140249664749568, 140249710600191, ++STORE, 140249664741376, 140249710600191, ++STORE, 140249664708608, 140249710600191, ++STORE, 140249664700416, 140249710600191, ++STORE, 140249664684032, 140249710600191, ++STORE, 140249664675840, 140249710600191, ++STORE, 140249664659456, 140249710600191, ++STORE, 140249664651264, 140249710600191, ++STORE, 140249664634880, 140249710600191, ++STORE, 140249664626688, 140249710600191, ++STORE, 140249664593920, 140249710600191, ++STORE, 140249664585728, 140249710600191, ++STORE, 140249664569344, 140249710600191, ++STORE, 140249664561152, 140249710600191, ++STORE, 140249664544768, 140249710600191, ++STORE, 140249664536576, 140249710600191, ++STORE, 140249664520192, 140249710600191, ++STORE, 140249664512000, 140249710600191, ++STORE, 140249664479232, 140249710600191, ++STORE, 140249664471040, 140249710600191, ++STORE, 140249664454656, 140249710600191, ++STORE, 140249664446464, 140249710600191, ++STORE, 140249664430080, 140249710600191, ++STORE, 140249664421888, 140249710600191, ++STORE, 140249664405504, 140249710600191, ++STORE, 140249664397312, 140249710600191, ++STORE, 140249664364544, 140249710600191, ++STORE, 140249664356352, 140249710600191, ++STORE, 140249664339968, 140249710600191, ++STORE, 140249664331776, 140249710600191, ++STORE, 140249664315392, 140249710600191, ++STORE, 140249664307200, 140249710600191, ++STORE, 140249664290816, 140249710600191, ++STORE, 140249664282624, 140249710600191, ++STORE, 140249664249856, 140249710600191, ++STORE, 140249664241664, 140249710600191, ++STORE, 140249664225280, 140249710600191, ++STORE, 140249664217088, 140249710600191, ++STORE, 140249664200704, 140249710600191, ++STORE, 140249664192512, 140249710600191, ++STORE, 140249664176128, 140249710600191, ++STORE, 140249664167936, 140249710600191, ++STORE, 140249664135168, 140249710600191, ++STORE, 140249664126976, 140249710600191, ++STORE, 140249664110592, 140249710600191, ++STORE, 140249664102400, 140249710600191, ++STORE, 140249664086016, 140249710600191, ++STORE, 140249664077824, 140249710600191, ++STORE, 140249664061440, 140249710600191, ++STORE, 140249664053248, 140249710600191, ++STORE, 140249664020480, 140249710600191, ++STORE, 140249664012288, 140249710600191, ++STORE, 140249663995904, 140249710600191, ++STORE, 140249663987712, 140249710600191, ++STORE, 140249663971328, 140249710600191, ++STORE, 140249663963136, 140249710600191, ++STORE, 140249663946752, 140249710600191, ++STORE, 140249663938560, 140249710600191, ++STORE, 140249663905792, 140249710600191, ++STORE, 140249663897600, 140249710600191, ++STORE, 140249663881216, 140249710600191, ++STORE, 140249663873024, 140249710600191, ++STORE, 140249663856640, 140249710600191, ++STORE, 140249663848448, 140249710600191, ++STORE, 140249663832064, 140249710600191, ++STORE, 140249663823872, 140249710600191, ++STORE, 140249663791104, 140249710600191, ++STORE, 140249663782912, 140249710600191, ++STORE, 140249663766528, 140249710600191, ++STORE, 140249663758336, 140249710600191, ++STORE, 140249663741952, 140249710600191, ++STORE, 140249663733760, 140249710600191, ++STORE, 140249663717376, 140249710600191, ++STORE, 140249663709184, 140249710600191, ++STORE, 140249663676416, 140249710600191, ++STORE, 140249663668224, 140249710600191, ++STORE, 140249663651840, 140249710600191, ++STORE, 140249663643648, 140249710600191, ++STORE, 140249663627264, 140249710600191, ++STORE, 33853440, 38526975, ++STORE, 140249663619072, 140249710600191, ++STORE, 140249663602688, 140249710600191, ++STORE, 140249661505536, 140249710600191, ++STORE, 140249661497344, 140249710600191, ++STORE, 140249661464576, 140249710600191, ++STORE, 140249661456384, 140249710600191, ++STORE, 140249661440000, 140249710600191, ++STORE, 140249661431808, 140249710600191, ++STORE, 140249661415424, 140249710600191, ++STORE, 140249661407232, 140249710600191, ++STORE, 140249661390848, 140249710600191, ++STORE, 140249661382656, 140249710600191, ++STORE, 140249661349888, 140249710600191, ++STORE, 140249661341696, 140249710600191, ++STORE, 140249661325312, 140249710600191, ++STORE, 140249661317120, 140249710600191, ++STORE, 140249661300736, 140249710600191, ++STORE, 140249661292544, 140249710600191, ++STORE, 140249661276160, 140249710600191, ++STORE, 140249661267968, 140249710600191, ++STORE, 140249661235200, 140249710600191, ++STORE, 140249661227008, 140249710600191, ++STORE, 140249661210624, 140249710600191, ++STORE, 140249661202432, 140249710600191, ++STORE, 140249661186048, 140249710600191, ++STORE, 140249661177856, 140249710600191, ++STORE, 140249661161472, 140249710600191, ++STORE, 140249661153280, 140249710600191, ++STORE, 140249661120512, 140249710600191, ++STORE, 140249661112320, 140249710600191, ++STORE, 140249661095936, 140249710600191, ++STORE, 140249661087744, 140249710600191, ++STORE, 140249661071360, 140249710600191, ++STORE, 140249661063168, 140249710600191, ++STORE, 140249661046784, 140249710600191, ++STORE, 140249661038592, 140249710600191, ++STORE, 140249661005824, 140249710600191, ++STORE, 140249660997632, 140249710600191, ++STORE, 140249660981248, 140249710600191, ++STORE, 140249660973056, 140249710600191, ++STORE, 140249660956672, 140249710600191, ++STORE, 140249660948480, 140249710600191, ++STORE, 140249660932096, 140249710600191, ++STORE, 140249660923904, 140249710600191, ++STORE, 140249660891136, 140249710600191, ++STORE, 140249660882944, 140249710600191, ++STORE, 140249660866560, 140249710600191, ++STORE, 140249660858368, 140249710600191, ++STORE, 140249660841984, 140249710600191, ++STORE, 140249660833792, 140249710600191, ++STORE, 140249660817408, 140249710600191, ++STORE, 140249660809216, 140249710600191, ++STORE, 140249660776448, 140249710600191, ++STORE, 140249660768256, 140249710600191, ++STORE, 140249660751872, 140249710600191, ++STORE, 140249660743680, 140249710600191, ++STORE, 140249660727296, 140249710600191, ++STORE, 140249660719104, 140249710600191, ++STORE, 140249660702720, 140249710600191, ++STORE, 140249660694528, 140249710600191, ++STORE, 140249660661760, 140249710600191, ++STORE, 140249660653568, 140249710600191, ++STORE, 140249660637184, 140249710600191, ++STORE, 140249660628992, 140249710600191, ++STORE, 140249660612608, 140249710600191, ++STORE, 140249660604416, 140249710600191, ++STORE, 140249660588032, 140249710600191, ++STORE, 140249660579840, 140249710600191, ++STORE, 140249660547072, 140249710600191, ++STORE, 140249660538880, 140249710600191, ++STORE, 140249660522496, 140249710600191, ++STORE, 140249660514304, 140249710600191, ++STORE, 140249660497920, 140249710600191, ++STORE, 140249660489728, 140249710600191, ++STORE, 140249660473344, 140249710600191, ++STORE, 140249660465152, 140249710600191, ++STORE, 140249660432384, 140249710600191, ++STORE, 140249660424192, 140249710600191, ++STORE, 140249660407808, 140249710600191, ++STORE, 140249660399616, 140249710600191, ++STORE, 140249660383232, 140249710600191, ++STORE, 140249660375040, 140249710600191, ++STORE, 140249660358656, 140249710600191, ++STORE, 140249660350464, 140249710600191, ++STORE, 140249660317696, 140249710600191, ++STORE, 140249660309504, 140249710600191, ++STORE, 140249660293120, 140249710600191, ++STORE, 140249660284928, 140249710600191, ++STORE, 140249660268544, 140249710600191, ++STORE, 140249660260352, 140249710600191, ++STORE, 140249660243968, 140249710600191, ++STORE, 140249660235776, 140249710600191, ++STORE, 140249660203008, 140249710600191, ++STORE, 140249660194816, 140249710600191, ++STORE, 140249660178432, 140249710600191, ++STORE, 140249660170240, 140249710600191, ++STORE, 140249660153856, 140249710600191, ++STORE, 140249660145664, 140249710600191, ++STORE, 140249660129280, 140249710600191, ++STORE, 140249660121088, 140249710600191, ++STORE, 140249660088320, 140249710600191, ++STORE, 140249660080128, 140249710600191, ++STORE, 140249660063744, 140249710600191, ++STORE, 140249660055552, 140249710600191, ++STORE, 140249660039168, 140249710600191, ++STORE, 140249660030976, 140249710600191, ++STORE, 140249660014592, 140249710600191, ++STORE, 140249660006400, 140249710600191, ++STORE, 140249659973632, 140249710600191, ++STORE, 140249659965440, 140249710600191, ++STORE, 140249659949056, 140249710600191, ++STORE, 140249659940864, 140249710600191, ++STORE, 140249659924480, 140249710600191, ++STORE, 140249659916288, 140249710600191, ++STORE, 140249659899904, 140249710600191, ++STORE, 140249659891712, 140249710600191, ++STORE, 140249659858944, 140249710600191, ++STORE, 140249659850752, 140249710600191, ++STORE, 140249659834368, 140249710600191, ++STORE, 140249659826176, 140249710600191, ++STORE, 140249659809792, 140249710600191, ++STORE, 140249659801600, 140249710600191, ++STORE, 140249659785216, 140249710600191, ++STORE, 140249657688064, 140249710600191, ++STORE, 140249657679872, 140249710600191, ++STORE, 140249657647104, 140249710600191, ++STORE, 140249657638912, 140249710600191, ++STORE, 140249657622528, 140249710600191, ++STORE, 140249657614336, 140249710600191, ++STORE, 140249657597952, 140249710600191, ++STORE, 140249657589760, 140249710600191, ++STORE, 140249657573376, 140249710600191, ++STORE, 140249657565184, 140249710600191, ++STORE, 140249657532416, 140249710600191, ++STORE, 140249657524224, 140249710600191, ++STORE, 140249657507840, 140249710600191, ++STORE, 140249657499648, 140249710600191, ++STORE, 140249657483264, 140249710600191, ++STORE, 140249657475072, 140249710600191, ++STORE, 140249657458688, 140249710600191, ++STORE, 140249657450496, 140249710600191, ++STORE, 140249657417728, 140249710600191, ++STORE, 140249657409536, 140249710600191, ++STORE, 140249657393152, 140249710600191, ++STORE, 140249657384960, 140249710600191, ++STORE, 140249657368576, 140249710600191, ++STORE, 140249657360384, 140249710600191, ++STORE, 140249657344000, 140249710600191, ++STORE, 140249657335808, 140249710600191, ++STORE, 140249657303040, 140249710600191, ++STORE, 140249657294848, 140249710600191, ++STORE, 140249657278464, 140249710600191, ++STORE, 140249657270272, 140249710600191, ++STORE, 140249657253888, 140249710600191, ++STORE, 140249657245696, 140249710600191, ++STORE, 140249657229312, 140249710600191, ++STORE, 140249657221120, 140249710600191, ++STORE, 140249657188352, 140249710600191, ++STORE, 140249657180160, 140249710600191, ++STORE, 140249657163776, 140249710600191, ++STORE, 140249657155584, 140249710600191, ++STORE, 140249657139200, 140249710600191, ++STORE, 140249657131008, 140249710600191, ++STORE, 140249657114624, 140249710600191, ++STORE, 140249657106432, 140249710600191, ++STORE, 140249657073664, 140249710600191, ++STORE, 140249657065472, 140249710600191, ++STORE, 140249657049088, 140249710600191, ++STORE, 140249657040896, 140249710600191, ++STORE, 140249657024512, 140249710600191, ++STORE, 140249657016320, 140249710600191, ++STORE, 140249656999936, 140249710600191, ++STORE, 140249656991744, 140249710600191, ++STORE, 140249656958976, 140249710600191, ++STORE, 140249656950784, 140249710600191, ++STORE, 140249656934400, 140249710600191, ++STORE, 140249656926208, 140249710600191, ++STORE, 140249656909824, 140249710600191, ++STORE, 140249656901632, 140249710600191, ++STORE, 140249656885248, 140249710600191, ++STORE, 140249656877056, 140249710600191, ++STORE, 140249656844288, 140249710600191, ++STORE, 140249656836096, 140249710600191, ++STORE, 140249656819712, 140249710600191, ++STORE, 140249656811520, 140249710600191, ++STORE, 140249656795136, 140249710600191, ++STORE, 33853440, 38662143, ++STORE, 140249656786944, 140249710600191, ++STORE, 140249656770560, 140249710600191, ++STORE, 140249656762368, 140249710600191, ++STORE, 140249656729600, 140249710600191, ++STORE, 140249656721408, 140249710600191, ++STORE, 140249656705024, 140249710600191, ++STORE, 140249656696832, 140249710600191, ++STORE, 140249656680448, 140249710600191, ++STORE, 140249656672256, 140249710600191, ++STORE, 140249656655872, 140249710600191, ++STORE, 140249656647680, 140249710600191, ++STORE, 140249656614912, 140249710600191, ++STORE, 140249656606720, 140249710600191, ++STORE, 140249656590336, 140249710600191, ++STORE, 140249656582144, 140249710600191, ++STORE, 140249656565760, 140249710600191, ++STORE, 140249656557568, 140249710600191, ++STORE, 140249656541184, 140249710600191, ++STORE, 140249656532992, 140249710600191, ++STORE, 140249656500224, 140249710600191, ++STORE, 140249656492032, 140249710600191, ++STORE, 140249656475648, 140249710600191, ++STORE, 140249656467456, 140249710600191, ++STORE, 140249656451072, 140249710600191, ++STORE, 140249656442880, 140249710600191, ++STORE, 140249656426496, 140249710600191, ++STORE, 140249656418304, 140249710600191, ++STORE, 140249656385536, 140249710600191, ++STORE, 140249656377344, 140249710600191, ++STORE, 140249656360960, 140249710600191, ++STORE, 140249656352768, 140249710600191, ++STORE, 140249656336384, 140249710600191, ++STORE, 140249656328192, 140249710600191, ++STORE, 140249656311808, 140249710600191, ++STORE, 140249656303616, 140249710600191, ++STORE, 140249656270848, 140249710600191, ++STORE, 140249656262656, 140249710600191, ++STORE, 140249656246272, 140249710600191, ++STORE, 140249656238080, 140249710600191, ++STORE, 140249656221696, 140249710600191, ++STORE, 140249656213504, 140249710600191, ++STORE, 140249656197120, 140249710600191, ++STORE, 140249656188928, 140249710600191, ++STORE, 140249656156160, 140249710600191, ++STORE, 140249656147968, 140249710600191, ++STORE, 140249656131584, 140249710600191, ++STORE, 140249656123392, 140249710600191, ++STORE, 140249656107008, 140249710600191, ++STORE, 140249656098816, 140249710600191, ++STORE, 140249656082432, 140249710600191, ++STORE, 140249656074240, 140249710600191, ++STORE, 140249656041472, 140249710600191, ++STORE, 140249656033280, 140249710600191, ++STORE, 140249656016896, 140249710600191, ++STORE, 140249656008704, 140249710600191, ++STORE, 140249655992320, 140249710600191, ++STORE, 140249655984128, 140249710600191, ++STORE, 140249655967744, 140249710600191, ++STORE, 140249653870592, 140249710600191, ++STORE, 140249653862400, 140249710600191, ++STORE, 140249653829632, 140249710600191, ++STORE, 140249653821440, 140249710600191, ++STORE, 140249653805056, 140249710600191, ++STORE, 140249653796864, 140249710600191, ++STORE, 140249653780480, 140249710600191, ++STORE, 140249653772288, 140249710600191, ++STORE, 140249653755904, 140249710600191, ++STORE, 140249652703232, 140249710600191, ++SNULL, 140249682087935, 140249710600191, ++STORE, 140249652703232, 140249682087935, ++STORE, 140249682087936, 140249710600191, ++ }; ++ ++ unsigned long set26[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140729464770560, 140737488351231, ++SNULL, 140729464774655, 140737488351231, ++STORE, 140729464770560, 140729464774655, ++STORE, 140729464639488, 140729464774655, ++STORE, 4194304, 5066751, ++STORE, 7159808, 7172095, ++STORE, 7172096, 7180287, ++STORE, 140729465114624, 140729465118719, ++STORE, 140729465102336, 140729465114623, ++STORE, 30867456, 30875647, ++STORE, 30867456, 31010815, ++STORE, 140109040988160, 140109042671615, ++STORE, 140109040959488, 140109040988159, ++STORE, 140109040943104, 140109040959487, ++ERASE, 140109040943104, 140109040959487, ++STORE, 140109040840704, 140109040959487, ++ERASE, 140109040840704, 140109040959487, ++STORE, 140109040951296, 140109040959487, ++ERASE, 140109040951296, 140109040959487, ++STORE, 140109040955392, 140109040959487, ++ERASE, 140109040955392, 140109040959487, ++ }; ++ unsigned long set27[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140726128070656, 140737488351231, ++SNULL, 140726128074751, 140737488351231, ++STORE, 140726128070656, 140726128074751, ++STORE, 140726127939584, 140726128074751, ++STORE, 94478497189888, 94478499303423, ++SNULL, 94478497202175, 94478499303423, ++STORE, 94478497189888, 94478497202175, ++STORE, 94478497202176, 94478499303423, ++ERASE, 94478497202176, 94478499303423, ++STORE, 94478499295232, 94478499303423, ++STORE, 140415605723136, 140415607975935, ++SNULL, 140415605866495, 140415607975935, ++STORE, 140415605723136, 140415605866495, ++STORE, 140415605866496, 140415607975935, ++ERASE, 140415605866496, 140415607975935, ++STORE, 140415607963648, 140415607971839, ++STORE, 140415607971840, 140415607975935, ++STORE, 140726130024448, 140726130028543, ++STORE, 140726130012160, 140726130024447, ++STORE, 140415607934976, 140415607963647, ++STORE, 140415607926784, 140415607934975, ++STORE, 140415603245056, 140415605723135, ++SNULL, 140415603245056, 140415603613695, ++STORE, 140415603613696, 140415605723135, ++STORE, 140415603245056, 140415603613695, ++SNULL, 140415605710847, 140415605723135, ++STORE, 140415603613696, 140415605710847, ++STORE, 140415605710848, 140415605723135, ++ERASE, 140415605710848, 140415605723135, ++STORE, 140415605710848, 140415605723135, ++STORE, 140415599370240, 140415603245055, ++SNULL, 140415599370240, 140415601111039, ++STORE, 140415601111040, 140415603245055, ++STORE, 140415599370240, 140415601111039, ++SNULL, 140415603208191, 140415603245055, ++STORE, 140415601111040, 140415603208191, ++STORE, 140415603208192, 140415603245055, ++ERASE, 140415603208192, 140415603245055, ++STORE, 140415603208192, 140415603245055, ++STORE, 140415595692032, 140415599370239, ++SNULL, 140415595692032, 140415597207551, ++STORE, 140415597207552, 140415599370239, ++STORE, 140415595692032, 140415597207551, ++SNULL, 140415599304703, 140415599370239, ++STORE, 140415597207552, 140415599304703, ++STORE, 140415599304704, 140415599370239, ++SNULL, 140415599304704, 140415599353855, ++STORE, 140415599353856, 140415599370239, ++STORE, 140415599304704, 140415599353855, ++ERASE, 140415599304704, 140415599353855, ++STORE, 140415599304704, 140415599353855, ++ERASE, 140415599353856, 140415599370239, ++STORE, 140415599353856, 140415599370239, ++STORE, 140415593500672, 140415595692031, ++SNULL, 140415593500672, 140415593590783, ++STORE, 140415593590784, 140415595692031, ++STORE, 140415593500672, 140415593590783, ++SNULL, 140415595683839, 140415595692031, ++STORE, 140415593590784, 140415595683839, ++STORE, 140415595683840, 140415595692031, ++ERASE, 140415595683840, 140415595692031, ++STORE, 140415595683840, 140415595692031, ++STORE, 140415589703680, 140415593500671, ++SNULL, 140415589703680, 140415591362559, ++STORE, 140415591362560, 140415593500671, ++STORE, 140415589703680, 140415591362559, ++SNULL, 140415593459711, 140415593500671, ++STORE, 140415591362560, 140415593459711, ++STORE, 140415593459712, 140415593500671, ++SNULL, 140415593459712, 140415593484287, ++STORE, 140415593484288, 140415593500671, ++STORE, 140415593459712, 140415593484287, ++ERASE, 140415593459712, 140415593484287, ++STORE, 140415593459712, 140415593484287, ++ERASE, 140415593484288, 140415593500671, ++STORE, 140415593484288, 140415593500671, ++STORE, 140415587590144, 140415589703679, ++SNULL, 140415587590144, 140415587602431, ++STORE, 140415587602432, 140415589703679, ++STORE, 140415587590144, 140415587602431, ++SNULL, 140415589695487, 140415589703679, ++STORE, 140415587602432, 140415589695487, ++STORE, 140415589695488, 140415589703679, ++ERASE, 140415589695488, 140415589703679, ++STORE, 140415589695488, 140415589703679, ++STORE, 140415607918592, 140415607934975, ++STORE, 140415585398784, 140415587590143, ++SNULL, 140415585398784, 140415585480703, ++STORE, 140415585480704, 140415587590143, ++STORE, 140415585398784, 140415585480703, ++SNULL, 140415587573759, 140415587590143, ++STORE, 140415585480704, 140415587573759, ++STORE, 140415587573760, 140415587590143, ++SNULL, 140415587573760, 140415587581951, ++STORE, 140415587581952, 140415587590143, ++STORE, 140415587573760, 140415587581951, ++ERASE, 140415587573760, 140415587581951, ++STORE, 140415587573760, 140415587581951, ++ERASE, 140415587581952, 140415587590143, ++STORE, 140415587581952, 140415587590143, ++STORE, 140415583182848, 140415585398783, ++SNULL, 140415583182848, 140415583281151, ++STORE, 140415583281152, 140415585398783, ++STORE, 140415583182848, 140415583281151, ++SNULL, 140415585374207, 140415585398783, ++STORE, 140415583281152, 140415585374207, ++STORE, 140415585374208, 140415585398783, ++SNULL, 140415585374208, 140415585382399, ++STORE, 140415585382400, 140415585398783, ++STORE, 140415585374208, 140415585382399, ++ERASE, 140415585374208, 140415585382399, ++STORE, 140415585374208, 140415585382399, ++ERASE, 140415585382400, 140415585398783, ++STORE, 140415585382400, 140415585398783, ++STORE, 140415580979200, 140415583182847, ++SNULL, 140415580979200, 140415581081599, ++STORE, 140415581081600, 140415583182847, ++STORE, 140415580979200, 140415581081599, ++SNULL, 140415583174655, 140415583182847, ++STORE, 140415581081600, 140415583174655, ++STORE, 140415583174656, 140415583182847, ++ERASE, 140415583174656, 140415583182847, ++STORE, 140415583174656, 140415583182847, ++STORE, 140415578816512, 140415580979199, ++SNULL, 140415578816512, 140415578877951, ++STORE, 140415578877952, 140415580979199, ++STORE, 140415578816512, 140415578877951, ++SNULL, 140415580971007, 140415580979199, ++STORE, 140415578877952, 140415580971007, ++STORE, 140415580971008, 140415580979199, ++ERASE, 140415580971008, 140415580979199, ++STORE, 140415580971008, 140415580979199, ++STORE, 140415576563712, 140415578816511, ++SNULL, 140415576563712, 140415576715263, ++STORE, 140415576715264, 140415578816511, ++STORE, 140415576563712, 140415576715263, ++SNULL, 140415578808319, 140415578816511, ++STORE, 140415576715264, 140415578808319, ++STORE, 140415578808320, 140415578816511, ++ERASE, 140415578808320, 140415578816511, ++STORE, 140415578808320, 140415578816511, ++STORE, 140415574392832, 140415576563711, ++SNULL, 140415574392832, 140415574462463, ++STORE, 140415574462464, 140415576563711, ++STORE, 140415574392832, 140415574462463, ++SNULL, 140415576555519, 140415576563711, ++STORE, 140415574462464, 140415576555519, ++STORE, 140415576555520, 140415576563711, ++ERASE, 140415576555520, 140415576563711, ++STORE, 140415576555520, 140415576563711, ++STORE, 140415607910400, 140415607934975, ++STORE, 140415571230720, 140415574392831, ++SNULL, 140415571230720, 140415572291583, ++STORE, 140415572291584, 140415574392831, ++STORE, 140415571230720, 140415572291583, ++SNULL, 140415574384639, 140415574392831, ++STORE, 140415572291584, 140415574384639, ++STORE, 140415574384640, 140415574392831, ++ERASE, 140415574384640, 140415574392831, ++STORE, 140415574384640, 140415574392831, ++STORE, 140415607902208, 140415607934975, ++SNULL, 140415593476095, 140415593484287, ++STORE, 140415593459712, 140415593476095, ++STORE, 140415593476096, 140415593484287, ++SNULL, 140415574388735, 140415574392831, ++STORE, 140415574384640, 140415574388735, ++STORE, 140415574388736, 140415574392831, ++SNULL, 140415576559615, 140415576563711, ++STORE, 140415576555520, 140415576559615, ++STORE, 140415576559616, 140415576563711, ++SNULL, 140415589699583, 140415589703679, ++STORE, 140415589695488, 140415589699583, ++STORE, 140415589699584, 140415589703679, ++SNULL, 140415585378303, 140415585382399, ++STORE, 140415585374208, 140415585378303, ++STORE, 140415585378304, 140415585382399, ++SNULL, 140415578812415, 140415578816511, ++STORE, 140415578808320, 140415578812415, ++STORE, 140415578812416, 140415578816511, ++SNULL, 140415580975103, 140415580979199, ++STORE, 140415580971008, 140415580975103, ++STORE, 140415580975104, 140415580979199, ++SNULL, 140415583178751, 140415583182847, ++STORE, 140415583174656, 140415583178751, ++STORE, 140415583178752, 140415583182847, ++SNULL, 140415587577855, 140415587581951, ++STORE, 140415587573760, 140415587577855, ++STORE, 140415587577856, 140415587581951, ++SNULL, 140415595687935, 140415595692031, ++STORE, 140415595683840, 140415595687935, ++STORE, 140415595687936, 140415595692031, ++STORE, 140415607894016, 140415607934975, ++SNULL, 140415599345663, 140415599353855, ++STORE, 140415599304704, 140415599345663, ++STORE, 140415599345664, 140415599353855, ++SNULL, 140415603240959, 140415603245055, ++STORE, 140415603208192, 140415603240959, ++STORE, 140415603240960, 140415603245055, ++SNULL, 140415605719039, 140415605723135, ++STORE, 140415605710848, 140415605719039, ++STORE, 140415605719040, 140415605723135, ++SNULL, 94478499299327, 94478499303423, ++STORE, 94478499295232, 94478499299327, ++STORE, 94478499299328, 94478499303423, ++SNULL, 140415607967743, 140415607971839, ++STORE, 140415607963648, 140415607967743, ++STORE, 140415607967744, 140415607971839, ++ERASE, 140415607934976, 140415607963647, ++STORE, 94478511173632, 94478511378431, ++STORE, 140415606210560, 140415607894015, ++STORE, 140415607934976, 140415607963647, ++STORE, 94478511173632, 94478511513599, ++STORE, 94478511173632, 94478511648767, ++SNULL, 94478511615999, 94478511648767, ++STORE, 94478511173632, 94478511615999, ++STORE, 94478511616000, 94478511648767, ++ERASE, 94478511616000, 94478511648767, ++STORE, 94478511173632, 94478511751167, ++SNULL, 94478511747071, 94478511751167, ++STORE, 94478511173632, 94478511747071, ++STORE, 94478511747072, 94478511751167, ++ERASE, 94478511747072, 94478511751167, ++STORE, 94478511173632, 94478511882239, ++SNULL, 94478511878143, 94478511882239, ++STORE, 94478511173632, 94478511878143, ++STORE, 94478511878144, 94478511882239, ++ERASE, 94478511878144, 94478511882239, ++STORE, 94478511173632, 94478512013311, ++SNULL, 94478512009215, 94478512013311, ++STORE, 94478511173632, 94478512009215, ++STORE, 94478512009216, 94478512013311, ++ERASE, 94478512009216, 94478512013311, ++STORE, 94478511173632, 94478512144383, ++STORE, 94478511173632, 94478512279551, ++STORE, 140415606181888, 140415606210559, ++STORE, 140415569100800, 140415571230719, ++SNULL, 140415569100800, 140415569129471, ++STORE, 140415569129472, 140415571230719, ++STORE, 140415569100800, 140415569129471, ++SNULL, 140415571222527, 140415571230719, ++STORE, 140415569129472, 140415571222527, ++STORE, 140415571222528, 140415571230719, ++ERASE, 140415571222528, 140415571230719, ++STORE, 140415571222528, 140415571230719, ++STORE, 140415566905344, 140415569100799, ++SNULL, 140415566905344, 140415566987263, ++STORE, 140415566987264, 140415569100799, ++STORE, 140415566905344, 140415566987263, ++SNULL, 140415569084415, 140415569100799, ++STORE, 140415566987264, 140415569084415, ++STORE, 140415569084416, 140415569100799, ++SNULL, 140415569084416, 140415569092607, ++STORE, 140415569092608, 140415569100799, ++STORE, 140415569084416, 140415569092607, ++ERASE, 140415569084416, 140415569092607, ++STORE, 140415569084416, 140415569092607, ++ERASE, 140415569092608, 140415569100799, ++STORE, 140415569092608, 140415569100799, ++SNULL, 140415569088511, 140415569092607, ++STORE, 140415569084416, 140415569088511, ++STORE, 140415569088512, 140415569092607, ++SNULL, 140415571226623, 140415571230719, ++STORE, 140415571222528, 140415571226623, ++STORE, 140415571226624, 140415571230719, ++ERASE, 140415606181888, 140415606210559, ++STORE, 140415606181888, 140415606210559, ++STORE, 140415564759040, 140415566905343, ++SNULL, 140415564759040, 140415564804095, ++STORE, 140415564804096, 140415566905343, ++STORE, 140415564759040, 140415564804095, ++SNULL, 140415566897151, 140415566905343, ++STORE, 140415564804096, 140415566897151, ++STORE, 140415566897152, 140415566905343, ++ERASE, 140415566897152, 140415566905343, ++STORE, 140415566897152, 140415566905343, ++STORE, 140415562588160, 140415564759039, ++SNULL, 140415562588160, 140415562629119, ++STORE, 140415562629120, 140415564759039, ++STORE, 140415562588160, 140415562629119, ++SNULL, 140415564726271, 140415564759039, ++STORE, 140415562629120, 140415564726271, ++STORE, 140415564726272, 140415564759039, ++SNULL, 140415564726272, 140415564734463, ++STORE, 140415564734464, 140415564759039, ++STORE, 140415564726272, 140415564734463, ++ERASE, 140415564726272, 140415564734463, ++STORE, 140415564726272, 140415564734463, ++ERASE, 140415564734464, 140415564759039, ++STORE, 140415564734464, 140415564759039, ++SNULL, 140415564730367, 140415564734463, ++STORE, 140415564726272, 140415564730367, ++STORE, 140415564730368, 140415564734463, ++SNULL, 140415566901247, 140415566905343, ++STORE, 140415566897152, 140415566901247, ++STORE, 140415566901248, 140415566905343, ++ERASE, 140415606181888, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415605944320, 140415606210559, ++ERASE, 140415605944320, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 94478511173632, 94478512414719, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 94478511173632, 94478512652287, ++STORE, 94478511173632, 94478512787455, ++STORE, 94478511173632, 94478512922623, ++STORE, 94478511173632, 94478513057791, ++STORE, 140415537422336, 140415562588159, ++STORE, 94478511173632, 94478513192959, ++STORE, 94478511173632, 94478513356799, ++STORE, 94478511173632, 94478513491967, ++STORE, 94478511173632, 94478513627135, ++STORE, 94478511173632, 94478513790975, ++STORE, 94478511173632, 94478513926143, ++STORE, 94478511173632, 94478514061311, ++STORE, 94478511173632, 94478514196479, ++STORE, 94478511173632, 94478514331647, ++STORE, 94478511173632, 94478514606079, ++STORE, 94478511173632, 94478514741247, ++STORE, 94478511173632, 94478514876415, ++STORE, 94478511173632, 94478515011583, ++STORE, 94478511173632, 94478515146751, ++STORE, 94478511173632, 94478515281919, ++STORE, 94478511173632, 94478515474431, ++STORE, 94478511173632, 94478515609599, ++STORE, 94478511173632, 94478515744767, ++STORE, 140415536922624, 140415562588159, ++STORE, 94478511173632, 94478515879935, ++STORE, 94478511173632, 94478516015103, ++STORE, 94478511173632, 94478516150271, ++STORE, 94478511173632, 94478516285439, ++STORE, 94478511173632, 94478516420607, ++STORE, 94478511173632, 94478516555775, ++STORE, 94478511173632, 94478516690943, ++STORE, 94478511173632, 94478516826111, ++STORE, 94478511173632, 94478516961279, ++STORE, 94478511173632, 94478517231615, ++STORE, 94478511173632, 94478517366783, ++STORE, 94478511173632, 94478517501951, ++STORE, 94478511173632, 94478517637119, ++STORE, 94478511173632, 94478517772287, ++STORE, 94478511173632, 94478517907455, ++STORE, 94478511173632, 94478518042623, ++STORE, 94478511173632, 94478518177791, ++STORE, 94478511173632, 94478518312959, ++STORE, 94478511173632, 94478518448127, ++STORE, 140415535910912, 140415562588159, ++SNULL, 140415536922623, 140415562588159, ++STORE, 140415535910912, 140415536922623, ++STORE, 140415536922624, 140415562588159, ++SNULL, 140415536922624, 140415537422335, ++STORE, 140415537422336, 140415562588159, ++STORE, 140415536922624, 140415537422335, ++ERASE, 140415536922624, 140415537422335, ++STORE, 94478511173632, 94478518583295, ++STORE, 94478511173632, 94478518718463, ++STORE, 94478511173632, 94478518853631, ++STORE, 94478511173632, 94478518988799, ++STORE, 94478511173632, 94478519123967, ++STORE, 94478511173632, 94478519259135, ++STORE, 140415509696512, 140415535910911, ++ERASE, 140415537422336, 140415562588159, ++STORE, 140415482433536, 140415509696511, ++ }; ++ unsigned long set28[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722475622400, 140737488351231, ++SNULL, 140722475626495, 140737488351231, ++STORE, 140722475622400, 140722475626495, ++STORE, 140722475491328, 140722475626495, ++STORE, 93865834291200, 93865836548095, ++SNULL, 93865834422271, 93865836548095, ++STORE, 93865834291200, 93865834422271, ++STORE, 93865834422272, 93865836548095, ++ERASE, 93865834422272, 93865836548095, ++STORE, 93865836519424, 93865836527615, ++STORE, 93865836527616, 93865836548095, ++STORE, 139918411104256, 139918413357055, ++SNULL, 139918411247615, 139918413357055, ++STORE, 139918411104256, 139918411247615, ++STORE, 139918411247616, 139918413357055, ++ERASE, 139918411247616, 139918413357055, ++STORE, 139918413344768, 139918413352959, ++STORE, 139918413352960, 139918413357055, ++STORE, 140722476642304, 140722476646399, ++STORE, 140722476630016, 140722476642303, ++STORE, 139918413316096, 139918413344767, ++STORE, 139918413307904, 139918413316095, ++STORE, 139918408888320, 139918411104255, ++SNULL, 139918408888320, 139918408986623, ++STORE, 139918408986624, 139918411104255, ++STORE, 139918408888320, 139918408986623, ++SNULL, 139918411079679, 139918411104255, ++STORE, 139918408986624, 139918411079679, ++STORE, 139918411079680, 139918411104255, ++SNULL, 139918411079680, 139918411087871, ++STORE, 139918411087872, 139918411104255, ++STORE, 139918411079680, 139918411087871, ++ERASE, 139918411079680, 139918411087871, ++STORE, 139918411079680, 139918411087871, ++ERASE, 139918411087872, 139918411104255, ++STORE, 139918411087872, 139918411104255, ++STORE, 139918405091328, 139918408888319, ++SNULL, 139918405091328, 139918406750207, ++STORE, 139918406750208, 139918408888319, ++STORE, 139918405091328, 139918406750207, ++SNULL, 139918408847359, 139918408888319, ++STORE, 139918406750208, 139918408847359, ++STORE, 139918408847360, 139918408888319, ++SNULL, 139918408847360, 139918408871935, ++STORE, 139918408871936, 139918408888319, ++STORE, 139918408847360, 139918408871935, ++ERASE, 139918408847360, 139918408871935, ++STORE, 139918408847360, 139918408871935, ++ERASE, 139918408871936, 139918408888319, ++STORE, 139918408871936, 139918408888319, ++STORE, 139918413299712, 139918413316095, ++SNULL, 139918408863743, 139918408871935, ++STORE, 139918408847360, 139918408863743, ++STORE, 139918408863744, 139918408871935, ++SNULL, 139918411083775, 139918411087871, ++STORE, 139918411079680, 139918411083775, ++STORE, 139918411083776, 139918411087871, ++SNULL, 93865836523519, 93865836527615, ++STORE, 93865836519424, 93865836523519, ++STORE, 93865836523520, 93865836527615, ++SNULL, 139918413348863, 139918413352959, ++STORE, 139918413344768, 139918413348863, ++STORE, 139918413348864, 139918413352959, ++ERASE, 139918413316096, 139918413344767, ++STORE, 93865848528896, 93865848664063, ++ }; ++ unsigned long set29[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734467944448, 140737488351231, ++SNULL, 140734467948543, 140737488351231, ++STORE, 140734467944448, 140734467948543, ++STORE, 140734467813376, 140734467948543, ++STORE, 94880407924736, 94880410177535, ++SNULL, 94880408055807, 94880410177535, ++STORE, 94880407924736, 94880408055807, ++STORE, 94880408055808, 94880410177535, ++ERASE, 94880408055808, 94880410177535, ++STORE, 94880410148864, 94880410157055, ++STORE, 94880410157056, 94880410177535, ++STORE, 140143367815168, 140143370067967, ++SNULL, 140143367958527, 140143370067967, ++STORE, 140143367815168, 140143367958527, ++STORE, 140143367958528, 140143370067967, ++ERASE, 140143367958528, 140143370067967, ++STORE, 140143370055680, 140143370063871, ++STORE, 140143370063872, 140143370067967, ++STORE, 140734468329472, 140734468333567, ++STORE, 140734468317184, 140734468329471, ++STORE, 140143370027008, 140143370055679, ++STORE, 140143370018816, 140143370027007, ++STORE, 140143365599232, 140143367815167, ++SNULL, 140143365599232, 140143365697535, ++STORE, 140143365697536, 140143367815167, ++STORE, 140143365599232, 140143365697535, ++SNULL, 140143367790591, 140143367815167, ++STORE, 140143365697536, 140143367790591, ++STORE, 140143367790592, 140143367815167, ++SNULL, 140143367790592, 140143367798783, ++STORE, 140143367798784, 140143367815167, ++STORE, 140143367790592, 140143367798783, ++ERASE, 140143367790592, 140143367798783, ++STORE, 140143367790592, 140143367798783, ++ERASE, 140143367798784, 140143367815167, ++STORE, 140143367798784, 140143367815167, ++STORE, 140143361802240, 140143365599231, ++SNULL, 140143361802240, 140143363461119, ++STORE, 140143363461120, 140143365599231, ++STORE, 140143361802240, 140143363461119, ++SNULL, 140143365558271, 140143365599231, ++STORE, 140143363461120, 140143365558271, ++STORE, 140143365558272, 140143365599231, ++SNULL, 140143365558272, 140143365582847, ++STORE, 140143365582848, 140143365599231, ++STORE, 140143365558272, 140143365582847, ++ERASE, 140143365558272, 140143365582847, ++STORE, 140143365558272, 140143365582847, ++ERASE, 140143365582848, 140143365599231, ++STORE, 140143365582848, 140143365599231, ++STORE, 140143370010624, 140143370027007, ++SNULL, 140143365574655, 140143365582847, ++STORE, 140143365558272, 140143365574655, ++STORE, 140143365574656, 140143365582847, ++SNULL, 140143367794687, 140143367798783, ++STORE, 140143367790592, 140143367794687, ++STORE, 140143367794688, 140143367798783, ++SNULL, 94880410152959, 94880410157055, ++STORE, 94880410148864, 94880410152959, ++STORE, 94880410152960, 94880410157055, ++SNULL, 140143370059775, 140143370063871, ++STORE, 140143370055680, 140143370059775, ++STORE, 140143370059776, 140143370063871, ++ERASE, 140143370027008, 140143370055679, ++STORE, 94880442400768, 94880442535935, ++STORE, 140143353409536, 140143361802239, ++SNULL, 140143353413631, 140143361802239, ++STORE, 140143353409536, 140143353413631, ++STORE, 140143353413632, 140143361802239, ++STORE, 140143345016832, 140143353409535, ++STORE, 140143210799104, 140143345016831, ++SNULL, 140143210799104, 140143239364607, ++STORE, 140143239364608, 140143345016831, ++STORE, 140143210799104, 140143239364607, ++ERASE, 140143210799104, 140143239364607, ++SNULL, 140143306473471, 140143345016831, ++STORE, 140143239364608, 140143306473471, ++STORE, 140143306473472, 140143345016831, ++ERASE, 140143306473472, 140143345016831, ++SNULL, 140143239499775, 140143306473471, ++STORE, 140143239364608, 140143239499775, ++STORE, 140143239499776, 140143306473471, ++SNULL, 140143345020927, 140143353409535, ++STORE, 140143345016832, 140143345020927, ++STORE, 140143345020928, 140143353409535, ++STORE, 140143336624128, 140143345016831, ++SNULL, 140143336628223, 140143345016831, ++STORE, 140143336624128, 140143336628223, ++STORE, 140143336628224, 140143345016831, ++STORE, 140143328231424, 140143336624127, ++SNULL, 140143328235519, 140143336624127, ++STORE, 140143328231424, 140143328235519, ++STORE, 140143328235520, 140143336624127, ++STORE, 140143319838720, 140143328231423, ++SNULL, 140143319842815, 140143328231423, ++STORE, 140143319838720, 140143319842815, ++STORE, 140143319842816, 140143328231423, ++STORE, 140143311446016, 140143319838719, ++STORE, 140143105146880, 140143239364607, ++STORE, 140143096754176, 140143105146879, ++STORE, 140143029645312, 140143096754175, ++ERASE, 140143029645312, 140143096754175, ++STORE, 140142962536448, 140143096754175, ++SNULL, 140142962536448, 140142970929151, ++STORE, 140142970929152, 140143096754175, ++STORE, 140142962536448, 140142970929151, ++ERASE, 140142962536448, 140142970929151, ++STORE, 140142962536448, 140142970929151, ++STORE, 140142828318720, 140142962536447, ++STORE, 140142819926016, 140142828318719, ++SNULL, 140142828318720, 140142836711423, ++STORE, 140142836711424, 140142962536447, ++STORE, 140142828318720, 140142836711423, ++ERASE, 140142828318720, 140142836711423, ++SNULL, 140143172255743, 140143239364607, ++STORE, 140143105146880, 140143172255743, ++STORE, 140143172255744, 140143239364607, ++ERASE, 140143172255744, 140143239364607, ++SNULL, 140143105282047, 140143172255743, ++STORE, 140143105146880, 140143105282047, ++STORE, 140143105282048, 140143172255743, ++SNULL, 140143038038015, 140143096754175, ++STORE, 140142970929152, 140143038038015, ++STORE, 140143038038016, 140143096754175, ++ERASE, 140143038038016, 140143096754175, ++SNULL, 140142971064319, 140143038038015, ++STORE, 140142970929152, 140142971064319, ++STORE, 140142971064320, 140143038038015, ++SNULL, 140142903820287, 140142962536447, ++STORE, 140142836711424, 140142903820287, ++STORE, 140142903820288, 140142962536447, ++ERASE, 140142903820288, 140142962536447, ++SNULL, 140142836846591, 140142903820287, ++STORE, 140142836711424, 140142836846591, ++STORE, 140142836846592, 140142903820287, ++STORE, 140142685708288, 140142819926015, ++SNULL, 140143311450111, 140143319838719, ++STORE, 140143311446016, 140143311450111, ++STORE, 140143311450112, 140143319838719, ++SNULL, 140142962540543, 140142970929151, ++STORE, 140142962536448, 140142962540543, ++STORE, 140142962540544, 140142970929151, ++SNULL, 140142685708288, 140142702493695, ++STORE, 140142702493696, 140142819926015, ++STORE, 140142685708288, 140142702493695, ++ERASE, 140142685708288, 140142702493695, ++SNULL, 140142769602559, 140142819926015, ++STORE, 140142702493696, 140142769602559, ++STORE, 140142769602560, 140142819926015, ++ERASE, 140142769602560, 140142819926015, ++SNULL, 140142702628863, 140142769602559, ++STORE, 140142702493696, 140142702628863, ++STORE, 140142702628864, 140142769602559, ++STORE, 140143230971904, 140143239364607, ++SNULL, 140143230975999, 140143239364607, ++STORE, 140143230971904, 140143230975999, ++STORE, 140143230976000, 140143239364607, ++SNULL, 140143096758271, 140143105146879, ++STORE, 140143096754176, 140143096758271, ++STORE, 140143096758272, 140143105146879, ++STORE, 140143222579200, 140143230971903, ++SNULL, 140143222583295, 140143230971903, ++STORE, 140143222579200, 140143222583295, ++STORE, 140143222583296, 140143230971903, ++STORE, 140143214186496, 140143222579199, ++SNULL, 140142819930111, 140142828318719, ++STORE, 140142819926016, 140142819930111, ++STORE, 140142819930112, 140142828318719, ++STORE, 140143205793792, 140143222579199, ++SNULL, 140143205793792, 140143214186495, ++STORE, 140143214186496, 140143222579199, ++STORE, 140143205793792, 140143214186495, ++SNULL, 140143214190591, 140143222579199, ++STORE, 140143214186496, 140143214190591, ++STORE, 140143214190592, 140143222579199, ++SNULL, 140143205797887, 140143214186495, ++STORE, 140143205793792, 140143205797887, ++STORE, 140143205797888, 140143214186495, ++STORE, 140143197401088, 140143205793791, ++SNULL, 140143197405183, 140143205793791, ++STORE, 140143197401088, 140143197405183, ++STORE, 140143197405184, 140143205793791, ++STORE, 140143189008384, 140143197401087, ++STORE, 140143180615680, 140143197401087, ++STORE, 140143088361472, 140143096754175, ++SNULL, 140143180619775, 140143197401087, ++STORE, 140143180615680, 140143180619775, ++STORE, 140143180619776, 140143197401087, ++SNULL, 140143180619776, 140143189008383, ++STORE, 140143189008384, 140143197401087, ++STORE, 140143180619776, 140143189008383, ++SNULL, 140143189012479, 140143197401087, ++STORE, 140143189008384, 140143189012479, ++STORE, 140143189012480, 140143197401087, ++SNULL, 140143088365567, 140143096754175, ++STORE, 140143088361472, 140143088365567, ++STORE, 140143088365568, 140143096754175, ++STORE, 140143079968768, 140143088361471, ++SNULL, 140143079972863, 140143088361471, ++STORE, 140143079968768, 140143079972863, ++STORE, 140143079972864, 140143088361471, ++STORE, 140143071576064, 140143079968767, ++SNULL, 140143071580159, 140143079968767, ++STORE, 140143071576064, 140143071580159, ++STORE, 140143071580160, 140143079968767, ++STORE, 140143063183360, 140143071576063, ++STORE, 140143054790656, 140143071576063, ++SNULL, 140143054794751, 140143071576063, ++STORE, 140143054790656, 140143054794751, ++STORE, 140143054794752, 140143071576063, ++SNULL, 140143054794752, 140143063183359, ++STORE, 140143063183360, 140143071576063, ++STORE, 140143054794752, 140143063183359, ++SNULL, 140143063187455, 140143071576063, ++STORE, 140143063183360, 140143063187455, ++STORE, 140143063187456, 140143071576063, ++STORE, 140143046397952, 140143054790655, ++STORE, 140142954143744, 140142962536447, ++STORE, 140142945751040, 140142962536447, ++STORE, 140142937358336, 140142962536447, ++STORE, 140142928965632, 140142962536447, ++STORE, 140142568275968, 140142702493695, ++SNULL, 140142635384831, 140142702493695, ++STORE, 140142568275968, 140142635384831, ++STORE, 140142635384832, 140142702493695, ++ERASE, 140142635384832, 140142702493695, ++STORE, 140142920572928, 140142962536447, ++STORE, 140142912180224, 140142962536447, ++STORE, 140142568275968, 140142702493695, ++SNULL, 140142568275968, 140142635384831, ++STORE, 140142635384832, 140142702493695, ++STORE, 140142568275968, 140142635384831, ++SNULL, 140142635519999, 140142702493695, ++STORE, 140142635384832, 140142635519999, ++STORE, 140142635520000, 140142702493695, ++STORE, 140142819930112, 140142836711423, ++STORE, 140142811533312, 140142819926015, ++STORE, 140142434058240, 140142635384831, ++SNULL, 140142501167103, 140142635384831, ++STORE, 140142434058240, 140142501167103, ++STORE, 140142501167104, 140142635384831, ++SNULL, 140142501167104, 140142568275967, ++STORE, 140142568275968, 140142635384831, ++STORE, 140142501167104, 140142568275967, ++ERASE, 140142501167104, 140142568275967, ++STORE, 140142299840512, 140142501167103, ++STORE, 140142803140608, 140142819926015, ++SNULL, 140142366949375, 140142501167103, ++STORE, 140142299840512, 140142366949375, ++STORE, 140142366949376, 140142501167103, ++SNULL, 140142366949376, 140142434058239, ++STORE, 140142434058240, 140142501167103, ++STORE, 140142366949376, 140142434058239, ++ERASE, 140142366949376, 140142434058239, ++STORE, 140142794747904, 140142819926015, ++STORE, 140142786355200, 140142819926015, ++STORE, 140142299840512, 140142501167103, ++STORE, 140142777962496, 140142819926015, ++STORE, 140142559883264, 140142568275967, ++STORE, 140142232731648, 140142501167103, ++STORE, 140142551490560, 140142568275967, ++SNULL, 140142777962496, 140142803140607, ++STORE, 140142803140608, 140142819926015, ++STORE, 140142777962496, 140142803140607, ++SNULL, 140142803144703, 140142819926015, ++STORE, 140142803140608, 140142803144703, ++STORE, 140142803144704, 140142819926015, ++STORE, 140142543097856, 140142568275967, ++STORE, 140142098513920, 140142501167103, ++SNULL, 140142165622783, 140142501167103, ++STORE, 140142098513920, 140142165622783, ++STORE, 140142165622784, 140142501167103, ++SNULL, 140142165622784, 140142232731647, ++STORE, 140142232731648, 140142501167103, ++STORE, 140142165622784, 140142232731647, ++ERASE, 140142165622784, 140142232731647, ++SNULL, 140142568411135, 140142635384831, ++STORE, 140142568275968, 140142568411135, ++STORE, 140142568411136, 140142635384831, ++STORE, 140141964296192, 140142165622783, ++SNULL, 140142912180224, 140142928965631, ++STORE, 140142928965632, 140142962536447, ++STORE, 140142912180224, 140142928965631, ++SNULL, 140142928969727, 140142962536447, ++STORE, 140142928965632, 140142928969727, ++STORE, 140142928969728, 140142962536447, ++STORE, 140141830078464, 140142165622783, ++SNULL, 140142912184319, 140142928965631, ++STORE, 140142912180224, 140142912184319, ++STORE, 140142912184320, 140142928965631, ++SNULL, 140142232731648, 140142434058239, ++STORE, 140142434058240, 140142501167103, ++STORE, 140142232731648, 140142434058239, ++SNULL, 140142434193407, 140142501167103, ++STORE, 140142434058240, 140142434193407, ++STORE, 140142434193408, 140142501167103, ++SNULL, 140142232731648, 140142299840511, ++STORE, 140142299840512, 140142434058239, ++STORE, 140142232731648, 140142299840511, ++SNULL, 140142299975679, 140142434058239, ++STORE, 140142299840512, 140142299975679, ++STORE, 140142299975680, 140142434058239, ++SNULL, 140142928969728, 140142954143743, ++STORE, 140142954143744, 140142962536447, ++STORE, 140142928969728, 140142954143743, ++SNULL, 140142954147839, 140142962536447, ++STORE, 140142954143744, 140142954147839, ++STORE, 140142954147840, 140142962536447, ++STORE, 140141830078464, 140142299840511, ++SNULL, 140142543097856, 140142559883263, ++STORE, 140142559883264, 140142568275967, ++STORE, 140142543097856, 140142559883263, ++SNULL, 140142559887359, 140142568275967, ++STORE, 140142559883264, 140142559887359, ++STORE, 140142559887360, 140142568275967, ++STORE, 140142534705152, 140142559883263, ++SNULL, 140142928969728, 140142945751039, ++STORE, 140142945751040, 140142954143743, ++STORE, 140142928969728, 140142945751039, ++SNULL, 140142945755135, 140142954143743, ++STORE, 140142945751040, 140142945755135, ++STORE, 140142945755136, 140142954143743, ++SNULL, 140142299975680, 140142366949375, ++STORE, 140142366949376, 140142434058239, ++STORE, 140142299975680, 140142366949375, ++SNULL, 140142367084543, 140142434058239, ++STORE, 140142366949376, 140142367084543, ++STORE, 140142367084544, 140142434058239, ++SNULL, 140142928969728, 140142937358335, ++STORE, 140142937358336, 140142945751039, ++STORE, 140142928969728, 140142937358335, ++SNULL, 140142937362431, 140142945751039, ++STORE, 140142937358336, 140142937362431, ++STORE, 140142937362432, 140142945751039, ++SNULL, 140141830078464, 140142232731647, ++STORE, 140142232731648, 140142299840511, ++STORE, 140141830078464, 140142232731647, ++SNULL, 140142232866815, 140142299840511, ++STORE, 140142232731648, 140142232866815, ++STORE, 140142232866816, 140142299840511, ++SNULL, 140142534705152, 140142543097855, ++STORE, 140142543097856, 140142559883263, ++STORE, 140142534705152, 140142543097855, ++SNULL, 140142543101951, 140142559883263, ++STORE, 140142543097856, 140142543101951, ++STORE, 140142543101952, 140142559883263, ++STORE, 140142526312448, 140142543097855, ++STORE, 140142517919744, 140142543097855, ++SNULL, 140141830078464, 140142098513919, ++STORE, 140142098513920, 140142232731647, ++STORE, 140141830078464, 140142098513919, ++SNULL, 140142098649087, 140142232731647, ++STORE, 140142098513920, 140142098649087, ++STORE, 140142098649088, 140142232731647, ++SNULL, 140142031405055, 140142098513919, ++STORE, 140141830078464, 140142031405055, ++STORE, 140142031405056, 140142098513919, ++ERASE, 140142031405056, 140142098513919, ++SNULL, 140141830078464, 140141964296191, ++STORE, 140141964296192, 140142031405055, ++STORE, 140141830078464, 140141964296191, ++SNULL, 140141964431359, 140142031405055, ++STORE, 140141964296192, 140141964431359, ++STORE, 140141964431360, 140142031405055, ++STORE, 140142509527040, 140142543097855, ++SNULL, 140141897187327, 140141964296191, ++STORE, 140141830078464, 140141897187327, ++STORE, 140141897187328, 140141964296191, ++ERASE, 140141897187328, 140141964296191, ++SNULL, 140141830213631, 140141897187327, ++STORE, 140141830078464, 140141830213631, ++STORE, 140141830213632, 140141897187327, ++SNULL, 140142803144704, 140142811533311, ++STORE, 140142811533312, 140142819926015, ++STORE, 140142803144704, 140142811533311, ++SNULL, 140142811537407, 140142819926015, ++STORE, 140142811533312, 140142811537407, ++STORE, 140142811537408, 140142819926015, ++SNULL, 140142098649088, 140142165622783, ++STORE, 140142165622784, 140142232731647, ++STORE, 140142098649088, 140142165622783, ++SNULL, 140142165757951, 140142232731647, ++STORE, 140142165622784, 140142165757951, ++STORE, 140142165757952, 140142232731647, ++STORE, 140142090121216, 140142098513919, ++SNULL, 140142777962496, 140142786355199, ++STORE, 140142786355200, 140142803140607, ++STORE, 140142777962496, 140142786355199, ++SNULL, 140142786359295, 140142803140607, ++STORE, 140142786355200, 140142786359295, ++STORE, 140142786359296, 140142803140607, ++SNULL, 140142509527040, 140142534705151, ++STORE, 140142534705152, 140142543097855, ++STORE, 140142509527040, 140142534705151, ++SNULL, 140142534709247, 140142543097855, ++STORE, 140142534705152, 140142534709247, ++STORE, 140142534709248, 140142543097855, ++STORE, 140142081728512, 140142098513919, ++SNULL, 140142786359296, 140142794747903, ++STORE, 140142794747904, 140142803140607, ++STORE, 140142786359296, 140142794747903, ++SNULL, 140142794751999, 140142803140607, ++STORE, 140142794747904, 140142794751999, ++STORE, 140142794752000, 140142803140607, ++STORE, 140142073335808, 140142098513919, ++SNULL, 140142073339903, 140142098513919, ++STORE, 140142073335808, 140142073339903, ++STORE, 140142073339904, 140142098513919, ++SNULL, 140142543101952, 140142551490559, ++STORE, 140142551490560, 140142559883263, ++STORE, 140142543101952, 140142551490559, ++SNULL, 140142551494655, 140142559883263, ++STORE, 140142551490560, 140142551494655, ++STORE, 140142551494656, 140142559883263, ++SNULL, 140142509527040, 140142517919743, ++STORE, 140142517919744, 140142534705151, ++STORE, 140142509527040, 140142517919743, ++SNULL, 140142517923839, 140142534705151, ++STORE, 140142517919744, 140142517923839, ++STORE, 140142517923840, 140142534705151, ++STORE, 140142064943104, 140142073335807, ++SNULL, 140142073339904, 140142090121215, ++STORE, 140142090121216, 140142098513919, ++STORE, 140142073339904, 140142090121215, ++SNULL, 140142090125311, 140142098513919, ++STORE, 140142090121216, 140142090125311, ++STORE, 140142090125312, 140142098513919, ++STORE, 140142056550400, 140142073335807, ++SNULL, 140142056554495, 140142073335807, ++STORE, 140142056550400, 140142056554495, ++STORE, 140142056554496, 140142073335807, ++STORE, 140142048157696, 140142056550399, ++SNULL, 140142509531135, 140142517919743, ++STORE, 140142509527040, 140142509531135, ++STORE, 140142509531136, 140142517919743, ++SNULL, 140142777966591, 140142786355199, ++STORE, 140142777962496, 140142777966591, ++STORE, 140142777966592, 140142786355199, ++SNULL, 140143046402047, 140143054790655, ++STORE, 140143046397952, 140143046402047, ++STORE, 140143046402048, 140143054790655, ++SNULL, 140142912184320, 140142920572927, ++STORE, 140142920572928, 140142928965631, ++STORE, 140142912184320, 140142920572927, ++SNULL, 140142920577023, 140142928965631, ++STORE, 140142920572928, 140142920577023, ++STORE, 140142920577024, 140142928965631, ++STORE, 140142039764992, 140142056550399, ++STORE, 140141955903488, 140141964296191, ++SNULL, 140142819930112, 140142828318719, ++STORE, 140142828318720, 140142836711423, ++STORE, 140142819930112, 140142828318719, ++SNULL, 140142828322815, 140142836711423, ++STORE, 140142828318720, 140142828322815, ++STORE, 140142828322816, 140142836711423, ++SNULL, 140142517923840, 140142526312447, ++STORE, 140142526312448, 140142534705151, ++STORE, 140142517923840, 140142526312447, ++SNULL, 140142526316543, 140142534705151, ++STORE, 140142526312448, 140142526316543, ++STORE, 140142526316544, 140142534705151, ++STORE, 140141947510784, 140141964296191, ++SNULL, 140142056554496, 140142064943103, ++STORE, 140142064943104, 140142073335807, ++STORE, 140142056554496, 140142064943103, ++SNULL, 140142064947199, 140142073335807, ++STORE, 140142064943104, 140142064947199, ++STORE, 140142064947200, 140142073335807, ++SNULL, 140142073339904, 140142081728511, ++STORE, 140142081728512, 140142090121215, ++STORE, 140142073339904, 140142081728511, ++SNULL, 140142081732607, 140142090121215, ++STORE, 140142081728512, 140142081732607, ++STORE, 140142081732608, 140142090121215, ++STORE, 140141939118080, 140141964296191, ++STORE, 140141930725376, 140141964296191, ++STORE, 140141922332672, 140141964296191, ++STORE, 140141913939968, 140141964296191, ++SNULL, 140141913939968, 140141922332671, ++STORE, 140141922332672, 140141964296191, ++STORE, 140141913939968, 140141922332671, ++SNULL, 140141922336767, 140141964296191, ++STORE, 140141922332672, 140141922336767, ++STORE, 140141922336768, 140141964296191, ++STORE, 140141905547264, 140141922332671, ++SNULL, 140141905551359, 140141922332671, ++STORE, 140141905547264, 140141905551359, ++STORE, 140141905551360, 140141922332671, ++STORE, 140141821685760, 140141830078463, ++STORE, 140141813293056, 140141830078463, ++STORE, 140141804900352, 140141830078463, ++STORE, 140141796507648, 140141830078463, ++SNULL, 140141796511743, 140141830078463, ++STORE, 140141796507648, 140141796511743, ++STORE, 140141796511744, 140141830078463, ++SNULL, 140141922336768, 140141955903487, ++STORE, 140141955903488, 140141964296191, ++STORE, 140141922336768, 140141955903487, ++SNULL, 140141955907583, 140141964296191, ++STORE, 140141955903488, 140141955907583, ++STORE, 140141955907584, 140141964296191, ++STORE, 140141788114944, 140141796507647, ++STORE, 140141779722240, 140141796507647, ++SNULL, 140141779722240, 140141788114943, ++STORE, 140141788114944, 140141796507647, ++STORE, 140141779722240, 140141788114943, ++SNULL, 140141788119039, 140141796507647, ++STORE, 140141788114944, 140141788119039, ++STORE, 140141788119040, 140141796507647, ++SNULL, 140141922336768, 140141947510783, ++STORE, 140141947510784, 140141955903487, ++STORE, 140141922336768, 140141947510783, ++SNULL, 140141947514879, 140141955903487, ++STORE, 140141947510784, 140141947514879, ++STORE, 140141947514880, 140141955903487, ++SNULL, 140142039764992, 140142048157695, ++STORE, 140142048157696, 140142056550399, ++STORE, 140142039764992, 140142048157695, ++SNULL, 140142048161791, 140142056550399, ++STORE, 140142048157696, 140142048161791, ++STORE, 140142048161792, 140142056550399, ++SNULL, 140142039769087, 140142048157695, ++STORE, 140142039764992, 140142039769087, ++STORE, 140142039769088, 140142048157695, ++SNULL, 140141796511744, 140141804900351, ++STORE, 140141804900352, 140141830078463, ++STORE, 140141796511744, 140141804900351, ++SNULL, 140141804904447, 140141830078463, ++STORE, 140141804900352, 140141804904447, ++STORE, 140141804904448, 140141830078463, ++STORE, 140141771329536, 140141788114943, ++STORE, 140141762936832, 140141788114943, ++STORE, 140141754544128, 140141788114943, ++SNULL, 140141804904448, 140141821685759, ++STORE, 140141821685760, 140141830078463, ++STORE, 140141804904448, 140141821685759, ++SNULL, 140141821689855, 140141830078463, ++STORE, 140141821685760, 140141821689855, ++STORE, 140141821689856, 140141830078463, ++SNULL, 140141922336768, 140141939118079, ++STORE, 140141939118080, 140141947510783, ++STORE, 140141922336768, 140141939118079, ++SNULL, 140141939122175, 140141947510783, ++STORE, 140141939118080, 140141939122175, ++STORE, 140141939122176, 140141947510783, ++SNULL, 140141905551360, 140141913939967, ++STORE, 140141913939968, 140141922332671, ++STORE, 140141905551360, 140141913939967, ++SNULL, 140141913944063, 140141922332671, ++STORE, 140141913939968, 140141913944063, ++STORE, 140141913944064, 140141922332671, ++STORE, 140141746151424, 140141788114943, ++STORE, 140141737758720, 140141788114943, ++SNULL, 140141804904448, 140141813293055, ++STORE, 140141813293056, 140141821685759, ++STORE, 140141804904448, 140141813293055, ++SNULL, 140141813297151, 140141821685759, ++STORE, 140141813293056, 140141813297151, ++STORE, 140141813297152, 140141821685759, ++STORE, 140141729366016, 140141788114943, ++STORE, 140141720973312, 140141788114943, ++STORE, 140141712580608, 140141788114943, ++SNULL, 140141712584703, 140141788114943, ++STORE, 140141712580608, 140141712584703, ++STORE, 140141712584704, 140141788114943, ++SNULL, 140141922336768, 140141930725375, ++STORE, 140141930725376, 140141939118079, ++STORE, 140141922336768, 140141930725375, ++SNULL, 140141930729471, 140141939118079, ++STORE, 140141930725376, 140141930729471, ++STORE, 140141930729472, 140141939118079, ++STORE, 140141704187904, 140141712580607, ++SNULL, 140141704191999, 140141712580607, ++STORE, 140141704187904, 140141704191999, ++STORE, 140141704192000, 140141712580607, ++STORE, 140141695795200, 140141704187903, ++STORE, 140141687402496, 140141704187903, ++SNULL, 140141712584704, 140141771329535, ++STORE, 140141771329536, 140141788114943, ++STORE, 140141712584704, 140141771329535, ++SNULL, 140141771333631, 140141788114943, ++STORE, 140141771329536, 140141771333631, ++STORE, 140141771333632, 140141788114943, ++SNULL, 140141771333632, 140141779722239, ++STORE, 140141779722240, 140141788114943, ++STORE, 140141771333632, 140141779722239, ++SNULL, 140141779726335, 140141788114943, ++STORE, 140141779722240, 140141779726335, ++STORE, 140141779726336, 140141788114943, ++STORE, 140141679009792, 140141704187903, ++SNULL, 140141679013887, 140141704187903, ++STORE, 140141679009792, 140141679013887, ++STORE, 140141679013888, 140141704187903, ++STORE, 140141670617088, 140141679009791, ++SNULL, 140141670621183, 140141679009791, ++STORE, 140141670617088, 140141670621183, ++STORE, 140141670621184, 140141679009791, ++STORE, 140141662224384, 140141670617087, ++SNULL, 140141712584704, 140141737758719, ++STORE, 140141737758720, 140141771329535, ++STORE, 140141712584704, 140141737758719, ++SNULL, 140141737762815, 140141771329535, ++STORE, 140141737758720, 140141737762815, ++STORE, 140141737762816, 140141771329535, ++SNULL, 140141712584704, 140141729366015, ++STORE, 140141729366016, 140141737758719, ++STORE, 140141712584704, 140141729366015, ++SNULL, 140141729370111, 140141737758719, ++STORE, 140141729366016, 140141729370111, ++STORE, 140141729370112, 140141737758719, ++SNULL, 140141737762816, 140141746151423, ++STORE, 140141746151424, 140141771329535, ++STORE, 140141737762816, 140141746151423, ++SNULL, 140141746155519, 140141771329535, ++STORE, 140141746151424, 140141746155519, ++STORE, 140141746155520, 140141771329535, ++STORE, 140141653831680, 140141670617087, ++SNULL, 140141746155520, 140141762936831, ++STORE, 140141762936832, 140141771329535, ++STORE, 140141746155520, 140141762936831, ++SNULL, 140141762940927, 140141771329535, ++STORE, 140141762936832, 140141762940927, ++STORE, 140141762940928, 140141771329535, ++STORE, 140141645438976, 140141670617087, ++SNULL, 140141645443071, 140141670617087, ++STORE, 140141645438976, 140141645443071, ++STORE, 140141645443072, 140141670617087, ++SNULL, 140141712584704, 140141720973311, ++STORE, 140141720973312, 140141729366015, ++STORE, 140141712584704, 140141720973311, ++SNULL, 140141720977407, 140141729366015, ++STORE, 140141720973312, 140141720977407, ++STORE, 140141720977408, 140141729366015, ++STORE, 140141637046272, 140141645438975, ++SNULL, 140141637050367, 140141645438975, ++STORE, 140141637046272, 140141637050367, ++STORE, 140141637050368, 140141645438975, ++STORE, 140141628653568, 140141637046271, ++SNULL, 140141628657663, 140141637046271, ++STORE, 140141628653568, 140141628657663, ++STORE, 140141628657664, 140141637046271, ++STORE, 140141620260864, 140141628653567, ++SNULL, 140141679013888, 140141687402495, ++STORE, 140141687402496, 140141704187903, ++STORE, 140141679013888, 140141687402495, ++SNULL, 140141687406591, 140141704187903, ++STORE, 140141687402496, 140141687406591, ++STORE, 140141687406592, 140141704187903, ++SNULL, 140141746155520, 140141754544127, ++STORE, 140141754544128, 140141762936831, ++STORE, 140141746155520, 140141754544127, ++SNULL, 140141754548223, 140141762936831, ++STORE, 140141754544128, 140141754548223, ++STORE, 140141754548224, 140141762936831, ++SNULL, 140141687406592, 140141695795199, ++STORE, 140141695795200, 140141704187903, ++STORE, 140141687406592, 140141695795199, ++SNULL, 140141695799295, 140141704187903, ++STORE, 140141695795200, 140141695799295, ++STORE, 140141695799296, 140141704187903, ++STORE, 140141611868160, 140141628653567, ++SNULL, 140141611872255, 140141628653567, ++STORE, 140141611868160, 140141611872255, ++STORE, 140141611872256, 140141628653567, ++SNULL, 140141645443072, 140141662224383, ++STORE, 140141662224384, 140141670617087, ++STORE, 140141645443072, 140141662224383, ++SNULL, 140141662228479, 140141670617087, ++STORE, 140141662224384, 140141662228479, ++STORE, 140141662228480, 140141670617087, ++STORE, 140141603475456, 140141611868159, ++SNULL, 140141603479551, 140141611868159, ++STORE, 140141603475456, 140141603479551, ++STORE, 140141603479552, 140141611868159, ++STORE, 140141595082752, 140141603475455, ++SNULL, 140141645443072, 140141653831679, ++STORE, 140141653831680, 140141662224383, ++STORE, 140141645443072, 140141653831679, ++SNULL, 140141653835775, 140141662224383, ++STORE, 140141653831680, 140141653835775, ++STORE, 140141653835776, 140141662224383, ++STORE, 140141586690048, 140141603475455, ++SNULL, 140141611872256, 140141620260863, ++STORE, 140141620260864, 140141628653567, ++STORE, 140141611872256, 140141620260863, ++SNULL, 140141620264959, 140141628653567, ++STORE, 140141620260864, 140141620264959, ++STORE, 140141620264960, 140141628653567, ++SNULL, 140141586690048, 140141595082751, ++STORE, 140141595082752, 140141603475455, ++STORE, 140141586690048, 140141595082751, ++SNULL, 140141595086847, 140141603475455, ++STORE, 140141595082752, 140141595086847, ++STORE, 140141595086848, 140141603475455, ++STORE, 140141578297344, 140141595082751, ++SNULL, 140141578301439, 140141595082751, ++STORE, 140141578297344, 140141578301439, ++STORE, 140141578301440, 140141595082751, ++SNULL, 140141578301440, 140141586690047, ++STORE, 140141586690048, 140141595082751, ++STORE, 140141578301440, 140141586690047, ++SNULL, 140141586694143, 140141595082751, ++STORE, 140141586690048, 140141586694143, ++STORE, 140141586694144, 140141595082751, ++STORE, 140143370027008, 140143370055679, ++STORE, 140143309254656, 140143311446015, ++SNULL, 140143309254656, 140143309344767, ++STORE, 140143309344768, 140143311446015, ++STORE, 140143309254656, 140143309344767, ++SNULL, 140143311437823, 140143311446015, ++STORE, 140143309344768, 140143311437823, ++STORE, 140143311437824, 140143311446015, ++ERASE, 140143311437824, 140143311446015, ++STORE, 140143311437824, 140143311446015, ++SNULL, 140143311441919, 140143311446015, ++STORE, 140143311437824, 140143311441919, ++STORE, 140143311441920, 140143311446015, ++ERASE, 140143370027008, 140143370055679, ++ERASE, 140142912180224, 140142912184319, ++ERASE, 140142912184320, 140142920572927, ++ERASE, 140142945751040, 140142945755135, ++ERASE, 140142945755136, 140142954143743, ++ERASE, 140142090121216, 140142090125311, ++ERASE, 140142090125312, 140142098513919, ++ERASE, 140142794747904, 140142794751999, ++ERASE, 140142794752000, 140142803140607, ++ERASE, 140141913939968, 140141913944063, ++ERASE, 140141913944064, 140141922332671, ++ERASE, 140141746151424, 140141746155519, ++ERASE, 140141746155520, 140141754544127, ++ERASE, 140142954143744, 140142954147839, ++ERASE, 140142954147840, 140142962536447, ++ERASE, 140142081728512, 140142081732607, ++ERASE, 140142081732608, 140142090121215, ++ERASE, 140141905547264, 140141905551359, ++ERASE, 140141905551360, 140141913939967, ++ERASE, 140141729366016, 140141729370111, ++ERASE, 140141729370112, 140141737758719, ++ERASE, 140142920572928, 140142920577023, ++ERASE, 140142920577024, 140142928965631, ++ERASE, 140142039764992, 140142039769087, ++ERASE, 140142039769088, 140142048157695, ++ERASE, 140141679009792, 140141679013887, ++ERASE, 140141679013888, 140141687402495, ++ERASE, 140142551490560, 140142551494655, ++ERASE, 140142551494656, 140142559883263, ++ERASE, 140141947510784, 140141947514879, ++ERASE, 140141947514880, 140141955903487, ++ERASE, 140141771329536, 140141771333631, ++ERASE, 140141771333632, 140141779722239, ++ERASE, 140142928965632, 140142928969727, ++ERASE, 140142928969728, 140142937358335, ++ERASE, 140142073335808, 140142073339903, ++ERASE, 140142073339904, 140142081728511, ++ERASE, 140142543097856, 140142543101951, ++ERASE, 140142543101952, 140142551490559, ++ERASE, 140141955903488, 140141955907583, ++ERASE, 140141955907584, 140141964296191, ++ERASE, 140141704187904, 140141704191999, ++ERASE, 140141704192000, 140141712580607, ++ERASE, 140142786355200, 140142786359295, ++ERASE, 140142786359296, 140142794747903, ++ERASE, 140142056550400, 140142056554495, ++ERASE, 140142056554496, 140142064943103, ++ERASE, 140142828318720, 140142828322815, ++ERASE, 140142828322816, 140142836711423, ++ERASE, 140141788114944, 140141788119039, ++ERASE, 140141788119040, 140141796507647, ++ERASE, 140141695795200, 140141695799295, ++ERASE, 140141695799296, 140141704187903, ++ERASE, 140141578297344, 140141578301439, ++ERASE, 140141578301440, 140141586690047, ++ERASE, 140141611868160, 140141611872255, ++ERASE, 140141611872256, 140141620260863, ++ERASE, 140142811533312, 140142811537407, ++ERASE, 140142811537408, 140142819926015, ++ERASE, 140142064943104, 140142064947199, ++ERASE, 140142064947200, 140142073335807, ++ERASE, 140141628653568, 140141628657663, ++ERASE, 140141628657664, 140141637046271, ++ERASE, 140143046397952, 140143046402047, ++ERASE, 140143046402048, 140143054790655, ++ERASE, 140141796507648, 140141796511743, ++ERASE, 140141796511744, 140141804900351, ++ERASE, 140142803140608, 140142803144703, ++ERASE, 140142803144704, 140142811533311, ++ERASE, 140142509527040, 140142509531135, ++ERASE, 140142509531136, 140142517919743, ++ERASE, 140141821685760, 140141821689855, ++ERASE, 140141821689856, 140141830078463, ++ERASE, 140142777962496, 140142777966591, ++ERASE, 140142777966592, 140142786355199, ++ERASE, 140141804900352, 140141804904447, ++ERASE, 140141804904448, 140141813293055, ++ERASE, 140141930725376, 140141930729471, ++ERASE, 140141930729472, 140141939118079, ++ERASE, 140142937358336, 140142937362431, ++ERASE, 140142937362432, 140142945751039, ++ERASE, 140142559883264, 140142559887359, ++ERASE, 140142559887360, 140142568275967, ++ERASE, 140142534705152, 140142534709247, ++ERASE, 140142534709248, 140142543097855, ++ERASE, 140142048157696, 140142048161791, ++ERASE, 140142048161792, 140142056550399, ++ERASE, 140141754544128, 140141754548223, ++ERASE, 140141754548224, 140141762936831, ++ERASE, 140141939118080, 140141939122175, ++ERASE, 140141939122176, 140141947510783, ++ERASE, 140141653831680, 140141653835775, ++ERASE, 140141653835776, 140141662224383, ++ERASE, 140141712580608, 140141712584703, ++ERASE, 140141712584704, 140141720973311, ++ERASE, 140141645438976, 140141645443071, ++ERASE, 140141645443072, 140141653831679, ++ERASE, 140141687402496, 140141687406591, ++ERASE, 140141687406592, 140141695795199, ++ERASE, 140141662224384, 140141662228479, ++ERASE, 140141662228480, 140141670617087, ++ERASE, 140141922332672, 140141922336767, ++ERASE, 140141922336768, 140141930725375, ++ERASE, 140141737758720, 140141737762815, ++ERASE, 140141737762816, 140141746151423, ++ERASE, 140141637046272, 140141637050367, ++ERASE, 140141637050368, 140141645438975, ++ERASE, 140142517919744, 140142517923839, ++ERASE, 140142517923840, 140142526312447, ++ERASE, 140143096754176, 140143096758271, ++ERASE, 140143096758272, 140143105146879, ++ERASE, 140141595082752, 140141595086847, ++ERASE, 140141595086848, 140141603475455, ++ERASE, 140141762936832, 140141762940927, ++ERASE, 140141762940928, 140141771329535, ++ERASE, 140143311446016, 140143311450111, ++ERASE, 140143311450112, 140143319838719, ++ERASE, 140142526312448, 140142526316543, ++ERASE, 140142526316544, 140142534705151, ++ERASE, 140142819926016, 140142819930111, ++ERASE, 140142819930112, 140142828318719, ++ERASE, 140143180615680, 140143180619775, ++ERASE, 140143180619776, 140143189008383, ++ERASE, 140142962536448, 140142962540543, ++ERASE, 140142962540544, 140142970929151, ++ERASE, 140143214186496, 140143214190591, ++ERASE, 140143214190592, 140143222579199, ++ERASE, 140143088361472, 140143088365567, ++ERASE, 140143088365568, 140143096754175, ++ERASE, 140141586690048, 140141586694143, ++ERASE, 140141586694144, 140141595082751, ++ERASE, 140143230971904, 140143230975999, ++ERASE, 140143230976000, 140143239364607, ++ERASE, 140141779722240, 140141779726335, ++ERASE, 140141779726336, 140141788114943, ++ERASE, 140141670617088, 140141670621183, ++ERASE, 140141670621184, 140141679009791, ++ERASE, 140141813293056, 140141813297151, ++ERASE, 140141813297152, 140141821685759, ++ERASE, 140143222579200, 140143222583295, ++ERASE, 140143222583296, 140143230971903, ++ERASE, 140143189008384, 140143189012479, ++ERASE, 140143189012480, 140143197401087, ++ERASE, 140143071576064, 140143071580159, ++ERASE, 140143071580160, 140143079968767, ++ERASE, 140141620260864, 140141620264959, ++ERASE, 140141620264960, 140141628653567, ++ERASE, 140141603475456, 140141603479551, ++ERASE, 140141603479552, 140141611868159, ++ERASE, 140141720973312, 140141720977407, ++ERASE, 140141720977408, 140141729366015, ++ERASE, 140143079968768, 140143079972863, ++ERASE, 140143079972864, 140143088361471, ++ERASE, 140143205793792, 140143205797887, ++ERASE, 140143205797888, 140143214186495, ++ }; ++ unsigned long set30[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140733436743680, 140737488351231, ++SNULL, 140733436747775, 140737488351231, ++STORE, 140733436743680, 140733436747775, ++STORE, 140733436612608, 140733436747775, ++STORE, 94630728904704, 94630731157503, ++SNULL, 94630729035775, 94630731157503, ++STORE, 94630728904704, 94630729035775, ++STORE, 94630729035776, 94630731157503, ++ERASE, 94630729035776, 94630731157503, ++STORE, 94630731128832, 94630731137023, ++STORE, 94630731137024, 94630731157503, ++STORE, 140165750841344, 140165753094143, ++SNULL, 140165750984703, 140165753094143, ++STORE, 140165750841344, 140165750984703, ++STORE, 140165750984704, 140165753094143, ++ERASE, 140165750984704, 140165753094143, ++STORE, 140165753081856, 140165753090047, ++STORE, 140165753090048, 140165753094143, ++STORE, 140733436887040, 140733436891135, ++STORE, 140733436874752, 140733436887039, ++STORE, 140165753053184, 140165753081855, ++STORE, 140165753044992, 140165753053183, ++STORE, 140165748625408, 140165750841343, ++SNULL, 140165748625408, 140165748723711, ++STORE, 140165748723712, 140165750841343, ++STORE, 140165748625408, 140165748723711, ++SNULL, 140165750816767, 140165750841343, ++STORE, 140165748723712, 140165750816767, ++STORE, 140165750816768, 140165750841343, ++SNULL, 140165750816768, 140165750824959, ++STORE, 140165750824960, 140165750841343, ++STORE, 140165750816768, 140165750824959, ++ERASE, 140165750816768, 140165750824959, ++STORE, 140165750816768, 140165750824959, ++ERASE, 140165750824960, 140165750841343, ++STORE, 140165750824960, 140165750841343, ++STORE, 140165744828416, 140165748625407, ++SNULL, 140165744828416, 140165746487295, ++STORE, 140165746487296, 140165748625407, ++STORE, 140165744828416, 140165746487295, ++SNULL, 140165748584447, 140165748625407, ++STORE, 140165746487296, 140165748584447, ++STORE, 140165748584448, 140165748625407, ++SNULL, 140165748584448, 140165748609023, ++STORE, 140165748609024, 140165748625407, ++STORE, 140165748584448, 140165748609023, ++ERASE, 140165748584448, 140165748609023, ++STORE, 140165748584448, 140165748609023, ++ERASE, 140165748609024, 140165748625407, ++STORE, 140165748609024, 140165748625407, ++STORE, 140165753036800, 140165753053183, ++SNULL, 140165748600831, 140165748609023, ++STORE, 140165748584448, 140165748600831, ++STORE, 140165748600832, 140165748609023, ++SNULL, 140165750820863, 140165750824959, ++STORE, 140165750816768, 140165750820863, ++STORE, 140165750820864, 140165750824959, ++SNULL, 94630731132927, 94630731137023, ++STORE, 94630731128832, 94630731132927, ++STORE, 94630731132928, 94630731137023, ++SNULL, 140165753085951, 140165753090047, ++STORE, 140165753081856, 140165753085951, ++STORE, 140165753085952, 140165753090047, ++ERASE, 140165753053184, 140165753081855, ++STORE, 94630743547904, 94630743683071, ++STORE, 140165736435712, 140165744828415, ++SNULL, 140165736439807, 140165744828415, ++STORE, 140165736435712, 140165736439807, ++STORE, 140165736439808, 140165744828415, ++STORE, 140165728043008, 140165736435711, ++STORE, 140165593825280, 140165728043007, ++SNULL, 140165593825280, 140165653725183, ++STORE, 140165653725184, 140165728043007, ++STORE, 140165593825280, 140165653725183, ++ERASE, 140165593825280, 140165653725183, ++SNULL, 140165720834047, 140165728043007, ++STORE, 140165653725184, 140165720834047, ++STORE, 140165720834048, 140165728043007, ++ERASE, 140165720834048, 140165728043007, ++SNULL, 140165653860351, 140165720834047, ++STORE, 140165653725184, 140165653860351, ++STORE, 140165653860352, 140165720834047, ++SNULL, 140165728047103, 140165736435711, ++STORE, 140165728043008, 140165728047103, ++STORE, 140165728047104, 140165736435711, ++STORE, 140165645332480, 140165653725183, ++SNULL, 140165645336575, 140165653725183, ++STORE, 140165645332480, 140165645336575, ++STORE, 140165645336576, 140165653725183, ++STORE, 140165636939776, 140165645332479, ++SNULL, 140165636943871, 140165645332479, ++STORE, 140165636939776, 140165636943871, ++STORE, 140165636943872, 140165645332479, ++STORE, 140165628547072, 140165636939775, ++SNULL, 140165628551167, 140165636939775, ++STORE, 140165628547072, 140165628551167, ++STORE, 140165628551168, 140165636939775, ++STORE, 140165620154368, 140165628547071, ++STORE, 140165611761664, 140165628547071, ++STORE, 140165603368960, 140165628547071, ++STORE, 140165469151232, 140165603368959, ++SNULL, 140165469151232, 140165519507455, ++STORE, 140165519507456, 140165603368959, ++STORE, 140165469151232, 140165519507455, ++ERASE, 140165469151232, 140165519507455, ++SNULL, 140165586616319, 140165603368959, ++STORE, 140165519507456, 140165586616319, ++STORE, 140165586616320, 140165603368959, ++ERASE, 140165586616320, 140165603368959, ++STORE, 140165594976256, 140165628547071, ++STORE, 140165385289728, 140165586616319, ++SNULL, 140165452398591, 140165586616319, ++STORE, 140165385289728, 140165452398591, ++STORE, 140165452398592, 140165586616319, ++SNULL, 140165452398592, 140165519507455, ++STORE, 140165519507456, 140165586616319, ++STORE, 140165452398592, 140165519507455, ++ERASE, 140165452398592, 140165519507455, ++STORE, 140165251072000, 140165452398591, ++SNULL, 140165318180863, 140165452398591, ++STORE, 140165251072000, 140165318180863, ++STORE, 140165318180864, 140165452398591, ++SNULL, 140165318180864, 140165385289727, ++STORE, 140165385289728, 140165452398591, ++STORE, 140165318180864, 140165385289727, ++ERASE, 140165318180864, 140165385289727, ++SNULL, 140165519642623, 140165586616319, ++STORE, 140165519507456, 140165519642623, ++STORE, 140165519642624, 140165586616319, ++SNULL, 140165594976256, 140165611761663, ++STORE, 140165611761664, 140165628547071, ++STORE, 140165594976256, 140165611761663, ++SNULL, 140165611765759, 140165628547071, ++STORE, 140165611761664, 140165611765759, ++STORE, 140165611765760, 140165628547071, ++STORE, 140165385289728, 140165519507455, ++SNULL, 140165385424895, 140165519507455, ++STORE, 140165385289728, 140165385424895, ++STORE, 140165385424896, 140165519507455, ++SNULL, 140165594976256, 140165603368959, ++STORE, 140165603368960, 140165611761663, ++STORE, 140165594976256, 140165603368959, ++SNULL, 140165603373055, 140165611761663, ++STORE, 140165603368960, 140165603373055, ++STORE, 140165603373056, 140165611761663, ++SNULL, 140165251207167, 140165318180863, ++STORE, 140165251072000, 140165251207167, ++STORE, 140165251207168, 140165318180863, ++STORE, 140165376897024, 140165385289727, ++SNULL, 140165376901119, 140165385289727, ++STORE, 140165376897024, 140165376901119, ++STORE, 140165376901120, 140165385289727, ++SNULL, 140165385424896, 140165452398591, ++STORE, 140165452398592, 140165519507455, ++STORE, 140165385424896, 140165452398591, ++SNULL, 140165452533759, 140165519507455, ++STORE, 140165452398592, 140165452533759, ++STORE, 140165452533760, 140165519507455, ++STORE, 140165368504320, 140165376897023, ++SNULL, 140165594980351, 140165603368959, ++STORE, 140165594976256, 140165594980351, ++STORE, 140165594980352, 140165603368959, ++SNULL, 140165368508415, 140165376897023, ++STORE, 140165368504320, 140165368508415, ++STORE, 140165368508416, 140165376897023, ++SNULL, 140165611765760, 140165620154367, ++STORE, 140165620154368, 140165628547071, ++STORE, 140165611765760, 140165620154367, ++SNULL, 140165620158463, 140165628547071, ++STORE, 140165620154368, 140165620158463, ++STORE, 140165620158464, 140165628547071, ++STORE, 140165360111616, 140165368504319, ++STORE, 140165351718912, 140165368504319, ++STORE, 140165343326208, 140165368504319, ++SNULL, 140165343326208, 140165351718911, ++STORE, 140165351718912, 140165368504319, ++STORE, 140165343326208, 140165351718911, ++SNULL, 140165351723007, 140165368504319, ++STORE, 140165351718912, 140165351723007, ++STORE, 140165351723008, 140165368504319, ++SNULL, 140165343330303, 140165351718911, ++STORE, 140165343326208, 140165343330303, ++STORE, 140165343330304, 140165351718911, ++SNULL, 140165351723008, 140165360111615, ++STORE, 140165360111616, 140165368504319, ++STORE, 140165351723008, 140165360111615, ++SNULL, 140165360115711, 140165368504319, ++STORE, 140165360111616, 140165360115711, ++STORE, 140165360115712, 140165368504319, ++STORE, 140165334933504, 140165343326207, ++SNULL, 140165334937599, 140165343326207, ++STORE, 140165334933504, 140165334937599, ++STORE, 140165334937600, 140165343326207, ++STORE, 140165326540800, 140165334933503, ++STORE, 140165242679296, 140165251071999, ++SNULL, 140165242683391, 140165251071999, ++STORE, 140165242679296, 140165242683391, ++STORE, 140165242683392, 140165251071999, ++STORE, 140165234286592, 140165242679295, ++STORE, 140165225893888, 140165242679295, ++SNULL, 140165225897983, 140165242679295, ++STORE, 140165225893888, 140165225897983, ++STORE, 140165225897984, 140165242679295, ++SNULL, 140165225897984, 140165234286591, ++STORE, 140165234286592, 140165242679295, ++STORE, 140165225897984, 140165234286591, ++SNULL, 140165234290687, 140165242679295, ++STORE, 140165234286592, 140165234290687, ++STORE, 140165234290688, 140165242679295, ++SNULL, 140165326544895, 140165334933503, ++STORE, 140165326540800, 140165326544895, ++STORE, 140165326544896, 140165334933503, ++STORE, 140165217501184, 140165225893887, ++STORE, 140165209108480, 140165225893887, ++SNULL, 140165209108480, 140165217501183, ++STORE, 140165217501184, 140165225893887, ++STORE, 140165209108480, 140165217501183, ++SNULL, 140165217505279, 140165225893887, ++STORE, 140165217501184, 140165217505279, ++STORE, 140165217505280, 140165225893887, ++SNULL, 140165209112575, 140165217501183, ++STORE, 140165209108480, 140165209112575, ++STORE, 140165209112576, 140165217501183, ++STORE, 140165200715776, 140165209108479, ++STORE, 140165066498048, 140165200715775, ++SNULL, 140165066498048, 140165116854271, ++STORE, 140165116854272, 140165200715775, ++STORE, 140165066498048, 140165116854271, ++ERASE, 140165066498048, 140165116854271, ++SNULL, 140165183963135, 140165200715775, ++STORE, 140165116854272, 140165183963135, ++STORE, 140165183963136, 140165200715775, ++ERASE, 140165183963136, 140165200715775, ++SNULL, 140165116989439, 140165183963135, ++STORE, 140165116854272, 140165116989439, ++STORE, 140165116989440, 140165183963135, ++STORE, 140165192323072, 140165209108479, ++STORE, 140165108461568, 140165116854271, ++STORE, 140164974243840, 140165108461567, ++STORE, 140164965851136, 140164974243839, ++SNULL, 140164974243840, 140164982636543, ++STORE, 140164982636544, 140165108461567, ++STORE, 140164974243840, 140164982636543, ++ERASE, 140164974243840, 140164982636543, ++STORE, 140164965851136, 140164982636543, ++STORE, 140164957458432, 140164982636543, ++STORE, 140164949065728, 140164982636543, ++STORE, 140164940673024, 140164982636543, ++STORE, 140164806455296, 140164940673023, ++STORE, 140164798062592, 140164806455295, ++STORE, 140164789669888, 140164806455295, ++STORE, 140164655452160, 140164789669887, ++STORE, 140164647059456, 140164655452159, ++STORE, 140164638666752, 140164655452159, ++SNULL, 140164655452160, 140164714201087, ++STORE, 140164714201088, 140164789669887, ++STORE, 140164655452160, 140164714201087, ++ERASE, 140164655452160, 140164714201087, ++STORE, 140164705808384, 140164714201087, ++STORE, 140164697415680, 140164714201087, ++STORE, 140164504449024, 140164638666751, ++SNULL, 140164504449024, 140164512874495, ++STORE, 140164512874496, 140164638666751, ++STORE, 140164504449024, 140164512874495, ++ERASE, 140164504449024, 140164512874495, ++STORE, 140164689022976, 140164714201087, ++STORE, 140164680630272, 140164714201087, ++SNULL, 140164680634367, 140164714201087, ++STORE, 140164680630272, 140164680634367, ++STORE, 140164680634368, 140164714201087, ++STORE, 140164378656768, 140164638666751, ++SNULL, 140165192323072, 140165200715775, ++STORE, 140165200715776, 140165209108479, ++STORE, 140165192323072, 140165200715775, ++SNULL, 140165200719871, 140165209108479, ++STORE, 140165200715776, 140165200719871, ++STORE, 140165200719872, 140165209108479, ++SNULL, 140165049745407, 140165108461567, ++STORE, 140164982636544, 140165049745407, ++STORE, 140165049745408, 140165108461567, ++ERASE, 140165049745408, 140165108461567, ++SNULL, 140164982771711, 140165049745407, ++STORE, 140164982636544, 140164982771711, ++STORE, 140164982771712, 140165049745407, ++STORE, 140164244439040, 140164638666751, ++SNULL, 140164311547903, 140164638666751, ++STORE, 140164244439040, 140164311547903, ++STORE, 140164311547904, 140164638666751, ++SNULL, 140164311547904, 140164378656767, ++STORE, 140164378656768, 140164638666751, ++STORE, 140164311547904, 140164378656767, ++ERASE, 140164311547904, 140164378656767, ++SNULL, 140164806455296, 140164848418815, ++STORE, 140164848418816, 140164940673023, ++STORE, 140164806455296, 140164848418815, ++ERASE, 140164806455296, 140164848418815, ++SNULL, 140164915527679, 140164940673023, ++STORE, 140164848418816, 140164915527679, ++STORE, 140164915527680, 140164940673023, ++ERASE, 140164915527680, 140164940673023, ++STORE, 140164110221312, 140164311547903, ++SNULL, 140164177330175, 140164311547903, ++STORE, 140164110221312, 140164177330175, ++STORE, 140164177330176, 140164311547903, ++SNULL, 140164177330176, 140164244439039, ++STORE, 140164244439040, 140164311547903, ++STORE, 140164177330176, 140164244439039, ++ERASE, 140164177330176, 140164244439039, ++SNULL, 140164781309951, 140164789669887, ++STORE, 140164714201088, 140164781309951, ++STORE, 140164781309952, 140164789669887, ++ERASE, 140164781309952, 140164789669887, ++STORE, 140163976003584, 140164177330175, ++SNULL, 140164043112447, 140164177330175, ++STORE, 140163976003584, 140164043112447, ++STORE, 140164043112448, 140164177330175, ++SNULL, 140164043112448, 140164110221311, ++STORE, 140164110221312, 140164177330175, ++STORE, 140164043112448, 140164110221311, ++ERASE, 140164043112448, 140164110221311, ++SNULL, 140164579983359, 140164638666751, ++STORE, 140164378656768, 140164579983359, ++STORE, 140164579983360, 140164638666751, ++ERASE, 140164579983360, 140164638666751, ++STORE, 140163841785856, 140164043112447, ++SNULL, 140163908894719, 140164043112447, ++STORE, 140163841785856, 140163908894719, ++STORE, 140163908894720, 140164043112447, ++SNULL, 140163908894720, 140163976003583, ++STORE, 140163976003584, 140164043112447, ++STORE, 140163908894720, 140163976003583, ++ERASE, 140163908894720, 140163976003583, ++SNULL, 140164940673024, 140164965851135, ++STORE, 140164965851136, 140164982636543, ++STORE, 140164940673024, 140164965851135, ++SNULL, 140164965855231, 140164982636543, ++STORE, 140164965851136, 140164965855231, ++STORE, 140164965855232, 140164982636543, ++SNULL, 140164965855232, 140164974243839, ++STORE, 140164974243840, 140164982636543, ++STORE, 140164965855232, 140164974243839, ++SNULL, 140164974247935, 140164982636543, ++STORE, 140164974243840, 140164974247935, ++STORE, 140164974247936, 140164982636543, ++SNULL, 140164445765631, 140164579983359, ++STORE, 140164378656768, 140164445765631, ++STORE, 140164445765632, 140164579983359, ++SNULL, 140164445765632, 140164512874495, ++STORE, 140164512874496, 140164579983359, ++STORE, 140164445765632, 140164512874495, ++ERASE, 140164445765632, 140164512874495, ++SNULL, 140164378791935, 140164445765631, ++STORE, 140164378656768, 140164378791935, ++STORE, 140164378791936, 140164445765631, ++SNULL, 140164789673983, 140164806455295, ++STORE, 140164789669888, 140164789673983, ++STORE, 140164789673984, 140164806455295, ++SNULL, 140164789673984, 140164798062591, ++STORE, 140164798062592, 140164806455295, ++STORE, 140164789673984, 140164798062591, ++SNULL, 140164798066687, 140164806455295, ++STORE, 140164798062592, 140164798066687, ++STORE, 140164798066688, 140164806455295, ++SNULL, 140164638670847, 140164655452159, ++STORE, 140164638666752, 140164638670847, ++STORE, 140164638670848, 140164655452159, ++STORE, 140165100068864, 140165116854271, ++STORE, 140165091676160, 140165116854271, ++STORE, 140165083283456, 140165116854271, ++SNULL, 140164244574207, 140164311547903, ++STORE, 140164244439040, 140164244574207, ++STORE, 140164244574208, 140164311547903, ++SNULL, 140164848553983, 140164915527679, ++STORE, 140164848418816, 140164848553983, ++STORE, 140164848553984, 140164915527679, ++SNULL, 140164110356479, 140164177330175, ++STORE, 140164110221312, 140164110356479, ++STORE, 140164110356480, 140164177330175, ++SNULL, 140164714336255, 140164781309951, ++STORE, 140164714201088, 140164714336255, ++STORE, 140164714336256, 140164781309951, ++SNULL, 140163976138751, 140164043112447, ++STORE, 140163976003584, 140163976138751, ++STORE, 140163976138752, 140164043112447, ++SNULL, 140164513009663, 140164579983359, ++STORE, 140164512874496, 140164513009663, ++STORE, 140164513009664, 140164579983359, ++SNULL, 140163841921023, 140163908894719, ++STORE, 140163841785856, 140163841921023, ++STORE, 140163841921024, 140163908894719, ++SNULL, 140165083283456, 140165100068863, ++STORE, 140165100068864, 140165116854271, ++STORE, 140165083283456, 140165100068863, ++SNULL, 140165100072959, 140165116854271, ++STORE, 140165100068864, 140165100072959, ++STORE, 140165100072960, 140165116854271, ++SNULL, 140165100072960, 140165108461567, ++STORE, 140165108461568, 140165116854271, ++STORE, 140165100072960, 140165108461567, ++SNULL, 140165108465663, 140165116854271, ++STORE, 140165108461568, 140165108465663, ++STORE, 140165108465664, 140165116854271, ++STORE, 140165074890752, 140165100068863, ++SNULL, 140165074894847, 140165100068863, ++STORE, 140165074890752, 140165074894847, ++STORE, 140165074894848, 140165100068863, ++STORE, 140165066498048, 140165074890751, ++STORE, 140165058105344, 140165074890751, ++STORE, 140164932280320, 140164965851135, ++SNULL, 140165192327167, 140165200715775, ++STORE, 140165192323072, 140165192327167, ++STORE, 140165192327168, 140165200715775, ++STORE, 140164923887616, 140164965851135, ++SNULL, 140164923891711, 140164965851135, ++STORE, 140164923887616, 140164923891711, ++STORE, 140164923891712, 140164965851135, ++SNULL, 140164680634368, 140164705808383, ++STORE, 140164705808384, 140164714201087, ++STORE, 140164680634368, 140164705808383, ++SNULL, 140164705812479, 140164714201087, ++STORE, 140164705808384, 140164705812479, ++STORE, 140164705812480, 140164714201087, ++SNULL, 140164680634368, 140164697415679, ++STORE, 140164697415680, 140164705808383, ++STORE, 140164680634368, 140164697415679, ++SNULL, 140164697419775, 140164705808383, ++STORE, 140164697415680, 140164697419775, ++STORE, 140164697419776, 140164705808383, ++STORE, 140164840026112, 140164848418815, ++STORE, 140164831633408, 140164848418815, ++STORE, 140164823240704, 140164848418815, ++SNULL, 140165074894848, 140165083283455, ++STORE, 140165083283456, 140165100068863, ++STORE, 140165074894848, 140165083283455, ++SNULL, 140165083287551, 140165100068863, ++STORE, 140165083283456, 140165083287551, ++STORE, 140165083287552, 140165100068863, ++SNULL, 140165083287552, 140165091676159, ++STORE, 140165091676160, 140165100068863, ++STORE, 140165083287552, 140165091676159, ++SNULL, 140165091680255, 140165100068863, ++STORE, 140165091676160, 140165091680255, ++STORE, 140165091680256, 140165100068863, ++SNULL, 140164638670848, 140164647059455, ++STORE, 140164647059456, 140164655452159, ++STORE, 140164638670848, 140164647059455, ++SNULL, 140164647063551, 140164655452159, ++STORE, 140164647059456, 140164647063551, ++STORE, 140164647063552, 140164655452159, ++SNULL, 140164923891712, 140164940673023, ++STORE, 140164940673024, 140164965851135, ++STORE, 140164923891712, 140164940673023, ++SNULL, 140164940677119, 140164965851135, ++STORE, 140164940673024, 140164940677119, ++STORE, 140164940677120, 140164965851135, ++SNULL, 140164940677120, 140164949065727, ++STORE, 140164949065728, 140164965851135, ++STORE, 140164940677120, 140164949065727, ++SNULL, 140164949069823, 140164965851135, ++STORE, 140164949065728, 140164949069823, ++STORE, 140164949069824, 140164965851135, ++SNULL, 140164949069824, 140164957458431, ++STORE, 140164957458432, 140164965851135, ++STORE, 140164949069824, 140164957458431, ++SNULL, 140164957462527, 140164965851135, ++STORE, 140164957458432, 140164957462527, ++STORE, 140164957462528, 140164965851135, ++SNULL, 140164680634368, 140164689022975, ++STORE, 140164689022976, 140164697415679, ++STORE, 140164680634368, 140164689022975, ++SNULL, 140164689027071, 140164697415679, ++STORE, 140164689022976, 140164689027071, ++STORE, 140164689027072, 140164697415679, ++STORE, 140164814848000, 140164848418815, ++SNULL, 140165058105344, 140165066498047, ++STORE, 140165066498048, 140165074890751, ++STORE, 140165058105344, 140165066498047, ++SNULL, 140165066502143, 140165074890751, ++STORE, 140165066498048, 140165066502143, ++STORE, 140165066502144, 140165074890751, ++SNULL, 140165058109439, 140165066498047, ++STORE, 140165058105344, 140165058109439, ++STORE, 140165058109440, 140165066498047, ++STORE, 140164798066688, 140164814847999, ++SNULL, 140164798066688, 140164806455295, ++STORE, 140164806455296, 140164814847999, ++STORE, 140164798066688, 140164806455295, ++SNULL, 140164806459391, 140164814847999, ++STORE, 140164806455296, 140164806459391, ++STORE, 140164806459392, 140164814847999, ++SNULL, 140164923891712, 140164932280319, ++STORE, 140164932280320, 140164940673023, ++STORE, 140164923891712, 140164932280319, ++SNULL, 140164932284415, 140164940673023, ++STORE, 140164932280320, 140164932284415, ++STORE, 140164932284416, 140164940673023, ++STORE, 140164672237568, 140164680630271, ++STORE, 140164663844864, 140164680630271, ++STORE, 140164647063552, 140164680630271, ++SNULL, 140164647063552, 140164655452159, ++STORE, 140164655452160, 140164680630271, ++STORE, 140164647063552, 140164655452159, ++SNULL, 140164655456255, 140164680630271, ++STORE, 140164655452160, 140164655456255, ++STORE, 140164655456256, 140164680630271, ++STORE, 140164630274048, 140164638666751, ++SNULL, 140164814852095, 140164848418815, ++STORE, 140164814848000, 140164814852095, ++STORE, 140164814852096, 140164848418815, ++SNULL, 140164814852096, 140164831633407, ++STORE, 140164831633408, 140164848418815, ++STORE, 140164814852096, 140164831633407, ++SNULL, 140164831637503, 140164848418815, ++STORE, 140164831633408, 140164831637503, ++STORE, 140164831637504, 140164848418815, ++STORE, 140164621881344, 140164638666751, ++SNULL, 140164831637504, 140164840026111, ++STORE, 140164840026112, 140164848418815, ++STORE, 140164831637504, 140164840026111, ++SNULL, 140164840030207, 140164848418815, ++STORE, 140164840026112, 140164840030207, ++STORE, 140164840030208, 140164848418815, ++STORE, 140164613488640, 140164638666751, ++SNULL, 140164613492735, 140164638666751, ++STORE, 140164613488640, 140164613492735, ++STORE, 140164613492736, 140164638666751, ++STORE, 140164605095936, 140164613488639, ++SNULL, 140164605100031, 140164613488639, ++STORE, 140164605095936, 140164605100031, ++STORE, 140164605100032, 140164613488639, ++STORE, 140164596703232, 140164605095935, ++STORE, 140164588310528, 140164605095935, ++SNULL, 140164588314623, 140164605095935, ++STORE, 140164588310528, 140164588314623, ++STORE, 140164588314624, 140164605095935, ++STORE, 140164504481792, 140164512874495, ++STORE, 140164496089088, 140164512874495, ++SNULL, 140164496089088, 140164504481791, ++STORE, 140164504481792, 140164512874495, ++STORE, 140164496089088, 140164504481791, ++SNULL, 140164504485887, 140164512874495, ++STORE, 140164504481792, 140164504485887, ++STORE, 140164504485888, 140164512874495, ++SNULL, 140164613492736, 140164630274047, ++STORE, 140164630274048, 140164638666751, ++STORE, 140164613492736, 140164630274047, ++SNULL, 140164630278143, 140164638666751, ++STORE, 140164630274048, 140164630278143, ++STORE, 140164630278144, 140164638666751, ++STORE, 140164487696384, 140164504481791, ++STORE, 140164479303680, 140164504481791, ++SNULL, 140164814852096, 140164823240703, ++STORE, 140164823240704, 140164831633407, ++STORE, 140164814852096, 140164823240703, ++SNULL, 140164823244799, 140164831633407, ++STORE, 140164823240704, 140164823244799, ++STORE, 140164823244800, 140164831633407, ++STORE, 140164470910976, 140164504481791, ++SNULL, 140164470910976, 140164496089087, ++STORE, 140164496089088, 140164504481791, ++STORE, 140164470910976, 140164496089087, ++SNULL, 140164496093183, 140164504481791, ++STORE, 140164496089088, 140164496093183, ++STORE, 140164496093184, 140164504481791, ++SNULL, 140164655456256, 140164672237567, ++STORE, 140164672237568, 140164680630271, ++STORE, 140164655456256, 140164672237567, ++SNULL, 140164672241663, 140164680630271, ++STORE, 140164672237568, 140164672241663, ++STORE, 140164672241664, 140164680630271, ++STORE, 140164462518272, 140164496089087, ++STORE, 140164454125568, 140164496089087, ++SNULL, 140164655456256, 140164663844863, ++STORE, 140164663844864, 140164672237567, ++STORE, 140164655456256, 140164663844863, ++SNULL, 140164663848959, 140164672237567, ++STORE, 140164663844864, 140164663848959, ++STORE, 140164663848960, 140164672237567, ++STORE, 140164370264064, 140164378656767, ++STORE, 140164361871360, 140164378656767, ++STORE, 140164353478656, 140164378656767, ++STORE, 140164345085952, 140164378656767, ++SNULL, 140164345085952, 140164353478655, ++STORE, 140164353478656, 140164378656767, ++STORE, 140164345085952, 140164353478655, ++SNULL, 140164353482751, 140164378656767, ++STORE, 140164353478656, 140164353482751, ++STORE, 140164353482752, 140164378656767, ++SNULL, 140164454125568, 140164487696383, ++STORE, 140164487696384, 140164496089087, ++STORE, 140164454125568, 140164487696383, ++SNULL, 140164487700479, 140164496089087, ++STORE, 140164487696384, 140164487700479, ++STORE, 140164487700480, 140164496089087, ++STORE, 140164336693248, 140164353478655, ++SNULL, 140164336697343, 140164353478655, ++STORE, 140164336693248, 140164336697343, ++STORE, 140164336697344, 140164353478655, ++STORE, 140164328300544, 140164336693247, ++SNULL, 140164454125568, 140164479303679, ++STORE, 140164479303680, 140164487696383, ++STORE, 140164454125568, 140164479303679, ++SNULL, 140164479307775, 140164487696383, ++STORE, 140164479303680, 140164479307775, ++STORE, 140164479307776, 140164487696383, ++STORE, 140164319907840, 140164336693247, ++STORE, 140164236046336, 140164244439039, ++SNULL, 140164588314624, 140164596703231, ++STORE, 140164596703232, 140164605095935, ++STORE, 140164588314624, 140164596703231, ++SNULL, 140164596707327, 140164605095935, ++STORE, 140164596703232, 140164596707327, ++STORE, 140164596707328, 140164605095935, ++SNULL, 140164454125568, 140164462518271, ++STORE, 140164462518272, 140164479303679, ++STORE, 140164454125568, 140164462518271, ++SNULL, 140164462522367, 140164479303679, ++STORE, 140164462518272, 140164462522367, ++STORE, 140164462522368, 140164479303679, ++STORE, 140164227653632, 140164244439039, ++SNULL, 140164227657727, 140164244439039, ++STORE, 140164227653632, 140164227657727, ++STORE, 140164227657728, 140164244439039, ++SNULL, 140164462522368, 140164470910975, ++STORE, 140164470910976, 140164479303679, ++STORE, 140164462522368, 140164470910975, ++SNULL, 140164470915071, 140164479303679, ++STORE, 140164470910976, 140164470915071, ++STORE, 140164470915072, 140164479303679, ++SNULL, 140164613492736, 140164621881343, ++STORE, 140164621881344, 140164630274047, ++STORE, 140164613492736, 140164621881343, ++SNULL, 140164621885439, 140164630274047, ++STORE, 140164621881344, 140164621885439, ++STORE, 140164621885440, 140164630274047, ++SNULL, 140164353482752, 140164370264063, ++STORE, 140164370264064, 140164378656767, ++STORE, 140164353482752, 140164370264063, ++SNULL, 140164370268159, 140164378656767, ++STORE, 140164370264064, 140164370268159, ++STORE, 140164370268160, 140164378656767, ++STORE, 140164219260928, 140164227653631, ++SNULL, 140164319911935, 140164336693247, ++STORE, 140164319907840, 140164319911935, ++STORE, 140164319911936, 140164336693247, ++SNULL, 140164336697344, 140164345085951, ++STORE, 140164345085952, 140164353478655, ++STORE, 140164336697344, 140164345085951, ++SNULL, 140164345090047, 140164353478655, ++STORE, 140164345085952, 140164345090047, ++STORE, 140164345090048, 140164353478655, ++SNULL, 140164319911936, 140164328300543, ++STORE, 140164328300544, 140164336693247, ++STORE, 140164319911936, 140164328300543, ++SNULL, 140164328304639, 140164336693247, ++STORE, 140164328300544, 140164328304639, ++STORE, 140164328304640, 140164336693247, ++SNULL, 140164454129663, 140164462518271, ++STORE, 140164454125568, 140164454129663, ++STORE, 140164454129664, 140164462518271, ++STORE, 140164210868224, 140164227653631, ++STORE, 140164202475520, 140164227653631, ++STORE, 140164194082816, 140164227653631, ++SNULL, 140164194086911, 140164227653631, ++STORE, 140164194082816, 140164194086911, ++STORE, 140164194086912, 140164227653631, ++SNULL, 140164353482752, 140164361871359, ++STORE, 140164361871360, 140164370264063, ++STORE, 140164353482752, 140164361871359, ++SNULL, 140164361875455, 140164370264063, ++STORE, 140164361871360, 140164361875455, ++STORE, 140164361875456, 140164370264063, ++SNULL, 140164227657728, 140164236046335, ++STORE, 140164236046336, 140164244439039, ++STORE, 140164227657728, 140164236046335, ++SNULL, 140164236050431, 140164244439039, ++STORE, 140164236046336, 140164236050431, ++STORE, 140164236050432, 140164244439039, ++STORE, 140164185690112, 140164194082815, ++SNULL, 140164194086912, 140164219260927, ++STORE, 140164219260928, 140164227653631, ++STORE, 140164194086912, 140164219260927, ++SNULL, 140164219265023, 140164227653631, ++STORE, 140164219260928, 140164219265023, ++STORE, 140164219265024, 140164227653631, ++STORE, 140164101828608, 140164110221311, ++STORE, 140164093435904, 140164110221311, ++STORE, 140164085043200, 140164110221311, ++SNULL, 140164085047295, 140164110221311, ++STORE, 140164085043200, 140164085047295, ++STORE, 140164085047296, 140164110221311, ++STORE, 140164076650496, 140164085043199, ++SNULL, 140164185694207, 140164194082815, ++STORE, 140164185690112, 140164185694207, ++STORE, 140164185694208, 140164194082815, ++SNULL, 140164085047296, 140164101828607, ++STORE, 140164101828608, 140164110221311, ++STORE, 140164085047296, 140164101828607, ++SNULL, 140164101832703, 140164110221311, ++STORE, 140164101828608, 140164101832703, ++STORE, 140164101832704, 140164110221311, ++SNULL, 140164085047296, 140164093435903, ++STORE, 140164093435904, 140164101828607, ++STORE, 140164085047296, 140164093435903, ++SNULL, 140164093439999, 140164101828607, ++STORE, 140164093435904, 140164093439999, ++STORE, 140164093440000, 140164101828607, ++SNULL, 140164194086912, 140164202475519, ++STORE, 140164202475520, 140164219260927, ++STORE, 140164194086912, 140164202475519, ++SNULL, 140164202479615, 140164219260927, ++STORE, 140164202475520, 140164202479615, ++STORE, 140164202479616, 140164219260927, ++SNULL, 140164202479616, 140164210868223, ++STORE, 140164210868224, 140164219260927, ++STORE, 140164202479616, 140164210868223, ++SNULL, 140164210872319, 140164219260927, ++STORE, 140164210868224, 140164210872319, ++STORE, 140164210872320, 140164219260927, ++SNULL, 140164076654591, 140164085043199, ++STORE, 140164076650496, 140164076654591, ++STORE, 140164076654592, 140164085043199, ++STORE, 140164068257792, 140164076650495, ++SNULL, 140164068261887, 140164076650495, ++STORE, 140164068257792, 140164068261887, ++STORE, 140164068261888, 140164076650495, ++STORE, 140165753053184, 140165753081855, ++STORE, 140165725851648, 140165728043007, ++SNULL, 140165725851648, 140165725941759, ++STORE, 140165725941760, 140165728043007, ++STORE, 140165725851648, 140165725941759, ++SNULL, 140165728034815, 140165728043007, ++STORE, 140165725941760, 140165728034815, ++STORE, 140165728034816, 140165728043007, ++ERASE, 140165728034816, 140165728043007, ++STORE, 140165728034816, 140165728043007, ++SNULL, 140165728038911, 140165728043007, ++STORE, 140165728034816, 140165728038911, ++STORE, 140165728038912, 140165728043007, ++ERASE, 140165753053184, 140165753081855, ++ERASE, 140164638666752, 140164638670847, ++ERASE, 140164638670848, 140164647059455, ++ERASE, 140165091676160, 140165091680255, ++ERASE, 140165091680256, 140165100068863, ++ERASE, 140164613488640, 140164613492735, ++ERASE, 140164613492736, 140164621881343, ++ERASE, 140164319907840, 140164319911935, ++ERASE, 140164319911936, 140164328300543, ++ERASE, 140165620154368, 140165620158463, ++ERASE, 140165620158464, 140165628547071, ++ERASE, 140164798062592, 140164798066687, ++ERASE, 140164798066688, 140164806455295, ++ERASE, 140164789669888, 140164789673983, ++ERASE, 140164789673984, 140164798062591, ++ERASE, 140164965851136, 140164965855231, ++ERASE, 140164965855232, 140164974243839, ++ERASE, 140165074890752, 140165074894847, ++ERASE, 140165074894848, 140165083283455, ++ERASE, 140164672237568, 140164672241663, ++ERASE, 140164672241664, 140164680630271, ++ERASE, 140164454125568, 140164454129663, ++ERASE, 140164454129664, 140164462518271, ++ERASE, 140165200715776, 140165200719871, ++ERASE, 140165200719872, 140165209108479, ++ERASE, 140164932280320, 140164932284415, ++ERASE, 140164932284416, 140164940673023, ++ERASE, 140164663844864, 140164663848959, ++ERASE, 140164663848960, 140164672237567, ++ERASE, 140164697415680, 140164697419775, ++ERASE, 140164697419776, 140164705808383, ++ERASE, 140164831633408, 140164831637503, ++ERASE, 140164831637504, 140164840026111, ++ERASE, 140165192323072, 140165192327167, ++ERASE, 140165192327168, 140165200715775, ++ERASE, 140165108461568, 140165108465663, ++ERASE, 140165108465664, 140165116854271, ++ERASE, 140164840026112, 140164840030207, ++ERASE, 140164840030208, 140164848418815, ++ERASE, 140164647059456, 140164647063551, ++ERASE, 140164647063552, 140164655452159, ++ERASE, 140165083283456, 140165083287551, ++ERASE, 140165083287552, 140165091676159, ++ERASE, 140164923887616, 140164923891711, ++ERASE, 140164923891712, 140164932280319, ++ERASE, 140164823240704, 140164823244799, ++ERASE, 140164823244800, 140164831633407, ++ERASE, 140164227653632, 140164227657727, ++ERASE, 140164227657728, 140164236046335, ++ERASE, 140164957458432, 140164957462527, ++ERASE, 140164957462528, 140164965851135, ++ERASE, 140164680630272, 140164680634367, ++ERASE, 140164680634368, 140164689022975, ++ERASE, 140164974243840, 140164974247935, ++ERASE, 140164974247936, 140164982636543, ++ERASE, 140165066498048, 140165066502143, ++ERASE, 140165066502144, 140165074890751, ++ERASE, 140164621881344, 140164621885439, ++ERASE, 140164621885440, 140164630274047, ++ERASE, 140164949065728, 140164949069823, ++ERASE, 140164949069824, 140164957458431, ++ERASE, 140164588310528, 140164588314623, ++ERASE, 140164588314624, 140164596703231, ++ERASE, 140164806455296, 140164806459391, ++ERASE, 140164806459392, 140164814847999, ++ERASE, 140164940673024, 140164940677119, ++ERASE, 140164940677120, 140164949065727, ++ERASE, 140164596703232, 140164596707327, ++ERASE, 140164596707328, 140164605095935, ++ERASE, 140164605095936, 140164605100031, ++ERASE, 140164605100032, 140164613488639, ++ERASE, 140164655452160, 140164655456255, ++ERASE, 140164655456256, 140164663844863, ++ERASE, 140164705808384, 140164705812479, ++ERASE, 140164705812480, 140164714201087, ++ERASE, 140164689022976, 140164689027071, ++ERASE, 140164689027072, 140164697415679, ++ERASE, 140164630274048, 140164630278143, ++ERASE, 140164630278144, 140164638666751, ++ERASE, 140164479303680, 140164479307775, ++ERASE, 140164479307776, 140164487696383, ++ERASE, 140164236046336, 140164236050431, ++ERASE, 140164236050432, 140164244439039, ++ERASE, 140164085043200, 140164085047295, ++ERASE, 140164085047296, 140164093435903, ++ERASE, 140164345085952, 140164345090047, ++ERASE, 140164345090048, 140164353478655, ++ERASE, 140164101828608, 140164101832703, ++ERASE, 140164101832704, 140164110221311, ++ERASE, 140164370264064, 140164370268159, ++ERASE, 140164370268160, 140164378656767, ++ERASE, 140164336693248, 140164336697343, ++ERASE, 140164336697344, 140164345085951, ++ERASE, 140164194082816, 140164194086911, ++ERASE, 140164194086912, 140164202475519, ++ERASE, 140164353478656, 140164353482751, ++ERASE, 140164353482752, 140164361871359, ++ERASE, 140164210868224, 140164210872319, ++ERASE, 140164210872320, 140164219260927, ++ERASE, 140164814848000, 140164814852095, ++ERASE, 140164814852096, 140164823240703, ++ERASE, 140164504481792, 140164504485887, ++ERASE, 140164504485888, 140164512874495, ++ERASE, 140165100068864, 140165100072959, ++ERASE, 140165100072960, 140165108461567, ++ERASE, 140164361871360, 140164361875455, ++ERASE, 140164361875456, 140164370264063, ++ERASE, 140164470910976, 140164470915071, ++ERASE, 140164470915072, 140164479303679, ++ERASE, 140164076650496, 140164076654591, ++ERASE, 140164076654592, 140164085043199, ++ERASE, 140164202475520, 140164202479615, ++ERASE, 140164202479616, 140164210868223, ++ERASE, 140164462518272, 140164462522367, ++ERASE, 140164462522368, 140164470910975, ++ERASE, 140165351718912, 140165351723007, ++ERASE, 140165351723008, 140165360111615, ++ERASE, 140164328300544, 140164328304639, ++ERASE, 140164328304640, 140164336693247, ++ERASE, 140164093435904, 140164093439999, ++ERASE, 140164093440000, 140164101828607, ++ERASE, 140165603368960, 140165603373055, ++ERASE, 140165603373056, 140165611761663, ++ERASE, 140165368504320, 140165368508415, ++ERASE, 140165368508416, 140165376897023, ++ERASE, 140165334933504, 140165334937599, ++ERASE, 140165334937600, 140165343326207, ++ERASE, 140165594976256, 140165594980351, ++ERASE, 140165594980352, 140165603368959, ++ERASE, 140164487696384, 140164487700479, ++ERASE, 140164487700480, 140164496089087, ++ERASE, 140164219260928, 140164219265023, ++ERASE, 140164219265024, 140164227653631, ++ERASE, 140164185690112, 140164185694207, ++ERASE, 140164185694208, 140164194082815, ++ERASE, 140164068257792, 140164068261887, ++ERASE, 140164068261888, 140164076650495, ++ERASE, 140165225893888, 140165225897983, ++ERASE, 140165225897984, 140165234286591, ++ERASE, 140165058105344, 140165058109439, ++ }; ++ unsigned long set31[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140730890784768, 140737488351231, ++SNULL, 140730890788863, 140737488351231, ++STORE, 140730890784768, 140730890788863, ++STORE, 140730890653696, 140730890788863, ++STORE, 94577123659776, 94577125912575, ++SNULL, 94577123790847, 94577125912575, ++STORE, 94577123659776, 94577123790847, ++STORE, 94577123790848, 94577125912575, ++ERASE, 94577123790848, 94577125912575, ++STORE, 94577125883904, 94577125892095, ++STORE, 94577125892096, 94577125912575, ++STORE, 140624060407808, 140624062660607, ++SNULL, 140624060551167, 140624062660607, ++STORE, 140624060407808, 140624060551167, ++STORE, 140624060551168, 140624062660607, ++ERASE, 140624060551168, 140624062660607, ++STORE, 140624062648320, 140624062656511, ++STORE, 140624062656512, 140624062660607, ++STORE, 140730892140544, 140730892144639, ++STORE, 140730892128256, 140730892140543, ++STORE, 140624062619648, 140624062648319, ++STORE, 140624062611456, 140624062619647, ++STORE, 140624058191872, 140624060407807, ++SNULL, 140624058191872, 140624058290175, ++STORE, 140624058290176, 140624060407807, ++STORE, 140624058191872, 140624058290175, ++SNULL, 140624060383231, 140624060407807, ++STORE, 140624058290176, 140624060383231, ++STORE, 140624060383232, 140624060407807, ++SNULL, 140624060383232, 140624060391423, ++STORE, 140624060391424, 140624060407807, ++STORE, 140624060383232, 140624060391423, ++ERASE, 140624060383232, 140624060391423, ++STORE, 140624060383232, 140624060391423, ++ERASE, 140624060391424, 140624060407807, ++STORE, 140624060391424, 140624060407807, ++STORE, 140624054394880, 140624058191871, ++SNULL, 140624054394880, 140624056053759, ++STORE, 140624056053760, 140624058191871, ++STORE, 140624054394880, 140624056053759, ++SNULL, 140624058150911, 140624058191871, ++STORE, 140624056053760, 140624058150911, ++STORE, 140624058150912, 140624058191871, ++SNULL, 140624058150912, 140624058175487, ++STORE, 140624058175488, 140624058191871, ++STORE, 140624058150912, 140624058175487, ++ERASE, 140624058150912, 140624058175487, ++STORE, 140624058150912, 140624058175487, ++ERASE, 140624058175488, 140624058191871, ++STORE, 140624058175488, 140624058191871, ++STORE, 140624062603264, 140624062619647, ++SNULL, 140624058167295, 140624058175487, ++STORE, 140624058150912, 140624058167295, ++STORE, 140624058167296, 140624058175487, ++SNULL, 140624060387327, 140624060391423, ++STORE, 140624060383232, 140624060387327, ++STORE, 140624060387328, 140624060391423, ++SNULL, 94577125887999, 94577125892095, ++STORE, 94577125883904, 94577125887999, ++STORE, 94577125888000, 94577125892095, ++SNULL, 140624062652415, 140624062656511, ++STORE, 140624062648320, 140624062652415, ++STORE, 140624062652416, 140624062656511, ++ERASE, 140624062619648, 140624062648319, ++STORE, 94577157709824, 94577157844991, ++STORE, 140624046002176, 140624054394879, ++SNULL, 140624046006271, 140624054394879, ++STORE, 140624046002176, 140624046006271, ++STORE, 140624046006272, 140624054394879, ++STORE, 140624037609472, 140624046002175, ++STORE, 140623903391744, 140624037609471, ++SNULL, 140623903391744, 140623940157439, ++STORE, 140623940157440, 140624037609471, ++STORE, 140623903391744, 140623940157439, ++ERASE, 140623903391744, 140623940157439, ++SNULL, 140624007266303, 140624037609471, ++STORE, 140623940157440, 140624007266303, ++STORE, 140624007266304, 140624037609471, ++ERASE, 140624007266304, 140624037609471, ++SNULL, 140623940292607, 140624007266303, ++STORE, 140623940157440, 140623940292607, ++STORE, 140623940292608, 140624007266303, ++SNULL, 140624037613567, 140624046002175, ++STORE, 140624037609472, 140624037613567, ++STORE, 140624037613568, 140624046002175, ++STORE, 140624029216768, 140624037609471, ++SNULL, 140624029220863, 140624037609471, ++STORE, 140624029216768, 140624029220863, ++STORE, 140624029220864, 140624037609471, ++STORE, 140624020824064, 140624029216767, ++SNULL, 140624020828159, 140624029216767, ++STORE, 140624020824064, 140624020828159, ++STORE, 140624020828160, 140624029216767, ++STORE, 140624012431360, 140624020824063, ++SNULL, 140624012435455, 140624020824063, ++STORE, 140624012431360, 140624012435455, ++STORE, 140624012435456, 140624020824063, ++STORE, 140623931764736, 140623940157439, ++STORE, 140623797547008, 140623931764735, ++SNULL, 140623797547008, 140623805939711, ++STORE, 140623805939712, 140623931764735, ++STORE, 140623797547008, 140623805939711, ++ERASE, 140623797547008, 140623805939711, ++SNULL, 140623873048575, 140623931764735, ++STORE, 140623805939712, 140623873048575, ++STORE, 140623873048576, 140623931764735, ++ERASE, 140623873048576, 140623931764735, ++STORE, 140623923372032, 140623940157439, ++STORE, 140623914979328, 140623940157439, ++STORE, 140623906586624, 140623940157439, ++STORE, 140623671721984, 140623873048575, ++SNULL, 140623738830847, 140623873048575, ++STORE, 140623671721984, 140623738830847, ++STORE, 140623738830848, 140623873048575, ++SNULL, 140623738830848, 140623805939711, ++STORE, 140623805939712, 140623873048575, ++STORE, 140623738830848, 140623805939711, ++ERASE, 140623738830848, 140623805939711, ++SNULL, 140623806074879, 140623873048575, ++STORE, 140623805939712, 140623806074879, ++STORE, 140623806074880, 140623873048575, ++SNULL, 140623906586624, 140623931764735, ++STORE, 140623931764736, 140623940157439, ++STORE, 140623906586624, 140623931764735, ++SNULL, 140623931768831, 140623940157439, ++STORE, 140623931764736, 140623931768831, ++STORE, 140623931768832, 140623940157439, ++STORE, 140623537504256, 140623738830847, ++SNULL, 140623537504256, 140623671721983, ++STORE, 140623671721984, 140623738830847, ++STORE, 140623537504256, 140623671721983, ++SNULL, 140623671857151, 140623738830847, ++STORE, 140623671721984, 140623671857151, ++STORE, 140623671857152, 140623738830847, ++SNULL, 140623604613119, 140623671721983, ++STORE, 140623537504256, 140623604613119, ++STORE, 140623604613120, 140623671721983, ++ERASE, 140623604613120, 140623671721983, ++SNULL, 140623537639423, 140623604613119, ++STORE, 140623537504256, 140623537639423, ++STORE, 140623537639424, 140623604613119, ++STORE, 140623537639424, 140623671721983, ++SNULL, 140623537639424, 140623604613119, ++STORE, 140623604613120, 140623671721983, ++STORE, 140623537639424, 140623604613119, ++SNULL, 140623604748287, 140623671721983, ++STORE, 140623604613120, 140623604748287, ++STORE, 140623604748288, 140623671721983, ++STORE, 140623898193920, 140623931764735, ++SNULL, 140623898193920, 140623923372031, ++STORE, 140623923372032, 140623931764735, ++STORE, 140623898193920, 140623923372031, ++SNULL, 140623923376127, 140623931764735, ++STORE, 140623923372032, 140623923376127, ++STORE, 140623923376128, 140623931764735, ++STORE, 140623889801216, 140623923372031, ++SNULL, 140623889801216, 140623898193919, ++STORE, 140623898193920, 140623923372031, ++STORE, 140623889801216, 140623898193919, ++SNULL, 140623898198015, 140623923372031, ++STORE, 140623898193920, 140623898198015, ++STORE, 140623898198016, 140623923372031, ++SNULL, 140623889805311, 140623898193919, ++STORE, 140623889801216, 140623889805311, ++STORE, 140623889805312, 140623898193919, ++SNULL, 140623898198016, 140623906586623, ++STORE, 140623906586624, 140623923372031, ++STORE, 140623898198016, 140623906586623, ++SNULL, 140623906590719, 140623923372031, ++STORE, 140623906586624, 140623906590719, ++STORE, 140623906590720, 140623923372031, ++STORE, 140623881408512, 140623889801215, ++SNULL, 140623906590720, 140623914979327, ++STORE, 140623914979328, 140623923372031, ++STORE, 140623906590720, 140623914979327, ++SNULL, 140623914983423, 140623923372031, ++STORE, 140623914979328, 140623914983423, ++STORE, 140623914983424, 140623923372031, ++SNULL, 140623881412607, 140623889801215, ++STORE, 140623881408512, 140623881412607, ++STORE, 140623881412608, 140623889801215, ++STORE, 140623797547008, 140623805939711, ++STORE, 140623789154304, 140623805939711, ++STORE, 140623780761600, 140623805939711, ++SNULL, 140623780761600, 140623789154303, ++STORE, 140623789154304, 140623805939711, ++STORE, 140623780761600, 140623789154303, ++SNULL, 140623789158399, 140623805939711, ++STORE, 140623789154304, 140623789158399, ++STORE, 140623789158400, 140623805939711, ++STORE, 140623772368896, 140623789154303, ++STORE, 140623763976192, 140623789154303, ++SNULL, 140623763976192, 140623780761599, ++STORE, 140623780761600, 140623789154303, ++STORE, 140623763976192, 140623780761599, ++SNULL, 140623780765695, 140623789154303, ++STORE, 140623780761600, 140623780765695, ++STORE, 140623780765696, 140623789154303, ++SNULL, 140623789158400, 140623797547007, ++STORE, 140623797547008, 140623805939711, ++STORE, 140623789158400, 140623797547007, ++SNULL, 140623797551103, 140623805939711, ++STORE, 140623797547008, 140623797551103, ++STORE, 140623797551104, 140623805939711, ++SNULL, 140623763976192, 140623772368895, ++STORE, 140623772368896, 140623780761599, ++STORE, 140623763976192, 140623772368895, ++SNULL, 140623772372991, 140623780761599, ++STORE, 140623772368896, 140623772372991, ++STORE, 140623772372992, 140623780761599, ++SNULL, 140623763980287, 140623772368895, ++STORE, 140623763976192, 140623763980287, ++STORE, 140623763980288, 140623772368895, ++STORE, 140623755583488, 140623763976191, ++STORE, 140623747190784, 140623763976191, ++SNULL, 140623747190784, 140623755583487, ++STORE, 140623755583488, 140623763976191, ++STORE, 140623747190784, 140623755583487, ++SNULL, 140623755587583, 140623763976191, ++STORE, 140623755583488, 140623755587583, ++STORE, 140623755587584, 140623763976191, ++STORE, 140623529111552, 140623537504255, ++SNULL, 140623747194879, 140623755583487, ++STORE, 140623747190784, 140623747194879, ++STORE, 140623747194880, 140623755583487, ++SNULL, 140623529115647, 140623537504255, ++STORE, 140623529111552, 140623529115647, ++STORE, 140623529115648, 140623537504255, ++STORE, 140623520718848, 140623529111551, ++SNULL, 140623520722943, 140623529111551, ++STORE, 140623520718848, 140623520722943, ++STORE, 140623520722944, 140623529111551, ++STORE, 140623512326144, 140623520718847, ++STORE, 140623503933440, 140623520718847, ++STORE, 140623495540736, 140623520718847, ++STORE, 140623361323008, 140623495540735, ++STORE, 140623227105280, 140623495540735, ++STORE, 140623218712576, 140623227105279, ++STORE, 140623084494848, 140623218712575, ++STORE, 140623076102144, 140623084494847, ++STORE, 140622941884416, 140623076102143, ++SNULL, 140622941884416, 140623000633343, ++STORE, 140623000633344, 140623076102143, ++STORE, 140622941884416, 140623000633343, ++ERASE, 140622941884416, 140623000633343, ++STORE, 140622992240640, 140623000633343, ++STORE, 140622983847936, 140623000633343, ++STORE, 140622849630208, 140622983847935, ++STORE, 140622841237504, 140622849630207, ++SNULL, 140622849630208, 140622866415615, ++STORE, 140622866415616, 140622983847935, ++STORE, 140622849630208, 140622866415615, ++ERASE, 140622849630208, 140622866415615, ++STORE, 140622858022912, 140622866415615, ++SNULL, 140622933524479, 140622983847935, ++STORE, 140622866415616, 140622933524479, ++STORE, 140622933524480, 140622983847935, ++ERASE, 140622933524480, 140622983847935, ++STORE, 140622975455232, 140623000633343, ++STORE, 140622707019776, 140622841237503, ++STORE, 140622967062528, 140623000633343, ++STORE, 140622572802048, 140622841237503, ++STORE, 140622958669824, 140623000633343, ++STORE, 140622438584320, 140622841237503, ++STORE, 140622950277120, 140623000633343, ++SNULL, 140622858027007, 140622866415615, ++STORE, 140622858022912, 140622858027007, ++STORE, 140622858027008, 140622866415615, ++STORE, 140622941884416, 140623000633343, ++STORE, 140622841237504, 140622858022911, ++SNULL, 140622841237504, 140622849630207, ++STORE, 140622849630208, 140622858022911, ++STORE, 140622841237504, 140622849630207, ++SNULL, 140622849634303, 140622858022911, ++STORE, 140622849630208, 140622849634303, ++STORE, 140622849634304, 140622858022911, ++STORE, 140622430191616, 140622438584319, ++SNULL, 140622430195711, 140622438584319, ++STORE, 140622430191616, 140622430195711, ++STORE, 140622430195712, 140622438584319, ++SNULL, 140623361323007, 140623495540735, ++STORE, 140623227105280, 140623361323007, ++STORE, 140623361323008, 140623495540735, ++SNULL, 140623361323008, 140623403286527, ++STORE, 140623403286528, 140623495540735, ++STORE, 140623361323008, 140623403286527, ++ERASE, 140623361323008, 140623403286527, ++SNULL, 140623470395391, 140623495540735, ++STORE, 140623403286528, 140623470395391, ++STORE, 140623470395392, 140623495540735, ++ERASE, 140623470395392, 140623495540735, ++SNULL, 140623227105280, 140623269068799, ++STORE, 140623269068800, 140623361323007, ++STORE, 140623227105280, 140623269068799, ++ERASE, 140623227105280, 140623269068799, ++SNULL, 140623084494848, 140623134851071, ++STORE, 140623134851072, 140623218712575, ++STORE, 140623084494848, 140623134851071, ++ERASE, 140623084494848, 140623134851071, ++SNULL, 140623201959935, 140623218712575, ++STORE, 140623134851072, 140623201959935, ++STORE, 140623201959936, 140623218712575, ++ERASE, 140623201959936, 140623218712575, ++SNULL, 140623067742207, 140623076102143, ++STORE, 140623000633344, 140623067742207, ++STORE, 140623067742208, 140623076102143, ++ERASE, 140623067742208, 140623076102143, ++STORE, 140622295973888, 140622430191615, ++SNULL, 140622295973888, 140622329544703, ++STORE, 140622329544704, 140622430191615, ++STORE, 140622295973888, 140622329544703, ++ERASE, 140622295973888, 140622329544703, ++SNULL, 140622866550783, 140622933524479, ++STORE, 140622866415616, 140622866550783, ++STORE, 140622866550784, 140622933524479, ++SNULL, 140622707019775, 140622841237503, ++STORE, 140622438584320, 140622707019775, ++STORE, 140622707019776, 140622841237503, ++SNULL, 140622707019776, 140622732197887, ++STORE, 140622732197888, 140622841237503, ++STORE, 140622707019776, 140622732197887, ++ERASE, 140622707019776, 140622732197887, ++SNULL, 140622799306751, 140622841237503, ++STORE, 140622732197888, 140622799306751, ++STORE, 140622799306752, 140622841237503, ++ERASE, 140622799306752, 140622841237503, ++SNULL, 140622572802047, 140622707019775, ++STORE, 140622438584320, 140622572802047, ++STORE, 140622572802048, 140622707019775, ++SNULL, 140622572802048, 140622597980159, ++STORE, 140622597980160, 140622707019775, ++STORE, 140622572802048, 140622597980159, ++ERASE, 140622572802048, 140622597980159, ++SNULL, 140622438584320, 140622463762431, ++STORE, 140622463762432, 140622572802047, ++STORE, 140622438584320, 140622463762431, ++ERASE, 140622438584320, 140622463762431, ++SNULL, 140622530871295, 140622572802047, ++STORE, 140622463762432, 140622530871295, ++STORE, 140622530871296, 140622572802047, ++ERASE, 140622530871296, 140622572802047, ++STORE, 140622195326976, 140622430191615, ++SNULL, 140622262435839, 140622430191615, ++STORE, 140622195326976, 140622262435839, ++STORE, 140622262435840, 140622430191615, ++SNULL, 140622262435840, 140622329544703, ++STORE, 140622329544704, 140622430191615, ++STORE, 140622262435840, 140622329544703, ++ERASE, 140622262435840, 140622329544703, ++SNULL, 140622841241599, 140622849630207, ++STORE, 140622841237504, 140622841241599, ++STORE, 140622841241600, 140622849630207, ++STORE, 140623487148032, 140623520718847, ++STORE, 140623478755328, 140623520718847, ++SNULL, 140622941884416, 140622983847935, ++STORE, 140622983847936, 140623000633343, ++STORE, 140622941884416, 140622983847935, ++SNULL, 140622983852031, 140623000633343, ++STORE, 140622983847936, 140622983852031, ++STORE, 140622983852032, 140623000633343, ++STORE, 140623394893824, 140623403286527, ++SNULL, 140623394897919, 140623403286527, ++STORE, 140623394893824, 140623394897919, ++STORE, 140623394897920, 140623403286527, ++SNULL, 140623403421695, 140623470395391, ++STORE, 140623403286528, 140623403421695, ++STORE, 140623403421696, 140623470395391, ++SNULL, 140623478755328, 140623503933439, ++STORE, 140623503933440, 140623520718847, ++STORE, 140623478755328, 140623503933439, ++SNULL, 140623503937535, 140623520718847, ++STORE, 140623503933440, 140623503937535, ++STORE, 140623503937536, 140623520718847, ++SNULL, 140623336177663, 140623361323007, ++STORE, 140623269068800, 140623336177663, ++STORE, 140623336177664, 140623361323007, ++ERASE, 140623336177664, 140623361323007, ++SNULL, 140623269203967, 140623336177663, ++STORE, 140623269068800, 140623269203967, ++STORE, 140623269203968, 140623336177663, ++SNULL, 140623134986239, 140623201959935, ++STORE, 140623134851072, 140623134986239, ++STORE, 140623134986240, 140623201959935, ++SNULL, 140623000768511, 140623067742207, ++STORE, 140623000633344, 140623000768511, ++STORE, 140623000768512, 140623067742207, ++SNULL, 140622396653567, 140622430191615, ++STORE, 140622329544704, 140622396653567, ++STORE, 140622396653568, 140622430191615, ++ERASE, 140622396653568, 140622430191615, ++SNULL, 140622732333055, 140622799306751, ++STORE, 140622732197888, 140622732333055, ++STORE, 140622732333056, 140622799306751, ++SNULL, 140622941884416, 140622975455231, ++STORE, 140622975455232, 140622983847935, ++STORE, 140622941884416, 140622975455231, ++SNULL, 140622975459327, 140622983847935, ++STORE, 140622975455232, 140622975459327, ++STORE, 140622975459328, 140622983847935, ++SNULL, 140622665089023, 140622707019775, ++STORE, 140622597980160, 140622665089023, ++STORE, 140622665089024, 140622707019775, ++ERASE, 140622665089024, 140622707019775, ++SNULL, 140622598115327, 140622665089023, ++STORE, 140622597980160, 140622598115327, ++STORE, 140622598115328, 140622665089023, ++SNULL, 140622463897599, 140622530871295, ++STORE, 140622463762432, 140622463897599, ++STORE, 140622463897600, 140622530871295, ++SNULL, 140622195462143, 140622262435839, ++STORE, 140622195326976, 140622195462143, ++STORE, 140622195462144, 140622262435839, ++STORE, 140623386501120, 140623394893823, ++SNULL, 140622941884416, 140622950277119, ++STORE, 140622950277120, 140622975455231, ++STORE, 140622941884416, 140622950277119, ++SNULL, 140622950281215, 140622975455231, ++STORE, 140622950277120, 140622950281215, ++STORE, 140622950281216, 140622975455231, ++SNULL, 140622941888511, 140622950277119, ++STORE, 140622941884416, 140622941888511, ++STORE, 140622941888512, 140622950277119, ++STORE, 140623378108416, 140623394893823, ++SNULL, 140623478755328, 140623495540735, ++STORE, 140623495540736, 140623503933439, ++STORE, 140623478755328, 140623495540735, ++SNULL, 140623495544831, 140623503933439, ++STORE, 140623495540736, 140623495544831, ++STORE, 140623495544832, 140623503933439, ++SNULL, 140623478755328, 140623487148031, ++STORE, 140623487148032, 140623495540735, ++STORE, 140623478755328, 140623487148031, ++SNULL, 140623487152127, 140623495540735, ++STORE, 140623487148032, 140623487152127, ++STORE, 140623487152128, 140623495540735, ++SNULL, 140623218716671, 140623227105279, ++STORE, 140623218712576, 140623218716671, ++STORE, 140623218716672, 140623227105279, ++SNULL, 140623076106239, 140623084494847, ++STORE, 140623076102144, 140623076106239, ++STORE, 140623076106240, 140623084494847, ++SNULL, 140622329679871, 140622396653567, ++STORE, 140622329544704, 140622329679871, ++STORE, 140622329679872, 140622396653567, ++SNULL, 140622950281216, 140622958669823, ++STORE, 140622958669824, 140622975455231, ++STORE, 140622950281216, 140622958669823, ++SNULL, 140622958673919, 140622975455231, ++STORE, 140622958669824, 140622958673919, ++STORE, 140622958673920, 140622975455231, ++SNULL, 140623503937536, 140623512326143, ++STORE, 140623512326144, 140623520718847, ++STORE, 140623503937536, 140623512326143, ++SNULL, 140623512330239, 140623520718847, ++STORE, 140623512326144, 140623512330239, ++STORE, 140623512330240, 140623520718847, ++SNULL, 140623378108416, 140623386501119, ++STORE, 140623386501120, 140623394893823, ++STORE, 140623378108416, 140623386501119, ++SNULL, 140623386505215, 140623394893823, ++STORE, 140623386501120, 140623386505215, ++STORE, 140623386505216, 140623394893823, ++STORE, 140623369715712, 140623386501119, ++STORE, 140623361323008, 140623386501119, ++STORE, 140623352930304, 140623386501119, ++SNULL, 140623352930304, 140623361323007, ++STORE, 140623361323008, 140623386501119, ++STORE, 140623352930304, 140623361323007, ++SNULL, 140623361327103, 140623386501119, ++STORE, 140623361323008, 140623361327103, ++STORE, 140623361327104, 140623386501119, ++SNULL, 140623478759423, 140623487148031, ++STORE, 140623478755328, 140623478759423, ++STORE, 140623478759424, 140623487148031, ++STORE, 140623344537600, 140623361323007, ++STORE, 140623260676096, 140623269068799, ++SNULL, 140622958673920, 140622967062527, ++STORE, 140622967062528, 140622975455231, ++STORE, 140622958673920, 140622967062527, ++SNULL, 140622967066623, 140622975455231, ++STORE, 140622967062528, 140622967066623, ++STORE, 140622967066624, 140622975455231, ++STORE, 140623252283392, 140623269068799, ++STORE, 140623243890688, 140623269068799, ++SNULL, 140622983852032, 140622992240639, ++STORE, 140622992240640, 140623000633343, ++STORE, 140622983852032, 140622992240639, ++SNULL, 140622992244735, 140623000633343, ++STORE, 140622992240640, 140622992244735, ++STORE, 140622992244736, 140623000633343, ++STORE, 140623235497984, 140623269068799, ++STORE, 140623218716672, 140623235497983, ++STORE, 140623210319872, 140623218712575, ++STORE, 140623126458368, 140623134851071, ++SNULL, 140623210323967, 140623218712575, ++STORE, 140623210319872, 140623210323967, ++STORE, 140623210323968, 140623218712575, ++SNULL, 140623218716672, 140623227105279, ++STORE, 140623227105280, 140623235497983, ++STORE, 140623218716672, 140623227105279, ++SNULL, 140623227109375, 140623235497983, ++STORE, 140623227105280, 140623227109375, ++STORE, 140623227109376, 140623235497983, ++STORE, 140623118065664, 140623134851071, ++STORE, 140623109672960, 140623134851071, ++SNULL, 140623109677055, 140623134851071, ++STORE, 140623109672960, 140623109677055, ++STORE, 140623109677056, 140623134851071, ++STORE, 140623101280256, 140623109672959, ++STORE, 140623092887552, 140623109672959, ++SNULL, 140623092887552, 140623101280255, ++STORE, 140623101280256, 140623109672959, ++STORE, 140623092887552, 140623101280255, ++SNULL, 140623101284351, 140623109672959, ++STORE, 140623101280256, 140623101284351, ++STORE, 140623101284352, 140623109672959, ++SNULL, 140623361327104, 140623378108415, ++STORE, 140623378108416, 140623386501119, ++STORE, 140623361327104, 140623378108415, ++SNULL, 140623378112511, 140623386501119, ++STORE, 140623378108416, 140623378112511, ++STORE, 140623378112512, 140623386501119, ++SNULL, 140623235497984, 140623243890687, ++STORE, 140623243890688, 140623269068799, ++STORE, 140623235497984, 140623243890687, ++SNULL, 140623243894783, 140623269068799, ++STORE, 140623243890688, 140623243894783, ++STORE, 140623243894784, 140623269068799, ++SNULL, 140623361327104, 140623369715711, ++STORE, 140623369715712, 140623378108415, ++STORE, 140623361327104, 140623369715711, ++SNULL, 140623369719807, 140623378108415, ++STORE, 140623369715712, 140623369719807, ++STORE, 140623369719808, 140623378108415, ++SNULL, 140623243894784, 140623252283391, ++STORE, 140623252283392, 140623269068799, ++STORE, 140623243894784, 140623252283391, ++SNULL, 140623252287487, 140623269068799, ++STORE, 140623252283392, 140623252287487, ++STORE, 140623252287488, 140623269068799, ++SNULL, 140623235502079, 140623243890687, ++STORE, 140623235497984, 140623235502079, ++STORE, 140623235502080, 140623243890687, ++SNULL, 140623344541695, 140623361323007, ++STORE, 140623344537600, 140623344541695, ++STORE, 140623344541696, 140623361323007, ++STORE, 140623076106240, 140623092887551, ++SNULL, 140623076106240, 140623084494847, ++STORE, 140623084494848, 140623092887551, ++STORE, 140623076106240, 140623084494847, ++SNULL, 140623084498943, 140623092887551, ++STORE, 140623084494848, 140623084498943, ++STORE, 140623084498944, 140623092887551, ++SNULL, 140623344541696, 140623352930303, ++STORE, 140623352930304, 140623361323007, ++STORE, 140623344541696, 140623352930303, ++SNULL, 140623352934399, 140623361323007, ++STORE, 140623352930304, 140623352934399, ++STORE, 140623352934400, 140623361323007, ++SNULL, 140623109677056, 140623118065663, ++STORE, 140623118065664, 140623134851071, ++STORE, 140623109677056, 140623118065663, ++SNULL, 140623118069759, 140623134851071, ++STORE, 140623118065664, 140623118069759, ++STORE, 140623118069760, 140623134851071, ++STORE, 140622832844800, 140622841237503, ++STORE, 140622824452096, 140622841237503, ++SNULL, 140622824452096, 140622832844799, ++STORE, 140622832844800, 140622841237503, ++STORE, 140622824452096, 140622832844799, ++SNULL, 140622832848895, 140622841237503, ++STORE, 140622832844800, 140622832848895, ++STORE, 140622832848896, 140622841237503, ++STORE, 140622816059392, 140622832844799, ++SNULL, 140623092891647, 140623101280255, ++STORE, 140623092887552, 140623092891647, ++STORE, 140623092891648, 140623101280255, ++SNULL, 140623118069760, 140623126458367, ++STORE, 140623126458368, 140623134851071, ++STORE, 140623118069760, 140623126458367, ++SNULL, 140623126462463, 140623134851071, ++STORE, 140623126458368, 140623126462463, ++STORE, 140623126462464, 140623134851071, ++SNULL, 140623252287488, 140623260676095, ++STORE, 140623260676096, 140623269068799, ++STORE, 140623252287488, 140623260676095, ++SNULL, 140623260680191, 140623269068799, ++STORE, 140623260676096, 140623260680191, ++STORE, 140623260680192, 140623269068799, ++STORE, 140622807666688, 140622832844799, ++STORE, 140622723805184, 140622732197887, ++STORE, 140622715412480, 140622732197887, ++STORE, 140622707019776, 140622732197887, ++SNULL, 140622707023871, 140622732197887, ++STORE, 140622707019776, 140622707023871, ++STORE, 140622707023872, 140622732197887, ++STORE, 140622698627072, 140622707019775, ++STORE, 140622690234368, 140622707019775, ++SNULL, 140622690238463, 140622707019775, ++STORE, 140622690234368, 140622690238463, ++STORE, 140622690238464, 140622707019775, ++SNULL, 140622807666688, 140622816059391, ++STORE, 140622816059392, 140622832844799, ++STORE, 140622807666688, 140622816059391, ++SNULL, 140622816063487, 140622832844799, ++STORE, 140622816059392, 140622816063487, ++STORE, 140622816063488, 140622832844799, ++STORE, 140622681841664, 140622690234367, ++STORE, 140622673448960, 140622690234367, ++SNULL, 140622673453055, 140622690234367, ++STORE, 140622673448960, 140622673453055, ++STORE, 140622673453056, 140622690234367, ++STORE, 140622589587456, 140622597980159, ++SNULL, 140622807670783, 140622816059391, ++STORE, 140622807666688, 140622807670783, ++STORE, 140622807670784, 140622816059391, ++STORE, 140622581194752, 140622597980159, ++SNULL, 140622581198847, 140622597980159, ++STORE, 140622581194752, 140622581198847, ++STORE, 140622581198848, 140622597980159, ++SNULL, 140622816063488, 140622824452095, ++STORE, 140622824452096, 140622832844799, ++STORE, 140622816063488, 140622824452095, ++SNULL, 140622824456191, 140622832844799, ++STORE, 140622824452096, 140622824456191, ++STORE, 140622824456192, 140622832844799, ++STORE, 140622572802048, 140622581194751, ++SNULL, 140622572806143, 140622581194751, ++STORE, 140622572802048, 140622572806143, ++STORE, 140622572806144, 140622581194751, ++STORE, 140622564409344, 140622572802047, ++STORE, 140622556016640, 140622572802047, ++SNULL, 140622556016640, 140622564409343, ++STORE, 140622564409344, 140622572802047, ++STORE, 140622556016640, 140622564409343, ++SNULL, 140622564413439, 140622572802047, ++STORE, 140622564409344, 140622564413439, ++STORE, 140622564413440, 140622572802047, ++SNULL, 140622690238464, 140622698627071, ++STORE, 140622698627072, 140622707019775, ++STORE, 140622690238464, 140622698627071, ++SNULL, 140622698631167, 140622707019775, ++STORE, 140622698627072, 140622698631167, ++STORE, 140622698631168, 140622707019775, ++SNULL, 140622707023872, 140622723805183, ++STORE, 140622723805184, 140622732197887, ++STORE, 140622707023872, 140622723805183, ++SNULL, 140622723809279, 140622732197887, ++STORE, 140622723805184, 140622723809279, ++STORE, 140622723809280, 140622732197887, ++SNULL, 140622707023872, 140622715412479, ++STORE, 140622715412480, 140622723805183, ++STORE, 140622707023872, 140622715412479, ++SNULL, 140622715416575, 140622723805183, ++STORE, 140622715412480, 140622715416575, ++STORE, 140622715416576, 140622723805183, ++STORE, 140622547623936, 140622564409343, ++SNULL, 140622547628031, 140622564409343, ++STORE, 140622547623936, 140622547628031, ++STORE, 140622547628032, 140622564409343, ++STORE, 140622539231232, 140622547623935, ++SNULL, 140622539235327, 140622547623935, ++STORE, 140622539231232, 140622539235327, ++STORE, 140622539235328, 140622547623935, ++SNULL, 140622581198848, 140622589587455, ++STORE, 140622589587456, 140622597980159, ++STORE, 140622581198848, 140622589587455, ++SNULL, 140622589591551, 140622597980159, ++STORE, 140622589587456, 140622589591551, ++STORE, 140622589591552, 140622597980159, ++STORE, 140622455369728, 140622463762431, ++SNULL, 140622455373823, 140622463762431, ++STORE, 140622455369728, 140622455373823, ++STORE, 140622455373824, 140622463762431, ++STORE, 140622446977024, 140622455369727, ++SNULL, 140622446981119, 140622455369727, ++STORE, 140622446977024, 140622446981119, ++STORE, 140622446981120, 140622455369727, ++SNULL, 140622547628032, 140622556016639, ++STORE, 140622556016640, 140622564409343, ++STORE, 140622547628032, 140622556016639, ++SNULL, 140622556020735, 140622564409343, ++STORE, 140622556016640, 140622556020735, ++STORE, 140622556020736, 140622564409343, ++STORE, 140622430195712, 140622446977023, ++STORE, 140622421798912, 140622430191615, ++SNULL, 140622430195712, 140622438584319, ++STORE, 140622438584320, 140622446977023, ++STORE, 140622430195712, 140622438584319, ++SNULL, 140622438588415, 140622446977023, ++STORE, 140622438584320, 140622438588415, ++STORE, 140622438588416, 140622446977023, ++STORE, 140622413406208, 140622430191615, ++STORE, 140622405013504, 140622430191615, ++SNULL, 140622405013504, 140622413406207, ++STORE, 140622413406208, 140622430191615, ++STORE, 140622405013504, 140622413406207, ++SNULL, 140622413410303, 140622430191615, ++STORE, 140622413406208, 140622413410303, ++STORE, 140622413410304, 140622430191615, ++SNULL, 140622673453056, 140622681841663, ++STORE, 140622681841664, 140622690234367, ++STORE, 140622673453056, 140622681841663, ++SNULL, 140622681845759, 140622690234367, ++STORE, 140622681841664, 140622681845759, ++STORE, 140622681845760, 140622690234367, ++STORE, 140622321152000, 140622329544703, ++SNULL, 140622413410304, 140622421798911, ++STORE, 140622421798912, 140622430191615, ++STORE, 140622413410304, 140622421798911, ++SNULL, 140622421803007, 140622430191615, ++STORE, 140622421798912, 140622421803007, ++STORE, 140622421803008, 140622430191615, ++STORE, 140622312759296, 140622329544703, ++SNULL, 140622312763391, 140622329544703, ++STORE, 140622312759296, 140622312763391, ++STORE, 140622312763392, 140622329544703, ++SNULL, 140622405017599, 140622413406207, ++STORE, 140622405013504, 140622405017599, ++STORE, 140622405017600, 140622413406207, ++STORE, 140622304366592, 140622312759295, ++SNULL, 140622304370687, 140622312759295, ++STORE, 140622304366592, 140622304370687, ++STORE, 140622304370688, 140622312759295, ++SNULL, 140622312763392, 140622321151999, ++STORE, 140622321152000, 140622329544703, ++STORE, 140622312763392, 140622321151999, ++SNULL, 140622321156095, 140622329544703, ++STORE, 140622321152000, 140622321156095, ++STORE, 140622321156096, 140622329544703, ++STORE, 140624062619648, 140624062648319, ++STORE, 140624010240000, 140624012431359, ++SNULL, 140624010240000, 140624010330111, ++STORE, 140624010330112, 140624012431359, ++STORE, 140624010240000, 140624010330111, ++SNULL, 140624012423167, 140624012431359, ++STORE, 140624010330112, 140624012423167, ++STORE, 140624012423168, 140624012431359, ++ERASE, 140624012423168, 140624012431359, ++STORE, 140624012423168, 140624012431359, ++SNULL, 140624012427263, 140624012431359, ++STORE, 140624012423168, 140624012427263, ++STORE, 140624012427264, 140624012431359, ++ERASE, 140624062619648, 140624062648319, ++ERASE, 140622849630208, 140622849634303, ++ERASE, 140622849634304, 140622858022911, ++ERASE, 140623394893824, 140623394897919, ++ERASE, 140623394897920, 140623403286527, ++ERASE, 140623361323008, 140623361327103, ++ERASE, 140623361327104, 140623369715711, ++ERASE, 140623084494848, 140623084498943, ++ERASE, 140623084498944, 140623092887551, ++ERASE, 140623931764736, 140623931768831, ++ERASE, 140623931768832, 140623940157439, ++ERASE, 140622841237504, 140622841241599, ++ERASE, 140622841241600, 140622849630207, ++ERASE, 140623487148032, 140623487152127, ++ERASE, 140623487152128, 140623495540735, ++ERASE, 140623109672960, 140623109677055, ++ERASE, 140623109677056, 140623118065663, ++ERASE, 140622983847936, 140622983852031, ++ERASE, 140622983852032, 140622992240639, ++ERASE, 140623352930304, 140623352934399, ++ERASE, 140623352934400, 140623361323007, ++ERASE, 140622564409344, 140622564413439, ++ERASE, 140622564413440, 140622572802047, ++ERASE, 140622430191616, 140622430195711, ++ERASE, 140622430195712, 140622438584319, ++ERASE, 140622958669824, 140622958673919, ++ERASE, 140622958673920, 140622967062527, ++ERASE, 140622992240640, 140622992244735, ++ERASE, 140622992244736, 140623000633343, ++ERASE, 140623227105280, 140623227109375, ++ERASE, 140623227109376, 140623235497983, ++ERASE, 140622321152000, 140622321156095, ++ERASE, 140622321156096, 140622329544703, ++ERASE, 140622858022912, 140622858027007, ++ERASE, 140622858027008, 140622866415615, ++ERASE, 140622975455232, 140622975459327, ++ERASE, 140622975459328, 140622983847935, ++ERASE, 140623378108416, 140623378112511, ++ERASE, 140623378112512, 140623386501119, ++ERASE, 140623495540736, 140623495544831, ++ERASE, 140623495544832, 140623503933439, ++ERASE, 140623118065664, 140623118069759, ++ERASE, 140623118069760, 140623126458367, ++ERASE, 140622572802048, 140622572806143, ++ERASE, 140622572806144, 140622581194751, ++ERASE, 140622421798912, 140622421803007, ++ERASE, 140622421803008, 140622430191615, ++ERASE, 140622967062528, 140622967066623, ++ERASE, 140622967066624, 140622975455231, ++ERASE, 140623252283392, 140623252287487, ++ERASE, 140623252287488, 140623260676095, ++ERASE, 140622673448960, 140622673453055, ++ERASE, 140622673453056, 140622681841663, ++ERASE, 140623076102144, 140623076106239, ++ERASE, 140623076106240, 140623084494847, ++ERASE, 140623101280256, 140623101284351, ++ERASE, 140623101284352, 140623109672959, ++ERASE, 140622715412480, 140622715416575, ++ERASE, 140622715416576, 140622723805183, ++ERASE, 140622405013504, 140622405017599, ++ERASE, 140622405017600, 140622413406207, ++ERASE, 140623478755328, 140623478759423, ++ERASE, 140623478759424, 140623487148031, ++ERASE, 140623906586624, 140623906590719, ++ERASE, 140623906590720, 140623914979327, ++ERASE, 140622950277120, 140622950281215, ++ERASE, 140622950281216, 140622958669823, ++ }; ++ unsigned long set32[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140731244212224, 140737488351231, ++SNULL, 140731244216319, 140737488351231, ++STORE, 140731244212224, 140731244216319, ++STORE, 140731244081152, 140731244216319, ++STORE, 94427773984768, 94427776237567, ++SNULL, 94427774115839, 94427776237567, ++STORE, 94427773984768, 94427774115839, ++STORE, 94427774115840, 94427776237567, ++ERASE, 94427774115840, 94427776237567, ++STORE, 94427776208896, 94427776217087, ++STORE, 94427776217088, 94427776237567, ++STORE, 140401464893440, 140401467146239, ++SNULL, 140401465036799, 140401467146239, ++STORE, 140401464893440, 140401465036799, ++STORE, 140401465036800, 140401467146239, ++ERASE, 140401465036800, 140401467146239, ++STORE, 140401467133952, 140401467142143, ++STORE, 140401467142144, 140401467146239, ++STORE, 140731244507136, 140731244511231, ++STORE, 140731244494848, 140731244507135, ++STORE, 140401467105280, 140401467133951, ++STORE, 140401467097088, 140401467105279, ++STORE, 140401462677504, 140401464893439, ++SNULL, 140401462677504, 140401462775807, ++STORE, 140401462775808, 140401464893439, ++STORE, 140401462677504, 140401462775807, ++SNULL, 140401464868863, 140401464893439, ++STORE, 140401462775808, 140401464868863, ++STORE, 140401464868864, 140401464893439, ++SNULL, 140401464868864, 140401464877055, ++STORE, 140401464877056, 140401464893439, ++STORE, 140401464868864, 140401464877055, ++ERASE, 140401464868864, 140401464877055, ++STORE, 140401464868864, 140401464877055, ++ERASE, 140401464877056, 140401464893439, ++STORE, 140401464877056, 140401464893439, ++STORE, 140401458880512, 140401462677503, ++SNULL, 140401458880512, 140401460539391, ++STORE, 140401460539392, 140401462677503, ++STORE, 140401458880512, 140401460539391, ++SNULL, 140401462636543, 140401462677503, ++STORE, 140401460539392, 140401462636543, ++STORE, 140401462636544, 140401462677503, ++SNULL, 140401462636544, 140401462661119, ++STORE, 140401462661120, 140401462677503, ++STORE, 140401462636544, 140401462661119, ++ERASE, 140401462636544, 140401462661119, ++STORE, 140401462636544, 140401462661119, ++ERASE, 140401462661120, 140401462677503, ++STORE, 140401462661120, 140401462677503, ++STORE, 140401467088896, 140401467105279, ++SNULL, 140401462652927, 140401462661119, ++STORE, 140401462636544, 140401462652927, ++STORE, 140401462652928, 140401462661119, ++SNULL, 140401464872959, 140401464877055, ++STORE, 140401464868864, 140401464872959, ++STORE, 140401464872960, 140401464877055, ++SNULL, 94427776212991, 94427776217087, ++STORE, 94427776208896, 94427776212991, ++STORE, 94427776212992, 94427776217087, ++SNULL, 140401467138047, 140401467142143, ++STORE, 140401467133952, 140401467138047, ++STORE, 140401467138048, 140401467142143, ++ERASE, 140401467105280, 140401467133951, ++STORE, 94427784683520, 94427784818687, ++STORE, 140401450487808, 140401458880511, ++SNULL, 140401450491903, 140401458880511, ++STORE, 140401450487808, 140401450491903, ++STORE, 140401450491904, 140401458880511, ++STORE, 140401442095104, 140401450487807, ++STORE, 140401307877376, 140401442095103, ++SNULL, 140401307877376, 140401340055551, ++STORE, 140401340055552, 140401442095103, ++STORE, 140401307877376, 140401340055551, ++ERASE, 140401307877376, 140401340055551, ++SNULL, 140401407164415, 140401442095103, ++STORE, 140401340055552, 140401407164415, ++STORE, 140401407164416, 140401442095103, ++ERASE, 140401407164416, 140401442095103, ++SNULL, 140401340190719, 140401407164415, ++STORE, 140401340055552, 140401340190719, ++STORE, 140401340190720, 140401407164415, ++SNULL, 140401442099199, 140401450487807, ++STORE, 140401442095104, 140401442099199, ++STORE, 140401442099200, 140401450487807, ++STORE, 140401433702400, 140401442095103, ++SNULL, 140401433706495, 140401442095103, ++STORE, 140401433702400, 140401433706495, ++STORE, 140401433706496, 140401442095103, ++STORE, 140401425309696, 140401433702399, ++SNULL, 140401425313791, 140401433702399, ++STORE, 140401425309696, 140401425313791, ++STORE, 140401425313792, 140401433702399, ++STORE, 140401416916992, 140401425309695, ++SNULL, 140401416921087, 140401425309695, ++STORE, 140401416916992, 140401416921087, ++STORE, 140401416921088, 140401425309695, ++STORE, 140401408524288, 140401416916991, ++STORE, 140401205837824, 140401340055551, ++SNULL, 140401272946687, 140401340055551, ++STORE, 140401205837824, 140401272946687, ++STORE, 140401272946688, 140401340055551, ++ERASE, 140401272946688, 140401340055551, ++SNULL, 140401205972991, 140401272946687, ++STORE, 140401205837824, 140401205972991, ++STORE, 140401205972992, 140401272946687, ++STORE, 140401331662848, 140401340055551, ++STORE, 140401323270144, 140401340055551, ++STORE, 140401138728960, 140401205837823, ++STORE, 140401314877440, 140401340055551, ++SNULL, 140401408528383, 140401416916991, ++STORE, 140401408524288, 140401408528383, ++STORE, 140401408528384, 140401416916991, ++SNULL, 140401138864127, 140401205837823, ++STORE, 140401138728960, 140401138864127, ++STORE, 140401138864128, 140401205837823, ++STORE, 140401004511232, 140401138728959, ++SNULL, 140401071620095, 140401138728959, ++STORE, 140401004511232, 140401071620095, ++STORE, 140401071620096, 140401138728959, ++ERASE, 140401071620096, 140401138728959, ++STORE, 140400870293504, 140401071620095, ++SNULL, 140400937402367, 140401071620095, ++STORE, 140400870293504, 140400937402367, ++STORE, 140400937402368, 140401071620095, ++SNULL, 140400937402368, 140401004511231, ++STORE, 140401004511232, 140401071620095, ++STORE, 140400937402368, 140401004511231, ++ERASE, 140400937402368, 140401004511231, ++STORE, 140401306484736, 140401340055551, ++SNULL, 140401306484736, 140401323270143, ++STORE, 140401323270144, 140401340055551, ++STORE, 140401306484736, 140401323270143, ++SNULL, 140401323274239, 140401340055551, ++STORE, 140401323270144, 140401323274239, ++STORE, 140401323274240, 140401340055551, ++SNULL, 140401004646399, 140401071620095, ++STORE, 140401004511232, 140401004646399, ++STORE, 140401004646400, 140401071620095, ++SNULL, 140400870428671, 140400937402367, ++STORE, 140400870293504, 140400870428671, ++STORE, 140400870428672, 140400937402367, ++SNULL, 140401306488831, 140401323270143, ++STORE, 140401306484736, 140401306488831, ++STORE, 140401306488832, 140401323270143, ++STORE, 140401298092032, 140401306484735, ++SNULL, 140401306488832, 140401314877439, ++STORE, 140401314877440, 140401323270143, ++STORE, 140401306488832, 140401314877439, ++SNULL, 140401314881535, 140401323270143, ++STORE, 140401314877440, 140401314881535, ++STORE, 140401314881536, 140401323270143, ++SNULL, 140401323274240, 140401331662847, ++STORE, 140401331662848, 140401340055551, ++STORE, 140401323274240, 140401331662847, ++SNULL, 140401331666943, 140401340055551, ++STORE, 140401331662848, 140401331666943, ++STORE, 140401331666944, 140401340055551, ++SNULL, 140401298096127, 140401306484735, ++STORE, 140401298092032, 140401298096127, ++STORE, 140401298096128, 140401306484735, ++STORE, 140401289699328, 140401298092031, ++STORE, 140401281306624, 140401298092031, ++STORE, 140401130336256, 140401138728959, ++SNULL, 140401281306624, 140401289699327, ++STORE, 140401289699328, 140401298092031, ++STORE, 140401281306624, 140401289699327, ++SNULL, 140401289703423, 140401298092031, ++STORE, 140401289699328, 140401289703423, ++STORE, 140401289703424, 140401298092031, ++STORE, 140401121943552, 140401138728959, ++STORE, 140401113550848, 140401138728959, ++SNULL, 140401281310719, 140401289699327, ++STORE, 140401281306624, 140401281310719, ++STORE, 140401281310720, 140401289699327, ++SNULL, 140401113550848, 140401121943551, ++STORE, 140401121943552, 140401138728959, ++STORE, 140401113550848, 140401121943551, ++SNULL, 140401121947647, 140401138728959, ++STORE, 140401121943552, 140401121947647, ++STORE, 140401121947648, 140401138728959, ++STORE, 140401105158144, 140401121943551, ++SNULL, 140401121947648, 140401130336255, ++STORE, 140401130336256, 140401138728959, ++STORE, 140401121947648, 140401130336255, ++SNULL, 140401130340351, 140401138728959, ++STORE, 140401130336256, 140401130340351, ++STORE, 140401130340352, 140401138728959, ++STORE, 140401096765440, 140401121943551, ++SNULL, 140401096765440, 140401113550847, ++STORE, 140401113550848, 140401121943551, ++STORE, 140401096765440, 140401113550847, ++SNULL, 140401113554943, 140401121943551, ++STORE, 140401113550848, 140401113554943, ++STORE, 140401113554944, 140401121943551, ++STORE, 140401088372736, 140401113550847, ++SNULL, 140401088372736, 140401096765439, ++STORE, 140401096765440, 140401113550847, ++STORE, 140401088372736, 140401096765439, ++SNULL, 140401096769535, 140401113550847, ++STORE, 140401096765440, 140401096769535, ++STORE, 140401096769536, 140401113550847, ++SNULL, 140401096769536, 140401105158143, ++STORE, 140401105158144, 140401113550847, ++STORE, 140401096769536, 140401105158143, ++SNULL, 140401105162239, 140401113550847, ++STORE, 140401105158144, 140401105162239, ++STORE, 140401105162240, 140401113550847, ++SNULL, 140401088376831, 140401096765439, ++STORE, 140401088372736, 140401088376831, ++STORE, 140401088376832, 140401096765439, ++STORE, 140401079980032, 140401088372735, ++STORE, 140400996118528, 140401004511231, ++SNULL, 140401079984127, 140401088372735, ++STORE, 140401079980032, 140401079984127, ++STORE, 140401079984128, 140401088372735, ++SNULL, 140400996122623, 140401004511231, ++STORE, 140400996118528, 140400996122623, ++STORE, 140400996122624, 140401004511231, ++STORE, 140400987725824, 140400996118527, ++STORE, 140400979333120, 140400996118527, ++STORE, 140400803184640, 140400870293503, ++SNULL, 140400803319807, 140400870293503, ++STORE, 140400803184640, 140400803319807, ++STORE, 140400803319808, 140400870293503, ++SNULL, 140400979333120, 140400987725823, ++STORE, 140400987725824, 140400996118527, ++STORE, 140400979333120, 140400987725823, ++SNULL, 140400987729919, 140400996118527, ++STORE, 140400987725824, 140400987729919, ++STORE, 140400987729920, 140400996118527, ++STORE, 140400970940416, 140400987725823, ++STORE, 140400962547712, 140400987725823, ++STORE, 140400668966912, 140400803184639, ++STORE, 140400954155008, 140400987725823, ++STORE, 140400945762304, 140400987725823, ++STORE, 140400660574208, 140400668966911, ++STORE, 140400593465344, 140400660574207, ++STORE, 140400585072640, 140400593465343, ++STORE, 140400450854912, 140400585072639, ++STORE, 140400442462208, 140400450854911, ++STORE, 140400434069504, 140400450854911, ++STORE, 140400299851776, 140400434069503, ++STORE, 140400291459072, 140400299851775, ++SNULL, 140400299851776, 140400333422591, ++STORE, 140400333422592, 140400434069503, ++STORE, 140400299851776, 140400333422591, ++ERASE, 140400299851776, 140400333422591, ++STORE, 140400325029888, 140400333422591, ++STORE, 140400157241344, 140400291459071, ++STORE, 140400316637184, 140400333422591, ++STORE, 140400308244480, 140400333422591, ++STORE, 140400023023616, 140400291459071, ++STORE, 140400291459072, 140400333422591, ++SNULL, 140400023023616, 140400064987135, ++STORE, 140400064987136, 140400291459071, ++STORE, 140400023023616, 140400064987135, ++ERASE, 140400023023616, 140400064987135, ++STORE, 140400056594432, 140400064987135, ++SNULL, 140400056598527, 140400064987135, ++STORE, 140400056594432, 140400056598527, ++STORE, 140400056598528, 140400064987135, ++STORE, 140399989485568, 140400056594431, ++SNULL, 140400291459072, 140400316637183, ++STORE, 140400316637184, 140400333422591, ++STORE, 140400291459072, 140400316637183, ++SNULL, 140400316641279, 140400333422591, ++STORE, 140400316637184, 140400316641279, ++STORE, 140400316641280, 140400333422591, ++STORE, 140399855267840, 140400056594431, ++SNULL, 140399855267840, 140399863660543, ++STORE, 140399863660544, 140400056594431, ++STORE, 140399855267840, 140399863660543, ++ERASE, 140399855267840, 140399863660543, ++SNULL, 140400736075775, 140400803184639, ++STORE, 140400668966912, 140400736075775, ++STORE, 140400736075776, 140400803184639, ++ERASE, 140400736075776, 140400803184639, ++SNULL, 140400669102079, 140400736075775, ++STORE, 140400668966912, 140400669102079, ++STORE, 140400669102080, 140400736075775, ++STORE, 140400669102080, 140400803184639, ++SNULL, 140400669102080, 140400736075775, ++STORE, 140400736075776, 140400803184639, ++STORE, 140400669102080, 140400736075775, ++SNULL, 140400736210943, 140400803184639, ++STORE, 140400736075776, 140400736210943, ++STORE, 140400736210944, 140400803184639, ++ERASE, 140400593465344, 140400660574207, ++SNULL, 140400450854912, 140400467640319, ++STORE, 140400467640320, 140400585072639, ++STORE, 140400450854912, 140400467640319, ++ERASE, 140400450854912, 140400467640319, ++STORE, 140399729442816, 140400056594431, ++SNULL, 140400400531455, 140400434069503, ++STORE, 140400333422592, 140400400531455, ++STORE, 140400400531456, 140400434069503, ++ERASE, 140400400531456, 140400434069503, ++SNULL, 140400333557759, 140400400531455, ++STORE, 140400333422592, 140400333557759, ++STORE, 140400333557760, 140400400531455, ++SNULL, 140400157241343, 140400291459071, ++STORE, 140400064987136, 140400157241343, ++STORE, 140400157241344, 140400291459071, ++SNULL, 140400157241344, 140400199204863, ++STORE, 140400199204864, 140400291459071, ++STORE, 140400157241344, 140400199204863, ++ERASE, 140400157241344, 140400199204863, ++SNULL, 140400266313727, 140400291459071, ++STORE, 140400199204864, 140400266313727, ++STORE, 140400266313728, 140400291459071, ++ERASE, 140400266313728, 140400291459071, ++SNULL, 140400132095999, 140400157241343, ++STORE, 140400064987136, 140400132095999, ++STORE, 140400132096000, 140400157241343, ++ERASE, 140400132096000, 140400157241343, ++SNULL, 140400065122303, 140400132095999, ++STORE, 140400064987136, 140400065122303, ++STORE, 140400065122304, 140400132095999, ++SNULL, 140400945762304, 140400954155007, ++STORE, 140400954155008, 140400987725823, ++STORE, 140400945762304, 140400954155007, ++SNULL, 140400954159103, 140400987725823, ++STORE, 140400954155008, 140400954159103, ++STORE, 140400954159104, 140400987725823, ++SNULL, 140400434069504, 140400442462207, ++STORE, 140400442462208, 140400450854911, ++STORE, 140400434069504, 140400442462207, ++SNULL, 140400442466303, 140400450854911, ++STORE, 140400442462208, 140400442466303, ++STORE, 140400442466304, 140400450854911, ++SNULL, 140400291463167, 140400316637183, ++STORE, 140400291459072, 140400291463167, ++STORE, 140400291463168, 140400316637183, ++STORE, 140400652181504, 140400668966911, ++STORE, 140400643788800, 140400668966911, ++SNULL, 140400291463168, 140400299851775, ++STORE, 140400299851776, 140400316637183, ++STORE, 140400291463168, 140400299851775, ++SNULL, 140400299855871, 140400316637183, ++STORE, 140400299851776, 140400299855871, ++STORE, 140400299855872, 140400316637183, ++STORE, 140400635396096, 140400668966911, ++SNULL, 140400635396096, 140400643788799, ++STORE, 140400643788800, 140400668966911, ++STORE, 140400635396096, 140400643788799, ++SNULL, 140400643792895, 140400668966911, ++STORE, 140400643788800, 140400643792895, ++STORE, 140400643792896, 140400668966911, ++SNULL, 140399989485567, 140400056594431, ++STORE, 140399729442816, 140399989485567, ++STORE, 140399989485568, 140400056594431, ++ERASE, 140399989485568, 140400056594431, ++SNULL, 140399930769407, 140399989485567, ++STORE, 140399729442816, 140399930769407, ++STORE, 140399930769408, 140399989485567, ++ERASE, 140399930769408, 140399989485567, ++SNULL, 140400945766399, 140400954155007, ++STORE, 140400945762304, 140400945766399, ++STORE, 140400945766400, 140400954155007, ++SNULL, 140400534749183, 140400585072639, ++STORE, 140400467640320, 140400534749183, ++STORE, 140400534749184, 140400585072639, ++ERASE, 140400534749184, 140400585072639, ++SNULL, 140399796551679, 140399930769407, ++STORE, 140399729442816, 140399796551679, ++STORE, 140399796551680, 140399930769407, ++SNULL, 140399796551680, 140399863660543, ++STORE, 140399863660544, 140399930769407, ++STORE, 140399796551680, 140399863660543, ++ERASE, 140399796551680, 140399863660543, ++SNULL, 140400199340031, 140400266313727, ++STORE, 140400199204864, 140400199340031, ++STORE, 140400199340032, 140400266313727, ++STORE, 140400627003392, 140400643788799, ++SNULL, 140400316641280, 140400325029887, ++STORE, 140400325029888, 140400333422591, ++STORE, 140400316641280, 140400325029887, ++SNULL, 140400325033983, 140400333422591, ++STORE, 140400325029888, 140400325033983, ++STORE, 140400325033984, 140400333422591, ++SNULL, 140400627003392, 140400635396095, ++STORE, 140400635396096, 140400643788799, ++STORE, 140400627003392, 140400635396095, ++SNULL, 140400635400191, 140400643788799, ++STORE, 140400635396096, 140400635400191, ++STORE, 140400635400192, 140400643788799, ++SNULL, 140400434073599, 140400442462207, ++STORE, 140400434069504, 140400434073599, ++STORE, 140400434073600, 140400442462207, ++STORE, 140400618610688, 140400635396095, ++STORE, 140400610217984, 140400635396095, ++SNULL, 140400954159104, 140400962547711, ++STORE, 140400962547712, 140400987725823, ++STORE, 140400954159104, 140400962547711, ++SNULL, 140400962551807, 140400987725823, ++STORE, 140400962547712, 140400962551807, ++STORE, 140400962551808, 140400987725823, ++SNULL, 140400299855872, 140400308244479, ++STORE, 140400308244480, 140400316637183, ++STORE, 140400299855872, 140400308244479, ++SNULL, 140400308248575, 140400316637183, ++STORE, 140400308244480, 140400308248575, ++STORE, 140400308248576, 140400316637183, ++STORE, 140400601825280, 140400635396095, ++SNULL, 140400601829375, 140400635396095, ++STORE, 140400601825280, 140400601829375, ++STORE, 140400601829376, 140400635396095, ++STORE, 140400576679936, 140400593465343, ++SNULL, 140400576684031, 140400593465343, ++STORE, 140400576679936, 140400576684031, ++STORE, 140400576684032, 140400593465343, ++SNULL, 140400643792896, 140400652181503, ++STORE, 140400652181504, 140400668966911, ++STORE, 140400643792896, 140400652181503, ++SNULL, 140400652185599, 140400668966911, ++STORE, 140400652181504, 140400652185599, ++STORE, 140400652185600, 140400668966911, ++STORE, 140399595225088, 140399796551679, ++SNULL, 140399662333951, 140399796551679, ++STORE, 140399595225088, 140399662333951, ++STORE, 140399662333952, 140399796551679, ++SNULL, 140399662333952, 140399729442815, ++STORE, 140399729442816, 140399796551679, ++STORE, 140399662333952, 140399729442815, ++ERASE, 140399662333952, 140399729442815, ++SNULL, 140399863795711, 140399930769407, ++STORE, 140399863660544, 140399863795711, ++STORE, 140399863795712, 140399930769407, ++STORE, 140400568287232, 140400576679935, ++SNULL, 140400568291327, 140400576679935, ++STORE, 140400568287232, 140400568291327, ++STORE, 140400568291328, 140400576679935, ++SNULL, 140400467775487, 140400534749183, ++STORE, 140400467640320, 140400467775487, ++STORE, 140400467775488, 140400534749183, ++SNULL, 140399729577983, 140399796551679, ++STORE, 140399729442816, 140399729577983, ++STORE, 140399729577984, 140399796551679, ++SNULL, 140400601829376, 140400627003391, ++STORE, 140400627003392, 140400635396095, ++STORE, 140400601829376, 140400627003391, ++SNULL, 140400627007487, 140400635396095, ++STORE, 140400627003392, 140400627007487, ++STORE, 140400627007488, 140400635396095, ++STORE, 140400559894528, 140400568287231, ++STORE, 140400551501824, 140400568287231, ++STORE, 140400543109120, 140400568287231, ++STORE, 140400459247616, 140400467640319, ++STORE, 140400442466304, 140400467640319, ++SNULL, 140399595360255, 140399662333951, ++STORE, 140399595225088, 140399595360255, ++STORE, 140399595360256, 140399662333951, ++SNULL, 140400962551808, 140400970940415, ++STORE, 140400970940416, 140400987725823, ++STORE, 140400962551808, 140400970940415, ++SNULL, 140400970944511, 140400987725823, ++STORE, 140400970940416, 140400970944511, ++STORE, 140400970944512, 140400987725823, ++SNULL, 140400652185600, 140400660574207, ++STORE, 140400660574208, 140400668966911, ++STORE, 140400652185600, 140400660574207, ++SNULL, 140400660578303, 140400668966911, ++STORE, 140400660574208, 140400660578303, ++STORE, 140400660578304, 140400668966911, ++SNULL, 140400576684032, 140400585072639, ++STORE, 140400585072640, 140400593465343, ++STORE, 140400576684032, 140400585072639, ++SNULL, 140400585076735, 140400593465343, ++STORE, 140400585072640, 140400585076735, ++STORE, 140400585076736, 140400593465343, ++STORE, 140400425676800, 140400434069503, ++STORE, 140400417284096, 140400434069503, ++STORE, 140400408891392, 140400434069503, ++SNULL, 140400408891392, 140400417284095, ++STORE, 140400417284096, 140400434069503, ++STORE, 140400408891392, 140400417284095, ++SNULL, 140400417288191, 140400434069503, ++STORE, 140400417284096, 140400417288191, ++STORE, 140400417288192, 140400434069503, ++STORE, 140400283066368, 140400291459071, ++SNULL, 140400601829376, 140400618610687, ++STORE, 140400618610688, 140400627003391, ++STORE, 140400601829376, 140400618610687, ++SNULL, 140400618614783, 140400627003391, ++STORE, 140400618610688, 140400618614783, ++STORE, 140400618614784, 140400627003391, ++SNULL, 140400601829376, 140400610217983, ++STORE, 140400610217984, 140400618610687, ++STORE, 140400601829376, 140400610217983, ++SNULL, 140400610222079, 140400618610687, ++STORE, 140400610217984, 140400610222079, ++STORE, 140400610222080, 140400618610687, ++STORE, 140400274673664, 140400291459071, ++STORE, 140400190812160, 140400199204863, ++STORE, 140400182419456, 140400199204863, ++SNULL, 140400442466304, 140400450854911, ++STORE, 140400450854912, 140400467640319, ++STORE, 140400442466304, 140400450854911, ++SNULL, 140400450859007, 140400467640319, ++STORE, 140400450854912, 140400450859007, ++STORE, 140400450859008, 140400467640319, ++SNULL, 140400543109120, 140400559894527, ++STORE, 140400559894528, 140400568287231, ++STORE, 140400543109120, 140400559894527, ++SNULL, 140400559898623, 140400568287231, ++STORE, 140400559894528, 140400559898623, ++STORE, 140400559898624, 140400568287231, ++SNULL, 140400450859008, 140400459247615, ++STORE, 140400459247616, 140400467640319, ++STORE, 140400450859008, 140400459247615, ++SNULL, 140400459251711, 140400467640319, ++STORE, 140400459247616, 140400459251711, ++STORE, 140400459251712, 140400467640319, ++SNULL, 140400543113215, 140400559894527, ++STORE, 140400543109120, 140400543113215, ++STORE, 140400543113216, 140400559894527, ++SNULL, 140400970944512, 140400979333119, ++STORE, 140400979333120, 140400987725823, ++STORE, 140400970944512, 140400979333119, ++SNULL, 140400979337215, 140400987725823, ++STORE, 140400979333120, 140400979337215, ++STORE, 140400979337216, 140400987725823, ++STORE, 140400174026752, 140400199204863, ++SNULL, 140400174030847, 140400199204863, ++STORE, 140400174026752, 140400174030847, ++STORE, 140400174030848, 140400199204863, ++SNULL, 140400274673664, 140400283066367, ++STORE, 140400283066368, 140400291459071, ++STORE, 140400274673664, 140400283066367, ++SNULL, 140400283070463, 140400291459071, ++STORE, 140400283066368, 140400283070463, ++STORE, 140400283070464, 140400291459071, ++STORE, 140400165634048, 140400174026751, ++SNULL, 140400165638143, 140400174026751, ++STORE, 140400165634048, 140400165638143, ++STORE, 140400165638144, 140400174026751, ++SNULL, 140400174030848, 140400182419455, ++STORE, 140400182419456, 140400199204863, ++STORE, 140400174030848, 140400182419455, ++SNULL, 140400182423551, 140400199204863, ++STORE, 140400182419456, 140400182423551, ++STORE, 140400182423552, 140400199204863, ++SNULL, 140400182423552, 140400190812159, ++STORE, 140400190812160, 140400199204863, ++STORE, 140400182423552, 140400190812159, ++SNULL, 140400190816255, 140400199204863, ++STORE, 140400190812160, 140400190816255, ++STORE, 140400190816256, 140400199204863, ++STORE, 140400157241344, 140400165634047, ++SNULL, 140400157245439, 140400165634047, ++STORE, 140400157241344, 140400157245439, ++STORE, 140400157245440, 140400165634047, ++SNULL, 140400408895487, 140400417284095, ++STORE, 140400408891392, 140400408895487, ++STORE, 140400408895488, 140400417284095, ++SNULL, 140400417288192, 140400425676799, ++STORE, 140400425676800, 140400434069503, ++STORE, 140400417288192, 140400425676799, ++SNULL, 140400425680895, 140400434069503, ++STORE, 140400425676800, 140400425680895, ++STORE, 140400425680896, 140400434069503, ++STORE, 140400148848640, 140400157241343, ++SNULL, 140400148852735, 140400157241343, ++STORE, 140400148848640, 140400148852735, ++STORE, 140400148852736, 140400157241343, ++SNULL, 140400543113216, 140400551501823, ++STORE, 140400551501824, 140400559894527, ++STORE, 140400543113216, 140400551501823, ++SNULL, 140400551505919, 140400559894527, ++STORE, 140400551501824, 140400551505919, ++STORE, 140400551505920, 140400559894527, ++STORE, 140400140455936, 140400148848639, ++STORE, 140400048201728, 140400056594431, ++SNULL, 140400140460031, 140400148848639, ++STORE, 140400140455936, 140400140460031, ++STORE, 140400140460032, 140400148848639, ++STORE, 140400039809024, 140400056594431, ++SNULL, 140400039813119, 140400056594431, ++STORE, 140400039809024, 140400039813119, ++STORE, 140400039813120, 140400056594431, ++STORE, 140400031416320, 140400039809023, ++STORE, 140400023023616, 140400039809023, ++SNULL, 140400274677759, 140400283066367, ++STORE, 140400274673664, 140400274677759, ++STORE, 140400274677760, 140400283066367, ++STORE, 140400014630912, 140400039809023, ++STORE, 140400006238208, 140400039809023, ++STORE, 140399997845504, 140400039809023, ++SNULL, 140399997849599, 140400039809023, ++STORE, 140399997845504, 140399997849599, ++STORE, 140399997849600, 140400039809023, ++STORE, 140399989452800, 140399997845503, ++SNULL, 140399989456895, 140399997845503, ++STORE, 140399989452800, 140399989456895, ++STORE, 140399989456896, 140399997845503, ++STORE, 140399981060096, 140399989452799, ++SNULL, 140399981064191, 140399989452799, ++STORE, 140399981060096, 140399981064191, ++STORE, 140399981064192, 140399989452799, ++STORE, 140399972667392, 140399981060095, ++STORE, 140399964274688, 140399981060095, ++SNULL, 140399964278783, 140399981060095, ++STORE, 140399964274688, 140399964278783, ++STORE, 140399964278784, 140399981060095, ++SNULL, 140400039813120, 140400048201727, ++STORE, 140400048201728, 140400056594431, ++STORE, 140400039813120, 140400048201727, ++SNULL, 140400048205823, 140400056594431, ++STORE, 140400048201728, 140400048205823, ++STORE, 140400048205824, 140400056594431, ++SNULL, 140399997849600, 140400031416319, ++STORE, 140400031416320, 140400039809023, ++STORE, 140399997849600, 140400031416319, ++SNULL, 140400031420415, 140400039809023, ++STORE, 140400031416320, 140400031420415, ++STORE, 140400031420416, 140400039809023, ++STORE, 140399955881984, 140399964274687, ++SNULL, 140399955886079, 140399964274687, ++STORE, 140399955881984, 140399955886079, ++STORE, 140399955886080, 140399964274687, ++STORE, 140399947489280, 140399955881983, ++STORE, 140399939096576, 140399955881983, ++STORE, 140399855267840, 140399863660543, ++SNULL, 140399939100671, 140399955881983, ++STORE, 140399939096576, 140399939100671, ++STORE, 140399939100672, 140399955881983, ++SNULL, 140399997849600, 140400014630911, ++STORE, 140400014630912, 140400031416319, ++STORE, 140399997849600, 140400014630911, ++SNULL, 140400014635007, 140400031416319, ++STORE, 140400014630912, 140400014635007, ++STORE, 140400014635008, 140400031416319, ++SNULL, 140400014635008, 140400023023615, ++STORE, 140400023023616, 140400031416319, ++STORE, 140400014635008, 140400023023615, ++SNULL, 140400023027711, 140400031416319, ++STORE, 140400023023616, 140400023027711, ++STORE, 140400023027712, 140400031416319, ++SNULL, 140399997849600, 140400006238207, ++STORE, 140400006238208, 140400014630911, ++STORE, 140399997849600, 140400006238207, ++SNULL, 140400006242303, 140400014630911, ++STORE, 140400006238208, 140400006242303, ++STORE, 140400006242304, 140400014630911, ++STORE, 140399846875136, 140399863660543, ++STORE, 140399838482432, 140399863660543, ++SNULL, 140399838486527, 140399863660543, ++STORE, 140399838482432, 140399838486527, ++STORE, 140399838486528, 140399863660543, ++SNULL, 140399939100672, 140399947489279, ++STORE, 140399947489280, 140399955881983, ++STORE, 140399939100672, 140399947489279, ++SNULL, 140399947493375, 140399955881983, ++STORE, 140399947489280, 140399947493375, ++STORE, 140399947493376, 140399955881983, ++SNULL, 140399964278784, 140399972667391, ++STORE, 140399972667392, 140399981060095, ++STORE, 140399964278784, 140399972667391, ++SNULL, 140399972671487, 140399981060095, ++STORE, 140399972667392, 140399972671487, ++STORE, 140399972671488, 140399981060095, ++SNULL, 140399838486528, 140399855267839, ++STORE, 140399855267840, 140399863660543, ++STORE, 140399838486528, 140399855267839, ++SNULL, 140399855271935, 140399863660543, ++STORE, 140399855267840, 140399855271935, ++STORE, 140399855271936, 140399863660543, ++STORE, 140399830089728, 140399838482431, ++SNULL, 140399830093823, 140399838482431, ++STORE, 140399830089728, 140399830093823, ++STORE, 140399830093824, 140399838482431, ++STORE, 140399821697024, 140399830089727, ++SNULL, 140399821701119, 140399830089727, ++STORE, 140399821697024, 140399821701119, ++STORE, 140399821701120, 140399830089727, ++SNULL, 140399838486528, 140399846875135, ++STORE, 140399846875136, 140399855267839, ++STORE, 140399838486528, 140399846875135, ++SNULL, 140399846879231, 140399855267839, ++STORE, 140399846875136, 140399846879231, ++STORE, 140399846879232, 140399855267839, ++STORE, 140399813304320, 140399821697023, ++STORE, 140399804911616, 140399821697023, ++SNULL, 140399804915711, 140399821697023, ++STORE, 140399804911616, 140399804915711, ++STORE, 140399804915712, 140399821697023, ++STORE, 140399721050112, 140399729442815, ++SNULL, 140399804915712, 140399813304319, ++STORE, 140399813304320, 140399821697023, ++STORE, 140399804915712, 140399813304319, ++SNULL, 140399813308415, 140399821697023, ++STORE, 140399813304320, 140399813308415, ++STORE, 140399813308416, 140399821697023, ++SNULL, 140399721054207, 140399729442815, ++STORE, 140399721050112, 140399721054207, ++STORE, 140399721054208, 140399729442815, ++STORE, 140401467105280, 140401467133951, ++STORE, 140401279115264, 140401281306623, ++SNULL, 140401279115264, 140401279205375, ++STORE, 140401279205376, 140401281306623, ++STORE, 140401279115264, 140401279205375, ++SNULL, 140401281298431, 140401281306623, ++STORE, 140401279205376, 140401281298431, ++STORE, 140401281298432, 140401281306623, ++ERASE, 140401281298432, 140401281306623, ++STORE, 140401281298432, 140401281306623, ++SNULL, 140401281302527, 140401281306623, ++STORE, 140401281298432, 140401281302527, ++STORE, 140401281302528, 140401281306623, ++ERASE, 140401467105280, 140401467133951, ++ERASE, 140400056594432, 140400056598527, ++ERASE, 140400056598528, 140400064987135, ++ERASE, 140400635396096, 140400635400191, ++ERASE, 140400635400192, 140400643788799, ++ERASE, 140400408891392, 140400408895487, ++ERASE, 140400408895488, 140400417284095, ++ERASE, 140400299851776, 140400299855871, ++ERASE, 140400299855872, 140400308244479, ++ERASE, 140400627003392, 140400627007487, ++ERASE, 140400627007488, 140400635396095, ++ERASE, 140400954155008, 140400954159103, ++ERASE, 140400954159104, 140400962547711, ++ERASE, 140400291459072, 140400291463167, ++ERASE, 140400291463168, 140400299851775, ++ERASE, 140400643788800, 140400643792895, ++ERASE, 140400643792896, 140400652181503, ++ERASE, 140400325029888, 140400325033983, ++ERASE, 140400325033984, 140400333422591, ++ERASE, 140400610217984, 140400610222079, ++ERASE, 140400610222080, 140400618610687, ++ERASE, 140400190812160, 140400190816255, ++ERASE, 140400190816256, 140400199204863, ++ERASE, 140399964274688, 140399964278783, ++ERASE, 140399964278784, 140399972667391, ++ERASE, 140400945762304, 140400945766399, ++ERASE, 140400945766400, 140400954155007, ++ERASE, 140400568287232, 140400568291327, ++ERASE, 140400568291328, 140400576679935, ++ERASE, 140399972667392, 140399972671487, ++ERASE, 140399972671488, 140399981060095, ++ERASE, 140400962547712, 140400962551807, ++ERASE, 140400962551808, 140400970940415, ++ERASE, 140400987725824, 140400987729919, ++ERASE, 140400987729920, 140400996118527, ++ERASE, 140400652181504, 140400652185599, ++ERASE, 140400652185600, 140400660574207, ++ERASE, 140400450854912, 140400450859007, ++ERASE, 140400450859008, 140400459247615, ++ERASE, 140400031416320, 140400031420415, ++ERASE, 140400031420416, 140400039809023, ++ERASE, 140400308244480, 140400308248575, ++ERASE, 140400308248576, 140400316637183, ++ERASE, 140400434069504, 140400434073599, ++ERASE, 140400434073600, 140400442462207, ++ERASE, 140400543109120, 140400543113215, ++ERASE, 140400543113216, 140400551501823, ++ERASE, 140400023023616, 140400023027711, ++ERASE, 140400023027712, 140400031416319, ++ERASE, 140399813304320, 140399813308415, ++ERASE, 140399813308416, 140399821697023, ++ERASE, 140400316637184, 140400316641279, ++ERASE, 140400316641280, 140400325029887, ++ERASE, 140400585072640, 140400585076735, ++ERASE, 140400585076736, 140400593465343, ++ERASE, 140400148848640, 140400148852735, ++ERASE, 140400148852736, 140400157241343, ++ERASE, 140399955881984, 140399955886079, ++ERASE, 140399955886080, 140399964274687, ++ERASE, 140399821697024, 140399821701119, ++ERASE, 140399821701120, 140399830089727, ++ERASE, 140400601825280, 140400601829375, ++ERASE, 140400601829376, 140400610217983, ++ERASE, 140400979333120, 140400979337215, ++ERASE, 140400979337216, 140400987725823, ++ERASE, 140399997845504, 140399997849599, ++ERASE, 140399997849600, 140400006238207, ++ERASE, 140400459247616, 140400459251711, ++ERASE, 140400459251712, 140400467640319, ++ERASE, 140400551501824, 140400551505919, ++ERASE, 140400551505920, 140400559894527, ++ERASE, 140399939096576, 140399939100671, ++ERASE, 140399939100672, 140399947489279, ++ERASE, 140400442462208, 140400442466303, ++ERASE, 140400442466304, 140400450854911, ++ERASE, 140400576679936, 140400576684031, ++ERASE, 140400576684032, 140400585072639, ++ERASE, 140400559894528, 140400559898623, ++ERASE, 140400559898624, 140400568287231, ++ERASE, 140400417284096, 140400417288191, ++ERASE, 140400417288192, 140400425676799, ++ERASE, 140400283066368, 140400283070463, ++ERASE, 140400283070464, 140400291459071, ++ }; ++ unsigned long set33[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734562918400, 140737488351231, ++SNULL, 140734562922495, 140737488351231, ++STORE, 140734562918400, 140734562922495, ++STORE, 140734562787328, 140734562922495, ++STORE, 94133878984704, 94133881237503, ++SNULL, 94133879115775, 94133881237503, ++STORE, 94133878984704, 94133879115775, ++STORE, 94133879115776, 94133881237503, ++ERASE, 94133879115776, 94133881237503, ++STORE, 94133881208832, 94133881217023, ++STORE, 94133881217024, 94133881237503, ++STORE, 140583654043648, 140583656296447, ++SNULL, 140583654187007, 140583656296447, ++STORE, 140583654043648, 140583654187007, ++STORE, 140583654187008, 140583656296447, ++ERASE, 140583654187008, 140583656296447, ++STORE, 140583656284160, 140583656292351, ++STORE, 140583656292352, 140583656296447, ++STORE, 140734564319232, 140734564323327, ++STORE, 140734564306944, 140734564319231, ++STORE, 140583656255488, 140583656284159, ++STORE, 140583656247296, 140583656255487, ++STORE, 140583651827712, 140583654043647, ++SNULL, 140583651827712, 140583651926015, ++STORE, 140583651926016, 140583654043647, ++STORE, 140583651827712, 140583651926015, ++SNULL, 140583654019071, 140583654043647, ++STORE, 140583651926016, 140583654019071, ++STORE, 140583654019072, 140583654043647, ++SNULL, 140583654019072, 140583654027263, ++STORE, 140583654027264, 140583654043647, ++STORE, 140583654019072, 140583654027263, ++ERASE, 140583654019072, 140583654027263, ++STORE, 140583654019072, 140583654027263, ++ERASE, 140583654027264, 140583654043647, ++STORE, 140583654027264, 140583654043647, ++STORE, 140583648030720, 140583651827711, ++SNULL, 140583648030720, 140583649689599, ++STORE, 140583649689600, 140583651827711, ++STORE, 140583648030720, 140583649689599, ++SNULL, 140583651786751, 140583651827711, ++STORE, 140583649689600, 140583651786751, ++STORE, 140583651786752, 140583651827711, ++SNULL, 140583651786752, 140583651811327, ++STORE, 140583651811328, 140583651827711, ++STORE, 140583651786752, 140583651811327, ++ERASE, 140583651786752, 140583651811327, ++STORE, 140583651786752, 140583651811327, ++ERASE, 140583651811328, 140583651827711, ++STORE, 140583651811328, 140583651827711, ++STORE, 140583656239104, 140583656255487, ++SNULL, 140583651803135, 140583651811327, ++STORE, 140583651786752, 140583651803135, ++STORE, 140583651803136, 140583651811327, ++SNULL, 140583654023167, 140583654027263, ++STORE, 140583654019072, 140583654023167, ++STORE, 140583654023168, 140583654027263, ++SNULL, 94133881212927, 94133881217023, ++STORE, 94133881208832, 94133881212927, ++STORE, 94133881212928, 94133881217023, ++SNULL, 140583656288255, 140583656292351, ++STORE, 140583656284160, 140583656288255, ++STORE, 140583656288256, 140583656292351, ++ERASE, 140583656255488, 140583656284159, ++STORE, 94133881733120, 94133881868287, ++STORE, 140583639638016, 140583648030719, ++SNULL, 140583639642111, 140583648030719, ++STORE, 140583639638016, 140583639642111, ++STORE, 140583639642112, 140583648030719, ++STORE, 140583631245312, 140583639638015, ++STORE, 140583497027584, 140583631245311, ++SNULL, 140583497027584, 140583540621311, ++STORE, 140583540621312, 140583631245311, ++STORE, 140583497027584, 140583540621311, ++ERASE, 140583497027584, 140583540621311, ++SNULL, 140583607730175, 140583631245311, ++STORE, 140583540621312, 140583607730175, ++STORE, 140583607730176, 140583631245311, ++ERASE, 140583607730176, 140583631245311, ++SNULL, 140583540756479, 140583607730175, ++STORE, 140583540621312, 140583540756479, ++STORE, 140583540756480, 140583607730175, ++SNULL, 140583631249407, 140583639638015, ++STORE, 140583631245312, 140583631249407, ++STORE, 140583631249408, 140583639638015, ++STORE, 140583622852608, 140583631245311, ++SNULL, 140583622856703, 140583631245311, ++STORE, 140583622852608, 140583622856703, ++STORE, 140583622856704, 140583631245311, ++STORE, 140583614459904, 140583622852607, ++SNULL, 140583614463999, 140583622852607, ++STORE, 140583614459904, 140583614463999, ++STORE, 140583614464000, 140583622852607, ++STORE, 140583532228608, 140583540621311, ++SNULL, 140583532232703, 140583540621311, ++STORE, 140583532228608, 140583532232703, ++STORE, 140583532232704, 140583540621311, ++STORE, 140583523835904, 140583532228607, ++STORE, 140583515443200, 140583532228607, ++STORE, 140583507050496, 140583532228607, ++STORE, 140583372832768, 140583507050495, ++STORE, 140583364440064, 140583372832767, ++STORE, 140583230222336, 140583364440063, ++STORE, 140583096004608, 140583364440063, ++SNULL, 140583230222335, 140583364440063, ++STORE, 140583096004608, 140583230222335, ++STORE, 140583230222336, 140583364440063, ++SNULL, 140583230222336, 140583272185855, ++STORE, 140583272185856, 140583364440063, ++STORE, 140583230222336, 140583272185855, ++ERASE, 140583230222336, 140583272185855, ++STORE, 140582961786880, 140583230222335, ++SNULL, 140583372832768, 140583406403583, ++STORE, 140583406403584, 140583507050495, ++STORE, 140583372832768, 140583406403583, ++ERASE, 140583372832768, 140583406403583, ++SNULL, 140583473512447, 140583507050495, ++STORE, 140583406403584, 140583473512447, ++STORE, 140583473512448, 140583507050495, ++ERASE, 140583473512448, 140583507050495, ++SNULL, 140583096004607, 140583230222335, ++STORE, 140582961786880, 140583096004607, ++STORE, 140583096004608, 140583230222335, ++SNULL, 140583096004608, 140583137968127, ++STORE, 140583137968128, 140583230222335, ++STORE, 140583096004608, 140583137968127, ++ERASE, 140583096004608, 140583137968127, ++SNULL, 140583339294719, 140583364440063, ++STORE, 140583272185856, 140583339294719, ++STORE, 140583339294720, 140583364440063, ++ERASE, 140583339294720, 140583364440063, ++SNULL, 140583272321023, 140583339294719, ++STORE, 140583272185856, 140583272321023, ++STORE, 140583272321024, 140583339294719, ++SNULL, 140582961786880, 140583003750399, ++STORE, 140583003750400, 140583096004607, ++STORE, 140582961786880, 140583003750399, ++ERASE, 140582961786880, 140583003750399, ++ }; ++ ++ unsigned long set34[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140731327180800, 140737488351231, ++SNULL, 140731327184895, 140737488351231, ++STORE, 140731327180800, 140731327184895, ++STORE, 140731327049728, 140731327184895, ++STORE, 94632924487680, 94632926740479, ++SNULL, 94632924618751, 94632926740479, ++STORE, 94632924487680, 94632924618751, ++STORE, 94632924618752, 94632926740479, ++ERASE, 94632924618752, 94632926740479, ++STORE, 94632926711808, 94632926719999, ++STORE, 94632926720000, 94632926740479, ++STORE, 140012544888832, 140012547141631, ++SNULL, 140012545032191, 140012547141631, ++STORE, 140012544888832, 140012545032191, ++STORE, 140012545032192, 140012547141631, ++ERASE, 140012545032192, 140012547141631, ++STORE, 140012547129344, 140012547137535, ++STORE, 140012547137536, 140012547141631, ++STORE, 140731327725568, 140731327729663, ++STORE, 140731327713280, 140731327725567, ++STORE, 140012547100672, 140012547129343, ++STORE, 140012547092480, 140012547100671, ++STORE, 140012542672896, 140012544888831, ++SNULL, 140012542672896, 140012542771199, ++STORE, 140012542771200, 140012544888831, ++STORE, 140012542672896, 140012542771199, ++SNULL, 140012544864255, 140012544888831, ++STORE, 140012542771200, 140012544864255, ++STORE, 140012544864256, 140012544888831, ++SNULL, 140012544864256, 140012544872447, ++STORE, 140012544872448, 140012544888831, ++STORE, 140012544864256, 140012544872447, ++ERASE, 140012544864256, 140012544872447, ++STORE, 140012544864256, 140012544872447, ++ERASE, 140012544872448, 140012544888831, ++STORE, 140012544872448, 140012544888831, ++STORE, 140012538875904, 140012542672895, ++SNULL, 140012538875904, 140012540534783, ++STORE, 140012540534784, 140012542672895, ++STORE, 140012538875904, 140012540534783, ++SNULL, 140012542631935, 140012542672895, ++STORE, 140012540534784, 140012542631935, ++STORE, 140012542631936, 140012542672895, ++SNULL, 140012542631936, 140012542656511, ++STORE, 140012542656512, 140012542672895, ++STORE, 140012542631936, 140012542656511, ++ERASE, 140012542631936, 140012542656511, ++STORE, 140012542631936, 140012542656511, ++ERASE, 140012542656512, 140012542672895, ++STORE, 140012542656512, 140012542672895, ++STORE, 140012547084288, 140012547100671, ++SNULL, 140012542648319, 140012542656511, ++STORE, 140012542631936, 140012542648319, ++STORE, 140012542648320, 140012542656511, ++SNULL, 140012544868351, 140012544872447, ++STORE, 140012544864256, 140012544868351, ++STORE, 140012544868352, 140012544872447, ++SNULL, 94632926715903, 94632926719999, ++STORE, 94632926711808, 94632926715903, ++STORE, 94632926715904, 94632926719999, ++SNULL, 140012547133439, 140012547137535, ++STORE, 140012547129344, 140012547133439, ++STORE, 140012547133440, 140012547137535, ++ERASE, 140012547100672, 140012547129343, ++STORE, 94632939606016, 94632939741183, ++STORE, 140012530483200, 140012538875903, ++SNULL, 140012530487295, 140012538875903, ++STORE, 140012530483200, 140012530487295, ++STORE, 140012530487296, 140012538875903, ++STORE, 140012522090496, 140012530483199, ++STORE, 140012387872768, 140012522090495, ++SNULL, 140012387872768, 140012444188671, ++STORE, 140012444188672, 140012522090495, ++STORE, 140012387872768, 140012444188671, ++ERASE, 140012387872768, 140012444188671, ++SNULL, 140012511297535, 140012522090495, ++STORE, 140012444188672, 140012511297535, ++STORE, 140012511297536, 140012522090495, ++ERASE, 140012511297536, 140012522090495, ++SNULL, 140012444323839, 140012511297535, ++STORE, 140012444188672, 140012444323839, ++STORE, 140012444323840, 140012511297535, ++SNULL, 140012522094591, 140012530483199, ++STORE, 140012522090496, 140012522094591, ++STORE, 140012522094592, 140012530483199, ++STORE, 140012513697792, 140012522090495, ++SNULL, 140012513701887, 140012522090495, ++STORE, 140012513697792, 140012513701887, ++STORE, 140012513701888, 140012522090495, ++STORE, 140012435795968, 140012444188671, ++SNULL, 140012435800063, 140012444188671, ++STORE, 140012435795968, 140012435800063, ++STORE, 140012435800064, 140012444188671, ++STORE, 140012427403264, 140012435795967, ++SNULL, 140012427407359, 140012435795967, ++STORE, 140012427403264, 140012427407359, ++STORE, 140012427407360, 140012435795967, ++STORE, 140012419010560, 140012427403263, ++STORE, 140012410617856, 140012427403263, ++STORE, 140012276400128, 140012410617855, ++STORE, 140012268007424, 140012276400127, ++STORE, 140012133789696, 140012268007423, ++SNULL, 140012133789696, 140012175753215, ++STORE, 140012175753216, 140012268007423, ++STORE, 140012133789696, 140012175753215, ++ERASE, 140012133789696, 140012175753215, ++STORE, 140012041535488, 140012268007423, ++SNULL, 140012108644351, 140012268007423, ++STORE, 140012041535488, 140012108644351, ++STORE, 140012108644352, 140012268007423, ++SNULL, 140012108644352, 140012175753215, ++STORE, 140012175753216, 140012268007423, ++STORE, 140012108644352, 140012175753215, ++ERASE, 140012108644352, 140012175753215, ++SNULL, 140012276400128, 140012309970943, ++STORE, 140012309970944, 140012410617855, ++STORE, 140012276400128, 140012309970943, ++ERASE, 140012276400128, 140012309970943, ++STORE, 140012301578240, 140012309970943, ++STORE, 140012041535488, 140012268007423, ++SNULL, 140012242862079, 140012268007423, ++STORE, 140012041535488, 140012242862079, ++STORE, 140012242862080, 140012268007423, ++ERASE, 140012242862080, 140012268007423, ++SNULL, 140012041670655, 140012242862079, ++STORE, 140012041535488, 140012041670655, ++STORE, 140012041670656, 140012242862079, ++SNULL, 140012041670656, 140012108644351, ++STORE, 140012108644352, 140012242862079, ++STORE, 140012041670656, 140012108644351, ++SNULL, 140012108779519, 140012242862079, ++STORE, 140012108644352, 140012108779519, ++STORE, 140012108779520, 140012242862079, ++SNULL, 140012377079807, 140012410617855, ++STORE, 140012309970944, 140012377079807, ++STORE, 140012377079808, 140012410617855, ++ERASE, 140012377079808, 140012410617855, ++SNULL, 140012310106111, 140012377079807, ++STORE, 140012309970944, 140012310106111, ++STORE, 140012310106112, 140012377079807, ++SNULL, 140012410621951, 140012427403263, ++STORE, 140012410617856, 140012410621951, ++STORE, 140012410621952, 140012427403263, ++SNULL, 140012108779520, 140012175753215, ++STORE, 140012175753216, 140012242862079, ++STORE, 140012108779520, 140012175753215, ++SNULL, 140012175888383, 140012242862079, ++STORE, 140012175753216, 140012175888383, ++STORE, 140012175888384, 140012242862079, ++SNULL, 140012301582335, 140012309970943, ++STORE, 140012301578240, 140012301582335, ++STORE, 140012301582336, 140012309970943, ++SNULL, 140012410621952, 140012419010559, ++STORE, 140012419010560, 140012427403263, ++STORE, 140012410621952, 140012419010559, ++SNULL, 140012419014655, 140012427403263, ++STORE, 140012419010560, 140012419014655, ++STORE, 140012419014656, 140012427403263, ++SNULL, 140012268011519, 140012276400127, ++STORE, 140012268007424, 140012268011519, ++STORE, 140012268011520, 140012276400127, ++STORE, 140012402225152, 140012410617855, ++STORE, 140012393832448, 140012410617855, ++SNULL, 140012393832448, 140012402225151, ++STORE, 140012402225152, 140012410617855, ++STORE, 140012393832448, 140012402225151, ++SNULL, 140012402229247, 140012410617855, ++STORE, 140012402225152, 140012402229247, ++STORE, 140012402229248, 140012410617855, ++STORE, 140012385439744, 140012402225151, ++SNULL, 140012385439744, 140012393832447, ++STORE, 140012393832448, 140012402225151, ++STORE, 140012385439744, 140012393832447, ++SNULL, 140012393836543, 140012402225151, ++STORE, 140012393832448, 140012393836543, ++STORE, 140012393836544, 140012402225151, ++STORE, 140012293185536, 140012301578239, ++STORE, 140012284792832, 140012301578239, ++SNULL, 140012284792832, 140012293185535, ++STORE, 140012293185536, 140012301578239, ++STORE, 140012284792832, 140012293185535, ++SNULL, 140012293189631, 140012301578239, ++STORE, 140012293185536, 140012293189631, ++STORE, 140012293189632, 140012301578239, ++STORE, 140012268011520, 140012284792831, ++SNULL, 140012385443839, 140012393832447, ++STORE, 140012385439744, 140012385443839, ++STORE, 140012385443840, 140012393832447, ++STORE, 140012259614720, 140012268007423, ++SNULL, 140012259618815, 140012268007423, ++STORE, 140012259614720, 140012259618815, ++STORE, 140012259618816, 140012268007423, ++STORE, 140012251222016, 140012259614719, ++SNULL, 140012251226111, 140012259614719, ++STORE, 140012251222016, 140012251226111, ++STORE, 140012251226112, 140012259614719, ++SNULL, 140012284796927, 140012293185535, ++STORE, 140012284792832, 140012284796927, ++STORE, 140012284796928, 140012293185535, ++SNULL, 140012268011520, 140012276400127, ++STORE, 140012276400128, 140012284792831, ++STORE, 140012268011520, 140012276400127, ++SNULL, 140012276404223, 140012284792831, ++STORE, 140012276400128, 140012276404223, ++STORE, 140012276404224, 140012284792831, ++STORE, 140012033142784, 140012041535487, ++SNULL, 140012033146879, 140012041535487, ++STORE, 140012033142784, 140012033146879, ++STORE, 140012033146880, 140012041535487, ++STORE, 140012024750080, 140012033142783, ++STORE, 140012016357376, 140012033142783, ++SNULL, 140012016357376, 140012024750079, ++STORE, 140012024750080, 140012033142783, ++STORE, 140012016357376, 140012024750079, ++SNULL, 140012024754175, 140012033142783, ++STORE, 140012024750080, 140012024754175, ++STORE, 140012024754176, 140012033142783, ++SNULL, 140012016361471, 140012024750079, ++STORE, 140012016357376, 140012016361471, ++STORE, 140012016361472, 140012024750079, ++STORE, 140012007964672, 140012016357375, ++SNULL, 140012007968767, 140012016357375, ++STORE, 140012007964672, 140012007968767, ++STORE, 140012007968768, 140012016357375, ++STORE, 140011999571968, 140012007964671, ++STORE, 140011991179264, 140012007964671, ++STORE, 140011856961536, 140011991179263, ++STORE, 140011848568832, 140011856961535, ++STORE, 140011714351104, 140011848568831, ++SNULL, 140011714351104, 140011773100031, ++STORE, 140011773100032, 140011848568831, ++STORE, 140011714351104, 140011773100031, ++ERASE, 140011714351104, 140011773100031, ++STORE, 140011764707328, 140011773100031, ++STORE, 140011756314624, 140011773100031, ++STORE, 140011622096896, 140011756314623, ++STORE, 140011613704192, 140011622096895, ++STORE, 140011479486464, 140011613704191, ++STORE, 140011471093760, 140011479486463, ++SNULL, 140011479486464, 140011504664575, ++STORE, 140011504664576, 140011613704191, ++STORE, 140011479486464, 140011504664575, ++ERASE, 140011479486464, 140011504664575, ++STORE, 140011496271872, 140011504664575, ++STORE, 140011487879168, 140011504664575, ++STORE, 140011336876032, 140011471093759, ++SNULL, 140011336876032, 140011370446847, ++STORE, 140011370446848, 140011471093759, ++STORE, 140011336876032, 140011370446847, ++ERASE, 140011336876032, 140011370446847, ++STORE, 140011471093760, 140011487879167, ++STORE, 140011362054144, 140011370446847, ++SNULL, 140011362058239, 140011370446847, ++STORE, 140011362054144, 140011362058239, ++STORE, 140011362058240, 140011370446847, ++STORE, 140011353661440, 140011362054143, ++STORE, 140011345268736, 140011362054143, ++SNULL, 140011345272831, 140011362054143, ++STORE, 140011345268736, 140011345272831, ++STORE, 140011345272832, 140011362054143, ++STORE, 140011336876032, 140011345268735, ++STORE, 140011328483328, 140011345268735, ++SNULL, 140011328487423, 140011345268735, ++STORE, 140011328483328, 140011328487423, ++STORE, 140011328487424, 140011345268735, ++STORE, 140011320090624, 140011328483327, ++STORE, 140011185872896, 140011320090623, ++SNULL, 140011185872896, 140011236229119, ++STORE, 140011236229120, 140011320090623, ++STORE, 140011185872896, 140011236229119, ++ERASE, 140011185872896, 140011236229119, ++SNULL, 140011856961536, 140011907317759, ++STORE, 140011907317760, 140011991179263, ++STORE, 140011856961536, 140011907317759, ++ERASE, 140011856961536, 140011907317759, ++SNULL, 140011974426623, 140011991179263, ++STORE, 140011907317760, 140011974426623, ++STORE, 140011974426624, 140011991179263, ++ERASE, 140011974426624, 140011991179263, ++SNULL, 140011840208895, 140011848568831, ++STORE, 140011773100032, 140011840208895, ++STORE, 140011840208896, 140011848568831, ++ERASE, 140011840208896, 140011848568831, ++SNULL, 140011773235199, 140011840208895, ++STORE, 140011773100032, 140011773235199, ++STORE, 140011773235200, 140011840208895, ++STORE, 140011102011392, 140011320090623, ++SNULL, 140011169120255, 140011320090623, ++STORE, 140011102011392, 140011169120255, ++STORE, 140011169120256, 140011320090623, ++SNULL, 140011169120256, 140011236229119, ++STORE, 140011236229120, 140011320090623, ++STORE, 140011169120256, 140011236229119, ++ERASE, 140011169120256, 140011236229119, ++SNULL, 140011622096896, 140011638882303, ++STORE, 140011638882304, 140011756314623, ++STORE, 140011622096896, 140011638882303, ++ERASE, 140011622096896, 140011638882303, ++SNULL, 140011705991167, 140011756314623, ++STORE, 140011638882304, 140011705991167, ++STORE, 140011705991168, 140011756314623, ++ERASE, 140011705991168, 140011756314623, ++SNULL, 140011571773439, 140011613704191, ++STORE, 140011504664576, 140011571773439, ++STORE, 140011571773440, 140011613704191, ++ERASE, 140011571773440, 140011613704191, ++STORE, 140010967793664, 140011169120255, ++SNULL, 140011034902527, 140011169120255, ++STORE, 140010967793664, 140011034902527, ++STORE, 140011034902528, 140011169120255, ++SNULL, 140011034902528, 140011102011391, ++STORE, 140011102011392, 140011169120255, ++STORE, 140011034902528, 140011102011391, ++ERASE, 140011034902528, 140011102011391, ++STORE, 140010833575936, 140011034902527, ++SNULL, 140011437555711, 140011471093759, ++STORE, 140011370446848, 140011437555711, ++STORE, 140011437555712, 140011471093759, ++ERASE, 140011437555712, 140011471093759, ++SNULL, 140011370582015, 140011437555711, ++STORE, 140011370446848, 140011370582015, ++STORE, 140011370582016, 140011437555711, ++STORE, 140010699358208, 140011034902527, ++SNULL, 140011487883263, 140011504664575, ++STORE, 140011487879168, 140011487883263, ++STORE, 140011487883264, 140011504664575, ++SNULL, 140011345272832, 140011353661439, ++STORE, 140011353661440, 140011362054143, ++STORE, 140011345272832, 140011353661439, ++SNULL, 140011353665535, 140011362054143, ++STORE, 140011353661440, 140011353665535, ++STORE, 140011353665536, 140011362054143, ++SNULL, 140011328487424, 140011336876031, ++STORE, 140011336876032, 140011345268735, ++STORE, 140011328487424, 140011336876031, ++SNULL, 140011336880127, 140011345268735, ++STORE, 140011336876032, 140011336880127, ++STORE, 140011336880128, 140011345268735, ++SNULL, 140011303337983, 140011320090623, ++STORE, 140011236229120, 140011303337983, ++STORE, 140011303337984, 140011320090623, ++ERASE, 140011303337984, 140011320090623, ++SNULL, 140011907452927, 140011974426623, ++STORE, 140011907317760, 140011907452927, ++STORE, 140011907452928, 140011974426623, ++SNULL, 140011102146559, 140011169120255, ++STORE, 140011102011392, 140011102146559, ++STORE, 140011102146560, 140011169120255, ++SNULL, 140011639017471, 140011705991167, ++STORE, 140011638882304, 140011639017471, ++STORE, 140011639017472, 140011705991167, ++SNULL, 140011504799743, 140011571773439, ++STORE, 140011504664576, 140011504799743, ++STORE, 140011504799744, 140011571773439, ++SNULL, 140011613708287, 140011622096895, ++STORE, 140011613704192, 140011613708287, ++STORE, 140011613708288, 140011622096895, ++SNULL, 140010699358208, 140010967793663, ++STORE, 140010967793664, 140011034902527, ++STORE, 140010699358208, 140010967793663, ++SNULL, 140010967928831, 140011034902527, ++STORE, 140010967793664, 140010967928831, ++STORE, 140010967928832, 140011034902527, ++SNULL, 140010900684799, 140010967793663, ++STORE, 140010699358208, 140010900684799, ++STORE, 140010900684800, 140010967793663, ++ERASE, 140010900684800, 140010967793663, ++SNULL, 140010766467071, 140010900684799, ++STORE, 140010699358208, 140010766467071, ++STORE, 140010766467072, 140010900684799, ++SNULL, 140010766467072, 140010833575935, ++STORE, 140010833575936, 140010900684799, ++STORE, 140010766467072, 140010833575935, ++ERASE, 140010766467072, 140010833575935, ++SNULL, 140010699493375, 140010766467071, ++STORE, 140010699358208, 140010699493375, ++STORE, 140010699493376, 140010766467071, ++SNULL, 140011848572927, 140011856961535, ++STORE, 140011848568832, 140011848572927, ++STORE, 140011848572928, 140011856961535, ++STORE, 140011982786560, 140012007964671, ++STORE, 140011898925056, 140011907317759, ++SNULL, 140011898929151, 140011907317759, ++STORE, 140011898925056, 140011898929151, ++STORE, 140011898929152, 140011907317759, ++SNULL, 140011320094719, 140011328483327, ++STORE, 140011320090624, 140011320094719, ++STORE, 140011320094720, 140011328483327, ++STORE, 140011890532352, 140011898925055, ++STORE, 140011882139648, 140011898925055, ++SNULL, 140011882143743, 140011898925055, ++STORE, 140011882139648, 140011882143743, ++STORE, 140011882143744, 140011898925055, ++STORE, 140011873746944, 140011882139647, ++SNULL, 140011873751039, 140011882139647, ++STORE, 140011873746944, 140011873751039, ++STORE, 140011873751040, 140011882139647, ++SNULL, 140011236364287, 140011303337983, ++STORE, 140011236229120, 140011236364287, ++STORE, 140011236364288, 140011303337983, ++SNULL, 140011756318719, 140011773100031, ++STORE, 140011756314624, 140011756318719, ++STORE, 140011756318720, 140011773100031, ++SNULL, 140011756318720, 140011764707327, ++STORE, 140011764707328, 140011773100031, ++STORE, 140011756318720, 140011764707327, ++SNULL, 140011764711423, 140011773100031, ++STORE, 140011764707328, 140011764711423, ++STORE, 140011764711424, 140011773100031, ++SNULL, 140011471097855, 140011487879167, ++STORE, 140011471093760, 140011471097855, ++STORE, 140011471097856, 140011487879167, ++SNULL, 140010833711103, 140010900684799, ++STORE, 140010833575936, 140010833711103, ++STORE, 140010833711104, 140010900684799, ++SNULL, 140011982790655, 140012007964671, ++STORE, 140011982786560, 140011982790655, ++STORE, 140011982790656, 140012007964671, ++STORE, 140011865354240, 140011873746943, ++STORE, 140011848572928, 140011865354239, ++SNULL, 140011848572928, 140011856961535, ++STORE, 140011856961536, 140011865354239, ++STORE, 140011848572928, 140011856961535, ++SNULL, 140011856965631, 140011865354239, ++STORE, 140011856961536, 140011856965631, ++STORE, 140011856965632, 140011865354239, ++STORE, 140011747921920, 140011756314623, ++STORE, 140011739529216, 140011756314623, ++SNULL, 140011471097856, 140011479486463, ++STORE, 140011479486464, 140011487879167, ++STORE, 140011471097856, 140011479486463, ++SNULL, 140011479490559, 140011487879167, ++STORE, 140011479486464, 140011479490559, ++STORE, 140011479490560, 140011487879167, ++STORE, 140011731136512, 140011756314623, ++STORE, 140011722743808, 140011756314623, ++SNULL, 140011982790656, 140011999571967, ++STORE, 140011999571968, 140012007964671, ++STORE, 140011982790656, 140011999571967, ++SNULL, 140011999576063, 140012007964671, ++STORE, 140011999571968, 140011999576063, ++STORE, 140011999576064, 140012007964671, ++STORE, 140011714351104, 140011756314623, ++SNULL, 140011882143744, 140011890532351, ++STORE, 140011890532352, 140011898925055, ++STORE, 140011882143744, 140011890532351, ++SNULL, 140011890536447, 140011898925055, ++STORE, 140011890532352, 140011890536447, ++STORE, 140011890536448, 140011898925055, ++STORE, 140011630489600, 140011638882303, ++STORE, 140011613708288, 140011638882303, ++STORE, 140011605311488, 140011613704191, ++STORE, 140011596918784, 140011613704191, ++STORE, 140011588526080, 140011613704191, ++SNULL, 140011487883264, 140011496271871, ++STORE, 140011496271872, 140011504664575, ++STORE, 140011487883264, 140011496271871, ++SNULL, 140011496275967, 140011504664575, ++STORE, 140011496271872, 140011496275967, ++STORE, 140011496275968, 140011504664575, ++STORE, 140011580133376, 140011613704191, ++SNULL, 140011580137471, 140011613704191, ++STORE, 140011580133376, 140011580137471, ++STORE, 140011580137472, 140011613704191, ++SNULL, 140011982790656, 140011991179263, ++STORE, 140011991179264, 140011999571967, ++STORE, 140011982790656, 140011991179263, ++SNULL, 140011991183359, 140011999571967, ++STORE, 140011991179264, 140011991183359, ++STORE, 140011991183360, 140011999571967, ++SNULL, 140011865358335, 140011873746943, ++STORE, 140011865354240, 140011865358335, ++STORE, 140011865358336, 140011873746943, ++STORE, 140011462701056, 140011471093759, ++SNULL, 140011714351104, 140011739529215, ++STORE, 140011739529216, 140011756314623, ++STORE, 140011714351104, 140011739529215, ++SNULL, 140011739533311, 140011756314623, ++STORE, 140011739529216, 140011739533311, ++STORE, 140011739533312, 140011756314623, ++SNULL, 140011739533312, 140011747921919, ++STORE, 140011747921920, 140011756314623, ++STORE, 140011739533312, 140011747921919, ++SNULL, 140011747926015, 140011756314623, ++STORE, 140011747921920, 140011747926015, ++STORE, 140011747926016, 140011756314623, ++SNULL, 140011613708288, 140011630489599, ++STORE, 140011630489600, 140011638882303, ++STORE, 140011613708288, 140011630489599, ++SNULL, 140011630493695, 140011638882303, ++STORE, 140011630489600, 140011630493695, ++STORE, 140011630493696, 140011638882303, ++SNULL, 140011714351104, 140011722743807, ++STORE, 140011722743808, 140011739529215, ++STORE, 140011714351104, 140011722743807, ++SNULL, 140011722747903, 140011739529215, ++STORE, 140011722743808, 140011722747903, ++STORE, 140011722747904, 140011739529215, ++SNULL, 140011714355199, 140011722743807, ++STORE, 140011714351104, 140011714355199, ++STORE, 140011714355200, 140011722743807, ++SNULL, 140011722747904, 140011731136511, ++STORE, 140011731136512, 140011739529215, ++STORE, 140011722747904, 140011731136511, ++SNULL, 140011731140607, 140011739529215, ++STORE, 140011731136512, 140011731140607, ++STORE, 140011731140608, 140011739529215, ++STORE, 140011454308352, 140011471093759, ++STORE, 140011445915648, 140011471093759, ++SNULL, 140011580137472, 140011588526079, ++STORE, 140011588526080, 140011613704191, ++STORE, 140011580137472, 140011588526079, ++SNULL, 140011588530175, 140011613704191, ++STORE, 140011588526080, 140011588530175, ++STORE, 140011588530176, 140011613704191, ++SNULL, 140011445915648, 140011462701055, ++STORE, 140011462701056, 140011471093759, ++STORE, 140011445915648, 140011462701055, ++SNULL, 140011462705151, 140011471093759, ++STORE, 140011462701056, 140011462705151, ++STORE, 140011462705152, 140011471093759, ++SNULL, 140011588530176, 140011596918783, ++STORE, 140011596918784, 140011613704191, ++STORE, 140011588530176, 140011596918783, ++SNULL, 140011596922879, 140011613704191, ++STORE, 140011596918784, 140011596922879, ++STORE, 140011596922880, 140011613704191, ++SNULL, 140011596922880, 140011605311487, ++STORE, 140011605311488, 140011613704191, ++STORE, 140011596922880, 140011605311487, ++SNULL, 140011605315583, 140011613704191, ++STORE, 140011605311488, 140011605315583, ++STORE, 140011605315584, 140011613704191, ++SNULL, 140011613708288, 140011622096895, ++STORE, 140011622096896, 140011630489599, ++STORE, 140011613708288, 140011622096895, ++SNULL, 140011622100991, 140011630489599, ++STORE, 140011622096896, 140011622100991, ++STORE, 140011622100992, 140011630489599, ++STORE, 140011311697920, 140011320090623, ++STORE, 140011227836416, 140011236229119, ++STORE, 140011219443712, 140011236229119, ++SNULL, 140011219447807, 140011236229119, ++STORE, 140011219443712, 140011219447807, ++STORE, 140011219447808, 140011236229119, ++STORE, 140011211051008, 140011219443711, ++STORE, 140011202658304, 140011219443711, ++SNULL, 140011202662399, 140011219443711, ++STORE, 140011202658304, 140011202662399, ++STORE, 140011202662400, 140011219443711, ++STORE, 140011194265600, 140011202658303, ++STORE, 140011185872896, 140011202658303, ++STORE, 140011177480192, 140011202658303, ++STORE, 140011093618688, 140011102011391, ++SNULL, 140011445915648, 140011454308351, ++STORE, 140011454308352, 140011462701055, ++STORE, 140011445915648, 140011454308351, ++SNULL, 140011454312447, 140011462701055, ++STORE, 140011454308352, 140011454312447, ++STORE, 140011454312448, 140011462701055, ++STORE, 140011085225984, 140011102011391, ++SNULL, 140011085230079, 140011102011391, ++STORE, 140011085225984, 140011085230079, ++STORE, 140011085230080, 140011102011391, ++SNULL, 140011177484287, 140011202658303, ++STORE, 140011177480192, 140011177484287, ++STORE, 140011177484288, 140011202658303, ++SNULL, 140011445919743, 140011454308351, ++STORE, 140011445915648, 140011445919743, ++STORE, 140011445919744, 140011454308351, ++SNULL, 140011177484288, 140011185872895, ++STORE, 140011185872896, 140011202658303, ++STORE, 140011177484288, 140011185872895, ++SNULL, 140011185876991, 140011202658303, ++STORE, 140011185872896, 140011185876991, ++STORE, 140011185876992, 140011202658303, ++STORE, 140011076833280, 140011085225983, ++SNULL, 140011202662400, 140011211051007, ++STORE, 140011211051008, 140011219443711, ++STORE, 140011202662400, 140011211051007, ++SNULL, 140011211055103, 140011219443711, ++STORE, 140011211051008, 140011211055103, ++STORE, 140011211055104, 140011219443711, ++SNULL, 140011185876992, 140011194265599, ++STORE, 140011194265600, 140011202658303, ++STORE, 140011185876992, 140011194265599, ++SNULL, 140011194269695, 140011202658303, ++STORE, 140011194265600, 140011194269695, ++STORE, 140011194269696, 140011202658303, ++STORE, 140011068440576, 140011085225983, ++SNULL, 140011311702015, 140011320090623, ++STORE, 140011311697920, 140011311702015, ++STORE, 140011311702016, 140011320090623, ++STORE, 140011060047872, 140011085225983, ++SNULL, 140011060051967, 140011085225983, ++STORE, 140011060047872, 140011060051967, ++STORE, 140011060051968, 140011085225983, ++STORE, 140011051655168, 140011060047871, ++STORE, 140011043262464, 140011060047871, ++SNULL, 140011043266559, 140011060047871, ++STORE, 140011043262464, 140011043266559, ++STORE, 140011043266560, 140011060047871, ++SNULL, 140011219447808, 140011227836415, ++STORE, 140011227836416, 140011236229119, ++STORE, 140011219447808, 140011227836415, ++SNULL, 140011227840511, 140011236229119, ++STORE, 140011227836416, 140011227840511, ++STORE, 140011227840512, 140011236229119, ++SNULL, 140011085230080, 140011093618687, ++STORE, 140011093618688, 140011102011391, ++STORE, 140011085230080, 140011093618687, ++SNULL, 140011093622783, 140011102011391, ++STORE, 140011093618688, 140011093622783, ++STORE, 140011093622784, 140011102011391, ++STORE, 140010959400960, 140010967793663, ++STORE, 140010951008256, 140010967793663, ++SNULL, 140010951008256, 140010959400959, ++STORE, 140010959400960, 140010967793663, ++STORE, 140010951008256, 140010959400959, ++SNULL, 140010959405055, 140010967793663, ++STORE, 140010959400960, 140010959405055, ++STORE, 140010959405056, 140010967793663, ++STORE, 140010942615552, 140010959400959, ++STORE, 140010934222848, 140010959400959, ++SNULL, 140011060051968, 140011076833279, ++STORE, 140011076833280, 140011085225983, ++STORE, 140011060051968, 140011076833279, ++SNULL, 140011076837375, 140011085225983, ++STORE, 140011076833280, 140011076837375, ++STORE, 140011076837376, 140011085225983, ++SNULL, 140011043266560, 140011051655167, ++STORE, 140011051655168, 140011060047871, ++STORE, 140011043266560, 140011051655167, ++SNULL, 140011051659263, 140011060047871, ++STORE, 140011051655168, 140011051659263, ++STORE, 140011051659264, 140011060047871, ++STORE, 140010925830144, 140010959400959, ++SNULL, 140011060051968, 140011068440575, ++STORE, 140011068440576, 140011076833279, ++STORE, 140011060051968, 140011068440575, ++SNULL, 140011068444671, 140011076833279, ++STORE, 140011068440576, 140011068444671, ++STORE, 140011068444672, 140011076833279, ++STORE, 140010917437440, 140010959400959, ++STORE, 140010909044736, 140010959400959, ++STORE, 140010825183232, 140010833575935, ++SNULL, 140010909044736, 140010942615551, ++STORE, 140010942615552, 140010959400959, ++STORE, 140010909044736, 140010942615551, ++SNULL, 140010942619647, 140010959400959, ++STORE, 140010942615552, 140010942619647, ++STORE, 140010942619648, 140010959400959, ++SNULL, 140010909044736, 140010934222847, ++STORE, 140010934222848, 140010942615551, ++STORE, 140010909044736, 140010934222847, ++SNULL, 140010934226943, 140010942615551, ++STORE, 140010934222848, 140010934226943, ++STORE, 140010934226944, 140010942615551, ++SNULL, 140010909048831, 140010934222847, ++STORE, 140010909044736, 140010909048831, ++STORE, 140010909048832, 140010934222847, ++STORE, 140010816790528, 140010833575935, ++SNULL, 140010816794623, 140010833575935, ++STORE, 140010816790528, 140010816794623, ++STORE, 140010816794624, 140010833575935, ++STORE, 140010808397824, 140010816790527, ++SNULL, 140010942619648, 140010951008255, ++STORE, 140010951008256, 140010959400959, ++STORE, 140010942619648, 140010951008255, ++SNULL, 140010951012351, 140010959400959, ++STORE, 140010951008256, 140010951012351, ++STORE, 140010951012352, 140010959400959, ++STORE, 140010800005120, 140010816790527, ++SNULL, 140010800009215, 140010816790527, ++STORE, 140010800005120, 140010800009215, ++STORE, 140010800009216, 140010816790527, ++SNULL, 140010909048832, 140010925830143, ++STORE, 140010925830144, 140010934222847, ++STORE, 140010909048832, 140010925830143, ++SNULL, 140010925834239, 140010934222847, ++STORE, 140010925830144, 140010925834239, ++STORE, 140010925834240, 140010934222847, ++SNULL, 140010816794624, 140010825183231, ++STORE, 140010825183232, 140010833575935, ++STORE, 140010816794624, 140010825183231, ++SNULL, 140010825187327, 140010833575935, ++STORE, 140010825183232, 140010825187327, ++STORE, 140010825187328, 140010833575935, ++SNULL, 140010909048832, 140010917437439, ++STORE, 140010917437440, 140010925830143, ++STORE, 140010909048832, 140010917437439, ++SNULL, 140010917441535, 140010925830143, ++STORE, 140010917437440, 140010917441535, ++STORE, 140010917441536, 140010925830143, ++SNULL, 140010800009216, 140010808397823, ++STORE, 140010808397824, 140010816790527, ++STORE, 140010800009216, 140010808397823, ++SNULL, 140010808401919, 140010816790527, ++STORE, 140010808397824, 140010808401919, ++STORE, 140010808401920, 140010816790527, ++STORE, 140010791612416, 140010800005119, ++SNULL, 140010791616511, 140010800005119, ++STORE, 140010791612416, 140010791616511, ++STORE, 140010791616512, 140010800005119, ++STORE, 140012547100672, 140012547129343, ++STORE, 140012511506432, 140012513697791, ++SNULL, 140012511506432, 140012511596543, ++STORE, 140012511596544, 140012513697791, ++STORE, 140012511506432, 140012511596543, ++SNULL, 140012513689599, 140012513697791, ++STORE, 140012511596544, 140012513689599, ++STORE, 140012513689600, 140012513697791, ++ERASE, 140012513689600, 140012513697791, ++STORE, 140012513689600, 140012513697791, ++SNULL, 140012513693695, 140012513697791, ++STORE, 140012513689600, 140012513693695, ++STORE, 140012513693696, 140012513697791, ++ERASE, 140012547100672, 140012547129343, ++ERASE, 140011362054144, 140011362058239, ++ERASE, 140011362058240, 140011370446847, ++ERASE, 140011882139648, 140011882143743, ++ERASE, 140011882143744, 140011890532351, ++ERASE, 140011873746944, 140011873751039, ++ERASE, 140011873751040, 140011882139647, ++ERASE, 140011588526080, 140011588530175, ++ERASE, 140011588530176, 140011596918783, ++ERASE, 140011328483328, 140011328487423, ++ERASE, 140011328487424, 140011336876031, ++ERASE, 140011898925056, 140011898929151, ++ERASE, 140011898929152, 140011907317759, ++ERASE, 140011353661440, 140011353665535, ++ERASE, 140011353665536, 140011362054143, ++ERASE, 140011336876032, 140011336880127, ++ERASE, 140011336880128, 140011345268735, ++ERASE, 140011731136512, 140011731140607, ++ERASE, 140011731140608, 140011739529215, ++ERASE, 140011479486464, 140011479490559, ++ERASE, 140011479490560, 140011487879167, ++ERASE, 140011756314624, 140011756318719, ++ERASE, 140011756318720, 140011764707327, ++ERASE, 140011580133376, 140011580137471, ++ERASE, 140011580137472, 140011588526079, ++ERASE, 140011219443712, 140011219447807, ++ERASE, 140011219447808, 140011227836415, ++ERASE, 140011051655168, 140011051659263, ++ERASE, 140011051659264, 140011060047871, ++ERASE, 140011999571968, 140011999576063, ++ERASE, 140011999576064, 140012007964671, ++ERASE, 140011714351104, 140011714355199, ++ERASE, 140011714355200, 140011722743807, ++ERASE, 140011739529216, 140011739533311, ++ERASE, 140011739533312, 140011747921919, ++ERASE, 140011320090624, 140011320094719, ++ERASE, 140011320094720, 140011328483327, ++ERASE, 140011630489600, 140011630493695, ++ERASE, 140011630493696, 140011638882303, ++ERASE, 140011345268736, 140011345272831, ++ERASE, 140011345272832, 140011353661439, ++ERASE, 140011496271872, 140011496275967, ++ERASE, 140011496275968, 140011504664575, ++ERASE, 140011194265600, 140011194269695, ++ERASE, 140011194269696, 140011202658303, ++ERASE, 140011068440576, 140011068444671, ++ERASE, 140011068444672, 140011076833279, ++ERASE, 140010909044736, 140010909048831, ++ERASE, 140010909048832, 140010917437439, ++ERASE, 140011764707328, 140011764711423, ++ERASE, 140011764711424, 140011773100031, ++ERASE, 140011462701056, 140011462705151, ++ERASE, 140011462705152, 140011471093759, ++ERASE, 140011076833280, 140011076837375, ++ERASE, 140011076837376, 140011085225983, ++ERASE, 140011991179264, 140011991183359, ++ERASE, 140011991183360, 140011999571967, ++ERASE, 140011211051008, 140011211055103, ++ERASE, 140011211055104, 140011219443711, ++ERASE, 140010917437440, 140010917441535, ++ERASE, 140010917441536, 140010925830143, ++ERASE, 140011085225984, 140011085230079, ++ERASE, 140011085230080, 140011093618687, ++ERASE, 140011487879168, 140011487883263, ++ERASE, 140011487883264, 140011496271871, ++ERASE, 140011856961536, 140011856965631, ++ERASE, 140011856965632, 140011865354239, ++ERASE, 140011982786560, 140011982790655, ++ERASE, 140011982790656, 140011991179263, ++ERASE, 140011722743808, 140011722747903, ++ERASE, 140011722747904, 140011731136511, ++ERASE, 140011177480192, 140011177484287, ++ERASE, 140011177484288, 140011185872895, ++ERASE, 140011848568832, 140011848572927, ++ERASE, 140011848572928, 140011856961535, ++ERASE, 140011890532352, 140011890536447, ++ERASE, 140011890536448, 140011898925055, ++ERASE, 140011622096896, 140011622100991, ++ERASE, 140011622100992, 140011630489599, ++ERASE, 140011311697920, 140011311702015, ++ERASE, 140011311702016, 140011320090623, ++ERASE, 140011471093760, 140011471097855, ++ERASE, 140011471097856, 140011479486463, ++ERASE, 140011605311488, 140011605315583, ++ERASE, 140011605315584, 140011613704191, ++ERASE, 140010791612416, 140010791616511, ++ERASE, 140010791616512, 140010800005119, ++ERASE, 140010959400960, 140010959405055, ++ERASE, 140010959405056, 140010967793663, ++ERASE, 140011185872896, 140011185876991, ++ERASE, 140011185876992, 140011194265599, ++ERASE, 140011454308352, 140011454312447, ++ERASE, 140011454312448, 140011462701055, ++ERASE, 140011596918784, 140011596922879, ++ERASE, 140011596922880, 140011605311487, ++ERASE, 140011060047872, 140011060051967, ++ERASE, 140011060051968, 140011068440575, ++ERASE, 140010925830144, 140010925834239, ++ERASE, 140010925834240, 140010934222847, ++ERASE, 140011747921920, 140011747926015, ++ERASE, 140011747926016, 140011756314623, ++ERASE, 140011202658304, 140011202662399, ++ERASE, 140011202662400, 140011211051007, ++ERASE, 140010800005120, 140010800009215, ++ERASE, 140010800009216, 140010808397823, ++ERASE, 140011093618688, 140011093622783, ++ERASE, 140011093622784, 140011102011391, ++ERASE, 140010808397824, 140010808401919, ++ERASE, 140010808401920, 140010816790527, ++ERASE, 140012419010560, 140012419014655, ++ERASE, 140012419014656, 140012427403263, ++ERASE, 140010934222848, 140010934226943, ++ERASE, 140010934226944, 140010942615551, ++ERASE, 140010942615552, 140010942619647, ++ERASE, 140010942619648, 140010951008255, ++ERASE, 140011613704192, 140011613708287, ++ERASE, 140011613708288, 140011622096895, ++ERASE, 140011865354240, 140011865358335, ++ERASE, 140011865358336, 140011873746943, ++ERASE, 140012301578240, 140012301582335, ++ERASE, 140012301582336, 140012309970943, ++ERASE, 140012393832448, 140012393836543, ++ERASE, 140012393836544, 140012402225151, ++ERASE, 140012410617856, 140012410621951, ++ERASE, 140012410621952, 140012419010559, ++ERASE, 140012402225152, 140012402229247, ++ERASE, 140012402229248, 140012410617855, ++ERASE, 140012259614720, 140012259618815, ++ERASE, 140012259618816, 140012268007423, ++ERASE, 140012251222016, 140012251226111, ++ERASE, 140012251226112, 140012259614719, ++ERASE, 140012284792832, 140012284796927, ++ERASE, 140012284796928, 140012293185535, ++ERASE, 140011445915648, 140011445919743, ++ERASE, 140011445919744, 140011454308351, ++ERASE, 140010951008256, 140010951012351, ++ERASE, 140010951012352, 140010959400959, ++ERASE, 140011043262464, 140011043266559, ++ERASE, 140011043266560, 140011051655167, ++ERASE, 140010825183232, 140010825187327, ++ERASE, 140010825187328, 140010833575935, ++ERASE, 140012293185536, 140012293189631, ++ERASE, 140012293189632, 140012301578239, ++ERASE, 140012276400128, 140012276404223, ++ERASE, 140012276404224, 140012284792831, ++ERASE, 140012016357376, 140012016361471, ++ERASE, 140012016361472, 140012024750079, ++ERASE, 140012024750080, 140012024754175, ++ERASE, 140012024754176, 140012033142783, ++ERASE, 140011227836416, 140011227840511, ++ERASE, 140011227840512, 140011236229119, ++ERASE, 140010816790528, 140010816794623, ++ERASE, 140010816794624, 140010825183231, ++ERASE, 140012268007424, 140012268011519, ++ERASE, 140012268011520, 140012276400127, ++ERASE, 140012385439744, 140012385443839, ++ERASE, 140012385443840, 140012393832447, ++ERASE, 140012522090496, 140012522094591, ++ERASE, 140012522094592, 140012530483199, ++ERASE, 140012033142784, 140012033146879, ++ERASE, 140012033146880, 140012041535487, ++ }; ++ unsigned long set35[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140730536939520, 140737488351231, ++SNULL, 140730536943615, 140737488351231, ++STORE, 140730536939520, 140730536943615, ++STORE, 140730536808448, 140730536943615, ++STORE, 94245239877632, 94245242130431, ++SNULL, 94245240008703, 94245242130431, ++STORE, 94245239877632, 94245240008703, ++STORE, 94245240008704, 94245242130431, ++ERASE, 94245240008704, 94245242130431, ++STORE, 94245242101760, 94245242109951, ++STORE, 94245242109952, 94245242130431, ++STORE, 140475575263232, 140475577516031, ++SNULL, 140475575406591, 140475577516031, ++STORE, 140475575263232, 140475575406591, ++STORE, 140475575406592, 140475577516031, ++ERASE, 140475575406592, 140475577516031, ++STORE, 140475577503744, 140475577511935, ++STORE, 140475577511936, 140475577516031, ++STORE, 140730538164224, 140730538168319, ++STORE, 140730538151936, 140730538164223, ++STORE, 140475577475072, 140475577503743, ++STORE, 140475577466880, 140475577475071, ++STORE, 140475573047296, 140475575263231, ++SNULL, 140475573047296, 140475573145599, ++STORE, 140475573145600, 140475575263231, ++STORE, 140475573047296, 140475573145599, ++SNULL, 140475575238655, 140475575263231, ++STORE, 140475573145600, 140475575238655, ++STORE, 140475575238656, 140475575263231, ++SNULL, 140475575238656, 140475575246847, ++STORE, 140475575246848, 140475575263231, ++STORE, 140475575238656, 140475575246847, ++ERASE, 140475575238656, 140475575246847, ++STORE, 140475575238656, 140475575246847, ++ERASE, 140475575246848, 140475575263231, ++STORE, 140475575246848, 140475575263231, ++STORE, 140475569250304, 140475573047295, ++SNULL, 140475569250304, 140475570909183, ++STORE, 140475570909184, 140475573047295, ++STORE, 140475569250304, 140475570909183, ++SNULL, 140475573006335, 140475573047295, ++STORE, 140475570909184, 140475573006335, ++STORE, 140475573006336, 140475573047295, ++SNULL, 140475573006336, 140475573030911, ++STORE, 140475573030912, 140475573047295, ++STORE, 140475573006336, 140475573030911, ++ERASE, 140475573006336, 140475573030911, ++STORE, 140475573006336, 140475573030911, ++ERASE, 140475573030912, 140475573047295, ++STORE, 140475573030912, 140475573047295, ++STORE, 140475577458688, 140475577475071, ++SNULL, 140475573022719, 140475573030911, ++STORE, 140475573006336, 140475573022719, ++STORE, 140475573022720, 140475573030911, ++SNULL, 140475575242751, 140475575246847, ++STORE, 140475575238656, 140475575242751, ++STORE, 140475575242752, 140475575246847, ++SNULL, 94245242105855, 94245242109951, ++STORE, 94245242101760, 94245242105855, ++STORE, 94245242105856, 94245242109951, ++SNULL, 140475577507839, 140475577511935, ++STORE, 140475577503744, 140475577507839, ++STORE, 140475577507840, 140475577511935, ++ERASE, 140475577475072, 140475577503743, ++STORE, 94245271216128, 94245271351295, ++STORE, 140475560857600, 140475569250303, ++SNULL, 140475560861695, 140475569250303, ++STORE, 140475560857600, 140475560861695, ++STORE, 140475560861696, 140475569250303, ++STORE, 140475552464896, 140475560857599, ++STORE, 140475418247168, 140475552464895, ++SNULL, 140475418247168, 140475428241407, ++STORE, 140475428241408, 140475552464895, ++STORE, 140475418247168, 140475428241407, ++ERASE, 140475418247168, 140475428241407, ++SNULL, 140475495350271, 140475552464895, ++STORE, 140475428241408, 140475495350271, ++STORE, 140475495350272, 140475552464895, ++ERASE, 140475495350272, 140475552464895, ++SNULL, 140475428376575, 140475495350271, ++STORE, 140475428241408, 140475428376575, ++STORE, 140475428376576, 140475495350271, ++SNULL, 140475552468991, 140475560857599, ++STORE, 140475552464896, 140475552468991, ++STORE, 140475552468992, 140475560857599, ++STORE, 140475544072192, 140475552464895, ++SNULL, 140475544076287, 140475552464895, ++STORE, 140475544072192, 140475544076287, ++STORE, 140475544076288, 140475552464895, ++STORE, 140475535679488, 140475544072191, ++SNULL, 140475535683583, 140475544072191, ++STORE, 140475535679488, 140475535683583, ++STORE, 140475535683584, 140475544072191, ++STORE, 140475527286784, 140475535679487, ++SNULL, 140475527290879, 140475535679487, ++STORE, 140475527286784, 140475527290879, ++STORE, 140475527290880, 140475535679487, ++STORE, 140475518894080, 140475527286783, ++STORE, 140475510501376, 140475527286783, ++STORE, 140475502108672, 140475527286783, ++STORE, 140475419848704, 140475428241407, ++STORE, 140475285630976, 140475419848703, ++SNULL, 140475285630976, 140475294023679, ++STORE, 140475294023680, 140475419848703, ++STORE, 140475285630976, 140475294023679, ++ERASE, 140475285630976, 140475294023679, ++STORE, 140475159805952, 140475419848703, ++STORE, 140475025588224, 140475419848703, ++SNULL, 140475092697087, 140475419848703, ++STORE, 140475025588224, 140475092697087, ++STORE, 140475092697088, 140475419848703, ++SNULL, 140475092697088, 140475159805951, ++STORE, 140475159805952, 140475419848703, ++STORE, 140475092697088, 140475159805951, ++ERASE, 140475092697088, 140475159805951, ++STORE, 140474891370496, 140475092697087, ++SNULL, 140474958479359, 140475092697087, ++STORE, 140474891370496, 140474958479359, ++STORE, 140474958479360, 140475092697087, ++SNULL, 140474958479360, 140475025588223, ++STORE, 140475025588224, 140475092697087, ++STORE, 140474958479360, 140475025588223, ++ERASE, 140474958479360, 140475025588223, ++SNULL, 140475361132543, 140475419848703, ++STORE, 140475159805952, 140475361132543, ++STORE, 140475361132544, 140475419848703, ++ERASE, 140475361132544, 140475419848703, ++SNULL, 140475159805952, 140475294023679, ++STORE, 140475294023680, 140475361132543, ++STORE, 140475159805952, 140475294023679, ++SNULL, 140475294158847, 140475361132543, ++STORE, 140475294023680, 140475294158847, ++STORE, 140475294158848, 140475361132543, ++SNULL, 140475226914815, 140475294023679, ++STORE, 140475159805952, 140475226914815, ++STORE, 140475226914816, 140475294023679, ++ERASE, 140475226914816, 140475294023679, ++SNULL, 140475025723391, 140475092697087, ++STORE, 140475025588224, 140475025723391, ++STORE, 140475025723392, 140475092697087, ++SNULL, 140475159941119, 140475226914815, ++STORE, 140475159805952, 140475159941119, ++STORE, 140475159941120, 140475226914815, ++SNULL, 140474891505663, 140474958479359, ++STORE, 140474891370496, 140474891505663, ++STORE, 140474891505664, 140474958479359, ++SNULL, 140475502108672, 140475518894079, ++STORE, 140475518894080, 140475527286783, ++STORE, 140475502108672, 140475518894079, ++SNULL, 140475518898175, 140475527286783, ++STORE, 140475518894080, 140475518898175, ++STORE, 140475518898176, 140475527286783, ++STORE, 140475411456000, 140475428241407, ++SNULL, 140475502112767, 140475518894079, ++STORE, 140475502108672, 140475502112767, ++STORE, 140475502112768, 140475518894079, ++SNULL, 140475411460095, 140475428241407, ++STORE, 140475411456000, 140475411460095, ++STORE, 140475411460096, 140475428241407, ++SNULL, 140475411460096, 140475419848703, ++STORE, 140475419848704, 140475428241407, ++STORE, 140475411460096, 140475419848703, ++SNULL, 140475419852799, 140475428241407, ++STORE, 140475419848704, 140475419852799, ++STORE, 140475419852800, 140475428241407, ++STORE, 140475403063296, 140475411455999, ++SNULL, 140475502112768, 140475510501375, ++STORE, 140475510501376, 140475518894079, ++STORE, 140475502112768, 140475510501375, ++SNULL, 140475510505471, 140475518894079, ++STORE, 140475510501376, 140475510505471, ++STORE, 140475510505472, 140475518894079, ++SNULL, 140475403067391, 140475411455999, ++STORE, 140475403063296, 140475403067391, ++STORE, 140475403067392, 140475411455999, ++STORE, 140475394670592, 140475403063295, ++SNULL, 140475394674687, 140475403063295, ++STORE, 140475394670592, 140475394674687, ++STORE, 140475394674688, 140475403063295, ++STORE, 140475386277888, 140475394670591, ++STORE, 140475377885184, 140475394670591, ++STORE, 140475369492480, 140475394670591, ++SNULL, 140475369496575, 140475394670591, ++STORE, 140475369492480, 140475369496575, ++STORE, 140475369496576, 140475394670591, ++SNULL, 140475369496576, 140475377885183, ++STORE, 140475377885184, 140475394670591, ++STORE, 140475369496576, 140475377885183, ++SNULL, 140475377889279, 140475394670591, ++STORE, 140475377885184, 140475377889279, ++STORE, 140475377889280, 140475394670591, ++STORE, 140475285630976, 140475294023679, ++SNULL, 140475377889280, 140475386277887, ++STORE, 140475386277888, 140475394670591, ++STORE, 140475377889280, 140475386277887, ++SNULL, 140475386281983, 140475394670591, ++STORE, 140475386277888, 140475386281983, ++STORE, 140475386281984, 140475394670591, ++SNULL, 140475285635071, 140475294023679, ++STORE, 140475285630976, 140475285635071, ++STORE, 140475285635072, 140475294023679, ++STORE, 140475277238272, 140475285630975, ++STORE, 140475268845568, 140475285630975, ++SNULL, 140475268845568, 140475277238271, ++STORE, 140475277238272, 140475285630975, ++STORE, 140475268845568, 140475277238271, ++SNULL, 140475277242367, 140475285630975, ++STORE, 140475277238272, 140475277242367, ++STORE, 140475277242368, 140475285630975, ++STORE, 140475260452864, 140475277238271, ++SNULL, 140475260452864, 140475268845567, ++STORE, 140475268845568, 140475277238271, ++STORE, 140475260452864, 140475268845567, ++SNULL, 140475268849663, 140475277238271, ++STORE, 140475268845568, 140475268849663, ++STORE, 140475268849664, 140475277238271, ++SNULL, 140475260456959, 140475268845567, ++STORE, 140475260452864, 140475260456959, ++STORE, 140475260456960, 140475268845567, ++STORE, 140475252060160, 140475260452863, ++SNULL, 140475252064255, 140475260452863, ++STORE, 140475252060160, 140475252064255, ++STORE, 140475252064256, 140475260452863, ++STORE, 140475243667456, 140475252060159, ++SNULL, 140475243671551, 140475252060159, ++STORE, 140475243667456, 140475243671551, ++STORE, 140475243671552, 140475252060159, ++STORE, 140475235274752, 140475243667455, ++STORE, 140475151413248, 140475159805951, ++STORE, 140474891505664, 140475025588223, ++STORE, 140475143020544, 140475159805951, ++SNULL, 140474891505664, 140474958479359, ++STORE, 140474958479360, 140475025588223, ++STORE, 140474891505664, 140474958479359, ++SNULL, 140474958614527, 140475025588223, ++STORE, 140474958479360, 140474958614527, ++STORE, 140474958614528, 140475025588223, ++STORE, 140474824261632, 140474891370495, ++SNULL, 140474824396799, 140474891370495, ++STORE, 140474824261632, 140474824396799, ++STORE, 140474824396800, 140474891370495, ++STORE, 140475134627840, 140475159805951, ++STORE, 140474690043904, 140474824261631, ++STORE, 140475126235136, 140475159805951, ++STORE, 140475117842432, 140475159805951, ++STORE, 140474622935040, 140474824261631, ++STORE, 140475109449728, 140475159805951, ++STORE, 140474488717312, 140474824261631, ++STORE, 140475101057024, 140475159805951, ++STORE, 140474480324608, 140474488717311, ++STORE, 140474413215744, 140474480324607, ++STORE, 140474404823040, 140474413215743, ++ERASE, 140474413215744, 140474480324607, ++STORE, 140474471931904, 140474488717311, ++STORE, 140474270605312, 140474404823039, ++SNULL, 140475101057024, 140475126235135, ++STORE, 140475126235136, 140475159805951, ++STORE, 140475101057024, 140475126235135, ++SNULL, 140475126239231, 140475159805951, ++STORE, 140475126235136, 140475126239231, ++STORE, 140475126239232, 140475159805951, ++STORE, 140474463539200, 140474488717311, ++STORE, 140474455146496, 140474488717311, ++SNULL, 140474455150591, 140474488717311, ++STORE, 140474455146496, 140474455150591, ++STORE, 140474455150592, 140474488717311, ++STORE, 140474446753792, 140474455146495, ++SNULL, 140474446757887, 140474455146495, ++STORE, 140474446753792, 140474446757887, ++STORE, 140474446757888, 140474455146495, ++STORE, 140474438361088, 140474446753791, ++STORE, 140474429968384, 140474446753791, ++SNULL, 140474429972479, 140474446753791, ++STORE, 140474429968384, 140474429972479, ++STORE, 140474429972480, 140474446753791, ++SNULL, 140475235278847, 140475243667455, ++STORE, 140475235274752, 140475235278847, ++STORE, 140475235278848, 140475243667455, ++SNULL, 140474757152767, 140474824261631, ++STORE, 140474488717312, 140474757152767, ++STORE, 140474757152768, 140474824261631, ++ERASE, 140474757152768, 140474824261631, ++SNULL, 140474488717312, 140474690043903, ++STORE, 140474690043904, 140474757152767, ++STORE, 140474488717312, 140474690043903, ++SNULL, 140474690179071, 140474757152767, ++STORE, 140474690043904, 140474690179071, ++STORE, 140474690179072, 140474757152767, ++SNULL, 140474488717312, 140474622935039, ++STORE, 140474622935040, 140474690043903, ++STORE, 140474488717312, 140474622935039, ++SNULL, 140474623070207, 140474690043903, ++STORE, 140474622935040, 140474623070207, ++STORE, 140474623070208, 140474690043903, ++SNULL, 140475101057024, 140475117842431, ++STORE, 140475117842432, 140475126235135, ++STORE, 140475101057024, 140475117842431, ++SNULL, 140475117846527, 140475126235135, ++STORE, 140475117842432, 140475117846527, ++STORE, 140475117846528, 140475126235135, ++SNULL, 140474555826175, 140474622935039, ++STORE, 140474488717312, 140474555826175, ++STORE, 140474555826176, 140474622935039, ++ERASE, 140474555826176, 140474622935039, ++STORE, 140474136387584, 140474404823039, ++SNULL, 140474136387584, 140474153172991, ++STORE, 140474153172992, 140474404823039, ++STORE, 140474136387584, 140474153172991, ++ERASE, 140474136387584, 140474153172991, ++STORE, 140474018955264, 140474404823039, ++STORE, 140473884737536, 140474404823039, ++SNULL, 140474086064127, 140474404823039, ++STORE, 140473884737536, 140474086064127, ++STORE, 140474086064128, 140474404823039, ++SNULL, 140474086064128, 140474153172991, ++STORE, 140474153172992, 140474404823039, ++STORE, 140474086064128, 140474153172991, ++ERASE, 140474086064128, 140474153172991, ++STORE, 140473750519808, 140474086064127, ++SNULL, 140473817628671, 140474086064127, ++STORE, 140473750519808, 140473817628671, ++STORE, 140473817628672, 140474086064127, ++SNULL, 140473817628672, 140473884737535, ++STORE, 140473884737536, 140474086064127, ++STORE, 140473817628672, 140473884737535, ++ERASE, 140473817628672, 140473884737535, ++SNULL, 140475126239232, 140475151413247, ++STORE, 140475151413248, 140475159805951, ++STORE, 140475126239232, 140475151413247, ++SNULL, 140475151417343, 140475159805951, ++STORE, 140475151413248, 140475151417343, ++STORE, 140475151417344, 140475159805951, ++SNULL, 140474270605311, 140474404823039, ++STORE, 140474153172992, 140474270605311, ++STORE, 140474270605312, 140474404823039, ++SNULL, 140474270605312, 140474287390719, ++STORE, 140474287390720, 140474404823039, ++STORE, 140474270605312, 140474287390719, ++ERASE, 140474270605312, 140474287390719, ++SNULL, 140474429972480, 140474438361087, ++STORE, 140474438361088, 140474446753791, ++STORE, 140474429972480, 140474438361087, ++SNULL, 140474438365183, 140474446753791, ++STORE, 140474438361088, 140474438365183, ++STORE, 140474438365184, 140474446753791, ++STORE, 140474815868928, 140474824261631, ++SNULL, 140474815873023, 140474824261631, ++STORE, 140474815868928, 140474815873023, ++STORE, 140474815873024, 140474824261631, ++SNULL, 140474220281855, 140474270605311, ++STORE, 140474153172992, 140474220281855, ++STORE, 140474220281856, 140474270605311, ++ERASE, 140474220281856, 140474270605311, ++SNULL, 140474488852479, 140474555826175, ++STORE, 140474488717312, 140474488852479, ++STORE, 140474488852480, 140474555826175, ++SNULL, 140475101057024, 140475109449727, ++STORE, 140475109449728, 140475117842431, ++STORE, 140475101057024, 140475109449727, ++SNULL, 140475109453823, 140475117842431, ++STORE, 140475109449728, 140475109453823, ++STORE, 140475109453824, 140475117842431, ++SNULL, 140473951846399, 140474086064127, ++STORE, 140473884737536, 140473951846399, ++STORE, 140473951846400, 140474086064127, ++SNULL, 140473951846400, 140474018955263, ++STORE, 140474018955264, 140474086064127, ++STORE, 140473951846400, 140474018955263, ++ERASE, 140473951846400, 140474018955263, ++SNULL, 140473884872703, 140473951846399, ++STORE, 140473884737536, 140473884872703, ++STORE, 140473884872704, 140473951846399, ++SNULL, 140474019090431, 140474086064127, ++STORE, 140474018955264, 140474019090431, ++STORE, 140474019090432, 140474086064127, ++SNULL, 140473750654975, 140473817628671, ++STORE, 140473750519808, 140473750654975, ++STORE, 140473750654976, 140473817628671, ++SNULL, 140474455150592, 140474463539199, ++STORE, 140474463539200, 140474488717311, ++STORE, 140474455150592, 140474463539199, ++SNULL, 140474463543295, 140474488717311, ++STORE, 140474463539200, 140474463543295, ++STORE, 140474463543296, 140474488717311, ++STORE, 140474807476224, 140474815868927, ++SNULL, 140474463543296, 140474471931903, ++STORE, 140474471931904, 140474488717311, ++STORE, 140474463543296, 140474471931903, ++SNULL, 140474471935999, 140474488717311, ++STORE, 140474471931904, 140474471935999, ++STORE, 140474471936000, 140474488717311, ++STORE, 140474799083520, 140474815868927, ++STORE, 140474790690816, 140474815868927, ++SNULL, 140474790690816, 140474799083519, ++STORE, 140474799083520, 140474815868927, ++STORE, 140474790690816, 140474799083519, ++SNULL, 140474799087615, 140474815868927, ++STORE, 140474799083520, 140474799087615, ++STORE, 140474799087616, 140474815868927, ++SNULL, 140474354499583, 140474404823039, ++STORE, 140474287390720, 140474354499583, ++STORE, 140474354499584, 140474404823039, ++ERASE, 140474354499584, 140474404823039, ++SNULL, 140474287525887, 140474354499583, ++STORE, 140474287390720, 140474287525887, ++STORE, 140474287525888, 140474354499583, ++STORE, 140474782298112, 140474799083519, ++STORE, 140474773905408, 140474799083519, ++SNULL, 140474773909503, 140474799083519, ++STORE, 140474773905408, 140474773909503, ++STORE, 140474773909504, 140474799083519, ++SNULL, 140475126239232, 140475134627839, ++STORE, 140475134627840, 140475151413247, ++STORE, 140475126239232, 140475134627839, ++SNULL, 140475134631935, 140475151413247, ++STORE, 140475134627840, 140475134631935, ++STORE, 140475134631936, 140475151413247, ++STORE, 140474765512704, 140474773905407, ++STORE, 140474614542336, 140474622935039, ++SNULL, 140474153308159, 140474220281855, ++STORE, 140474153172992, 140474153308159, ++STORE, 140474153308160, 140474220281855, ++SNULL, 140474404827135, 140474413215743, ++STORE, 140474404823040, 140474404827135, ++STORE, 140474404827136, 140474413215743, ++STORE, 140474606149632, 140474622935039, ++SNULL, 140474606153727, 140474622935039, ++STORE, 140474606149632, 140474606153727, ++STORE, 140474606153728, 140474622935039, ++STORE, 140474597756928, 140474606149631, ++SNULL, 140474597761023, 140474606149631, ++STORE, 140474597756928, 140474597761023, ++STORE, 140474597761024, 140474606149631, ++SNULL, 140475134631936, 140475143020543, ++STORE, 140475143020544, 140475151413247, ++STORE, 140475134631936, 140475143020543, ++SNULL, 140475143024639, 140475151413247, ++STORE, 140475143020544, 140475143024639, ++STORE, 140475143024640, 140475151413247, ++STORE, 140474589364224, 140474597756927, ++SNULL, 140474606153728, 140474614542335, ++STORE, 140474614542336, 140474622935039, ++STORE, 140474606153728, 140474614542335, ++SNULL, 140474614546431, 140474622935039, ++STORE, 140474614542336, 140474614546431, ++STORE, 140474614546432, 140474622935039, ++SNULL, 140474765516799, 140474773905407, ++STORE, 140474765512704, 140474765516799, ++STORE, 140474765516800, 140474773905407, ++STORE, 140474580971520, 140474597756927, ++SNULL, 140474773909504, 140474782298111, ++STORE, 140474782298112, 140474799083519, ++STORE, 140474773909504, 140474782298111, ++SNULL, 140474782302207, 140474799083519, ++STORE, 140474782298112, 140474782302207, ++STORE, 140474782302208, 140474799083519, ++SNULL, 140474471936000, 140474480324607, ++STORE, 140474480324608, 140474488717311, ++STORE, 140474471936000, 140474480324607, ++SNULL, 140474480328703, 140474488717311, ++STORE, 140474480324608, 140474480328703, ++STORE, 140474480328704, 140474488717311, ++STORE, 140474572578816, 140474597756927, ++SNULL, 140474572582911, 140474597756927, ++STORE, 140474572578816, 140474572582911, ++STORE, 140474572582912, 140474597756927, ++SNULL, 140474782302208, 140474790690815, ++STORE, 140474790690816, 140474799083519, ++STORE, 140474782302208, 140474790690815, ++SNULL, 140474790694911, 140474799083519, ++STORE, 140474790690816, 140474790694911, ++STORE, 140474790694912, 140474799083519, ++STORE, 140474564186112, 140474572578815, ++STORE, 140474421575680, 140474429968383, ++STORE, 140474396430336, 140474404823039, ++SNULL, 140474396434431, 140474404823039, ++STORE, 140474396430336, 140474396434431, ++STORE, 140474396434432, 140474404823039, ++STORE, 140474388037632, 140474396430335, ++SNULL, 140474799087616, 140474807476223, ++STORE, 140474807476224, 140474815868927, ++STORE, 140474799087616, 140474807476223, ++SNULL, 140474807480319, 140474815868927, ++STORE, 140474807476224, 140474807480319, ++STORE, 140474807480320, 140474815868927, ++SNULL, 140475101061119, 140475109449727, ++STORE, 140475101057024, 140475101061119, ++STORE, 140475101061120, 140475109449727, ++STORE, 140474379644928, 140474396430335, ++SNULL, 140474572582912, 140474589364223, ++STORE, 140474589364224, 140474597756927, ++STORE, 140474572582912, 140474589364223, ++SNULL, 140474589368319, 140474597756927, ++STORE, 140474589364224, 140474589368319, ++STORE, 140474589368320, 140474597756927, ++STORE, 140474371252224, 140474396430335, ++STORE, 140474362859520, 140474396430335, ++STORE, 140474278998016, 140474287390719, ++STORE, 140474270605312, 140474287390719, ++STORE, 140474262212608, 140474287390719, ++SNULL, 140474262216703, 140474287390719, ++STORE, 140474262212608, 140474262216703, ++STORE, 140474262216704, 140474287390719, ++STORE, 140474253819904, 140474262212607, ++SNULL, 140474253823999, 140474262212607, ++STORE, 140474253819904, 140474253823999, ++STORE, 140474253824000, 140474262212607, ++SNULL, 140474362859520, 140474388037631, ++STORE, 140474388037632, 140474396430335, ++STORE, 140474362859520, 140474388037631, ++SNULL, 140474388041727, 140474396430335, ++STORE, 140474388037632, 140474388041727, ++STORE, 140474388041728, 140474396430335, ++SNULL, 140474362859520, 140474379644927, ++STORE, 140474379644928, 140474388037631, ++STORE, 140474362859520, 140474379644927, ++SNULL, 140474379649023, 140474388037631, ++STORE, 140474379644928, 140474379649023, ++STORE, 140474379649024, 140474388037631, ++STORE, 140474245427200, 140474253819903, ++STORE, 140474237034496, 140474253819903, ++STORE, 140474228641792, 140474253819903, ++STORE, 140474144780288, 140474153172991, ++SNULL, 140474228645887, 140474253819903, ++STORE, 140474228641792, 140474228645887, ++STORE, 140474228645888, 140474253819903, ++SNULL, 140474564190207, 140474572578815, ++STORE, 140474564186112, 140474564190207, ++STORE, 140474564190208, 140474572578815, ++STORE, 140474136387584, 140474153172991, ++SNULL, 140474362859520, 140474371252223, ++STORE, 140474371252224, 140474379644927, ++STORE, 140474362859520, 140474371252223, ++SNULL, 140474371256319, 140474379644927, ++STORE, 140474371252224, 140474371256319, ++STORE, 140474371256320, 140474379644927, ++STORE, 140474127994880, 140474153172991, ++STORE, 140474119602176, 140474153172991, ++SNULL, 140474421579775, 140474429968383, ++STORE, 140474421575680, 140474421579775, ++STORE, 140474421579776, 140474429968383, ++STORE, 140474111209472, 140474153172991, ++SNULL, 140474111213567, 140474153172991, ++STORE, 140474111209472, 140474111213567, ++STORE, 140474111213568, 140474153172991, ++SNULL, 140474262216704, 140474270605311, ++STORE, 140474270605312, 140474287390719, ++STORE, 140474262216704, 140474270605311, ++SNULL, 140474270609407, 140474287390719, ++STORE, 140474270605312, 140474270609407, ++STORE, 140474270609408, 140474287390719, ++STORE, 140474102816768, 140474111209471, ++SNULL, 140474102820863, 140474111209471, ++STORE, 140474102816768, 140474102820863, ++STORE, 140474102820864, 140474111209471, ++SNULL, 140474270609408, 140474278998015, ++STORE, 140474278998016, 140474287390719, ++STORE, 140474270609408, 140474278998015, ++SNULL, 140474279002111, 140474287390719, ++STORE, 140474278998016, 140474279002111, ++STORE, 140474279002112, 140474287390719, ++STORE, 140474094424064, 140474102816767, ++SNULL, 140474572582912, 140474580971519, ++STORE, 140474580971520, 140474589364223, ++STORE, 140474572582912, 140474580971519, ++SNULL, 140474580975615, 140474589364223, ++STORE, 140474580971520, 140474580975615, ++STORE, 140474580975616, 140474589364223, ++SNULL, 140474362863615, 140474371252223, ++STORE, 140474362859520, 140474362863615, ++STORE, 140474362863616, 140474371252223, ++STORE, 140474010562560, 140474018955263, ++SNULL, 140474228645888, 140474245427199, ++STORE, 140474245427200, 140474253819903, ++STORE, 140474228645888, 140474245427199, ++SNULL, 140474245431295, 140474253819903, ++STORE, 140474245427200, 140474245431295, ++STORE, 140474245431296, 140474253819903, ++SNULL, 140474111213568, 140474136387583, ++STORE, 140474136387584, 140474153172991, ++STORE, 140474111213568, 140474136387583, ++SNULL, 140474136391679, 140474153172991, ++STORE, 140474136387584, 140474136391679, ++STORE, 140474136391680, 140474153172991, ++STORE, 140474002169856, 140474018955263, ++STORE, 140473993777152, 140474018955263, ++SNULL, 140474111213568, 140474127994879, ++STORE, 140474127994880, 140474136387583, ++STORE, 140474111213568, 140474127994879, ++SNULL, 140474127998975, 140474136387583, ++STORE, 140474127994880, 140474127998975, ++STORE, 140474127998976, 140474136387583, ++SNULL, 140474228645888, 140474237034495, ++STORE, 140474237034496, 140474245427199, ++STORE, 140474228645888, 140474237034495, ++SNULL, 140474237038591, 140474245427199, ++STORE, 140474237034496, 140474237038591, ++STORE, 140474237038592, 140474245427199, ++SNULL, 140474136391680, 140474144780287, ++STORE, 140474144780288, 140474153172991, ++STORE, 140474136391680, 140474144780287, ++SNULL, 140474144784383, 140474153172991, ++STORE, 140474144780288, 140474144784383, ++STORE, 140474144784384, 140474153172991, ++STORE, 140473985384448, 140474018955263, ++STORE, 140473976991744, 140474018955263, ++STORE, 140473968599040, 140474018955263, ++SNULL, 140473968603135, 140474018955263, ++STORE, 140473968599040, 140473968603135, ++STORE, 140473968603136, 140474018955263, ++SNULL, 140474111213568, 140474119602175, ++STORE, 140474119602176, 140474127994879, ++STORE, 140474111213568, 140474119602175, ++SNULL, 140474119606271, 140474127994879, ++STORE, 140474119602176, 140474119606271, ++STORE, 140474119606272, 140474127994879, ++STORE, 140473960206336, 140473968599039, ++SNULL, 140474094428159, 140474102816767, ++STORE, 140474094424064, 140474094428159, ++STORE, 140474094428160, 140474102816767, ++STORE, 140473876344832, 140473884737535, ++STORE, 140473867952128, 140473884737535, ++STORE, 140473859559424, 140473884737535, ++SNULL, 140473859563519, 140473884737535, ++STORE, 140473859559424, 140473859563519, ++STORE, 140473859563520, 140473884737535, ++SNULL, 140473968603136, 140473993777151, ++STORE, 140473993777152, 140474018955263, ++STORE, 140473968603136, 140473993777151, ++SNULL, 140473993781247, 140474018955263, ++STORE, 140473993777152, 140473993781247, ++STORE, 140473993781248, 140474018955263, ++SNULL, 140473960210431, 140473968599039, ++STORE, 140473960206336, 140473960210431, ++STORE, 140473960210432, 140473968599039, ++SNULL, 140473993781248, 140474010562559, ++STORE, 140474010562560, 140474018955263, ++STORE, 140473993781248, 140474010562559, ++SNULL, 140474010566655, 140474018955263, ++STORE, 140474010562560, 140474010566655, ++STORE, 140474010566656, 140474018955263, ++SNULL, 140473968603136, 140473985384447, ++STORE, 140473985384448, 140473993777151, ++STORE, 140473968603136, 140473985384447, ++SNULL, 140473985388543, 140473993777151, ++STORE, 140473985384448, 140473985388543, ++STORE, 140473985388544, 140473993777151, ++SNULL, 140473993781248, 140474002169855, ++STORE, 140474002169856, 140474010562559, ++STORE, 140473993781248, 140474002169855, ++SNULL, 140474002173951, 140474010562559, ++STORE, 140474002169856, 140474002173951, ++STORE, 140474002173952, 140474010562559, ++STORE, 140473851166720, 140473859559423, ++SNULL, 140473851170815, 140473859559423, ++STORE, 140473851166720, 140473851170815, ++STORE, 140473851170816, 140473859559423, ++SNULL, 140473968603136, 140473976991743, ++STORE, 140473976991744, 140473985384447, ++STORE, 140473968603136, 140473976991743, ++SNULL, 140473976995839, 140473985384447, ++STORE, 140473976991744, 140473976995839, ++STORE, 140473976995840, 140473985384447, ++STORE, 140473842774016, 140473851166719, ++SNULL, 140473859563520, 140473867952127, ++STORE, 140473867952128, 140473884737535, ++STORE, 140473859563520, 140473867952127, ++SNULL, 140473867956223, 140473884737535, ++STORE, 140473867952128, 140473867956223, ++STORE, 140473867956224, 140473884737535, ++SNULL, 140473867956224, 140473876344831, ++STORE, 140473876344832, 140473884737535, ++STORE, 140473867956224, 140473876344831, ++SNULL, 140473876348927, 140473884737535, ++STORE, 140473876344832, 140473876348927, ++STORE, 140473876348928, 140473884737535, ++STORE, 140473834381312, 140473851166719, ++SNULL, 140473834385407, 140473851166719, ++STORE, 140473834381312, 140473834385407, ++STORE, 140473834385408, 140473851166719, ++SNULL, 140473834385408, 140473842774015, ++STORE, 140473842774016, 140473851166719, ++STORE, 140473834385408, 140473842774015, ++SNULL, 140473842778111, 140473851166719, ++STORE, 140473842774016, 140473842778111, ++STORE, 140473842778112, 140473851166719, ++STORE, 140473825988608, 140473834381311, ++SNULL, 140473825992703, 140473834381311, ++STORE, 140473825988608, 140473825992703, ++STORE, 140473825992704, 140473834381311, ++STORE, 140475577475072, 140475577503743, ++STORE, 140475499917312, 140475502108671, ++SNULL, 140475499917312, 140475500007423, ++STORE, 140475500007424, 140475502108671, ++STORE, 140475499917312, 140475500007423, ++SNULL, 140475502100479, 140475502108671, ++STORE, 140475500007424, 140475502100479, ++STORE, 140475502100480, 140475502108671, ++ERASE, 140475502100480, 140475502108671, ++STORE, 140475502100480, 140475502108671, ++SNULL, 140475502104575, 140475502108671, ++STORE, 140475502100480, 140475502104575, ++STORE, 140475502104576, 140475502108671, ++ERASE, 140475577475072, 140475577503743, ++ERASE, 140475235274752, 140475235278847, ++ERASE, 140475235278848, 140475243667455, ++ERASE, 140474815868928, 140474815873023, ++ERASE, 140474815873024, 140474824261631, ++ERASE, 140474606149632, 140474606153727, ++ERASE, 140474606153728, 140474614542335, ++ERASE, 140474270605312, 140474270609407, ++ERASE, 140474270609408, 140474278998015, ++ERASE, 140474438361088, 140474438365183, ++ERASE, 140474438365184, 140474446753791, ++ERASE, 140474597756928, 140474597761023, ++ERASE, 140474597761024, 140474606149631, ++ERASE, 140475126235136, 140475126239231, ++ERASE, 140475126239232, 140475134627839, ++ERASE, 140474463539200, 140474463543295, ++ERASE, 140474463543296, 140474471931903, ++ERASE, 140474388037632, 140474388041727, ++ERASE, 140474388041728, 140474396430335, ++ERASE, 140474404823040, 140474404827135, ++ERASE, 140474404827136, 140474413215743, ++ERASE, 140474278998016, 140474279002111, ++ERASE, 140474279002112, 140474287390719, ++ERASE, 140474094424064, 140474094428159, ++ERASE, 140474094428160, 140474102816767, ++ERASE, 140473867952128, 140473867956223, ++ERASE, 140473867956224, 140473876344831, ++ERASE, 140475151413248, 140475151417343, ++ERASE, 140475151417344, 140475159805951, ++ERASE, 140474455146496, 140474455150591, ++ERASE, 140474455150592, 140474463539199, ++ERASE, 140474807476224, 140474807480319, ++ERASE, 140474807480320, 140474815868927, ++ERASE, 140475117842432, 140475117846527, ++ERASE, 140475117846528, 140475126235135, ++ERASE, 140474446753792, 140474446757887, ++ERASE, 140474446757888, 140474455146495, ++ERASE, 140474429968384, 140474429972479, ++ERASE, 140474429972480, 140474438361087, ++ERASE, 140474782298112, 140474782302207, ++ERASE, 140474782302208, 140474790690815, ++ERASE, 140474136387584, 140474136391679, ++ERASE, 140474136391680, 140474144780287, ++ERASE, 140474002169856, 140474002173951, ++ERASE, 140474002173952, 140474010562559, ++ERASE, 140475134627840, 140475134631935, ++ERASE, 140475134631936, 140475143020543, ++ERASE, 140474471931904, 140474471935999, ++ERASE, 140474471936000, 140474480324607, ++ERASE, 140474396430336, 140474396434431, ++ERASE, 140474396434432, 140474404823039, ++ }; ++ unsigned long set36[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140723893125120, 140737488351231, ++SNULL, 140723893129215, 140737488351231, ++STORE, 140723893125120, 140723893129215, ++STORE, 140723892994048, 140723893129215, ++STORE, 94076829786112, 94076832038911, ++SNULL, 94076829917183, 94076832038911, ++STORE, 94076829786112, 94076829917183, ++STORE, 94076829917184, 94076832038911, ++ERASE, 94076829917184, 94076832038911, ++STORE, 94076832010240, 94076832018431, ++STORE, 94076832018432, 94076832038911, ++STORE, 140122444345344, 140122446598143, ++SNULL, 140122444488703, 140122446598143, ++STORE, 140122444345344, 140122444488703, ++STORE, 140122444488704, 140122446598143, ++ERASE, 140122444488704, 140122446598143, ++STORE, 140122446585856, 140122446594047, ++STORE, 140122446594048, 140122446598143, ++STORE, 140723893538816, 140723893542911, ++STORE, 140723893526528, 140723893538815, ++STORE, 140122446557184, 140122446585855, ++STORE, 140122446548992, 140122446557183, ++STORE, 140122442129408, 140122444345343, ++SNULL, 140122442129408, 140122442227711, ++STORE, 140122442227712, 140122444345343, ++STORE, 140122442129408, 140122442227711, ++SNULL, 140122444320767, 140122444345343, ++STORE, 140122442227712, 140122444320767, ++STORE, 140122444320768, 140122444345343, ++SNULL, 140122444320768, 140122444328959, ++STORE, 140122444328960, 140122444345343, ++STORE, 140122444320768, 140122444328959, ++ERASE, 140122444320768, 140122444328959, ++STORE, 140122444320768, 140122444328959, ++ERASE, 140122444328960, 140122444345343, ++STORE, 140122444328960, 140122444345343, ++STORE, 140122438332416, 140122442129407, ++SNULL, 140122438332416, 140122439991295, ++STORE, 140122439991296, 140122442129407, ++STORE, 140122438332416, 140122439991295, ++SNULL, 140122442088447, 140122442129407, ++STORE, 140122439991296, 140122442088447, ++STORE, 140122442088448, 140122442129407, ++SNULL, 140122442088448, 140122442113023, ++STORE, 140122442113024, 140122442129407, ++STORE, 140122442088448, 140122442113023, ++ERASE, 140122442088448, 140122442113023, ++STORE, 140122442088448, 140122442113023, ++ERASE, 140122442113024, 140122442129407, ++STORE, 140122442113024, 140122442129407, ++STORE, 140122446540800, 140122446557183, ++SNULL, 140122442104831, 140122442113023, ++STORE, 140122442088448, 140122442104831, ++STORE, 140122442104832, 140122442113023, ++SNULL, 140122444324863, 140122444328959, ++STORE, 140122444320768, 140122444324863, ++STORE, 140122444324864, 140122444328959, ++SNULL, 94076832014335, 94076832018431, ++STORE, 94076832010240, 94076832014335, ++STORE, 94076832014336, 94076832018431, ++SNULL, 140122446589951, 140122446594047, ++STORE, 140122446585856, 140122446589951, ++STORE, 140122446589952, 140122446594047, ++ERASE, 140122446557184, 140122446585855, ++STORE, 94076845723648, 94076845858815, ++STORE, 140122429939712, 140122438332415, ++SNULL, 140122429943807, 140122438332415, ++STORE, 140122429939712, 140122429943807, ++STORE, 140122429943808, 140122438332415, ++STORE, 140122421547008, 140122429939711, ++STORE, 140122287329280, 140122421547007, ++SNULL, 140122287329280, 140122301399039, ++STORE, 140122301399040, 140122421547007, ++STORE, 140122287329280, 140122301399039, ++ERASE, 140122287329280, 140122301399039, ++SNULL, 140122368507903, 140122421547007, ++STORE, 140122301399040, 140122368507903, ++STORE, 140122368507904, 140122421547007, ++ERASE, 140122368507904, 140122421547007, ++SNULL, 140122301534207, 140122368507903, ++STORE, 140122301399040, 140122301534207, ++STORE, 140122301534208, 140122368507903, ++SNULL, 140122421551103, 140122429939711, ++STORE, 140122421547008, 140122421551103, ++STORE, 140122421551104, 140122429939711, ++STORE, 140122413154304, 140122421547007, ++SNULL, 140122413158399, 140122421547007, ++STORE, 140122413154304, 140122413158399, ++STORE, 140122413158400, 140122421547007, ++STORE, 140122404761600, 140122413154303, ++SNULL, 140122404765695, 140122413154303, ++STORE, 140122404761600, 140122404765695, ++STORE, 140122404765696, 140122413154303, ++STORE, 140122396368896, 140122404761599, ++SNULL, 140122396372991, 140122404761599, ++STORE, 140122396368896, 140122396372991, ++STORE, 140122396372992, 140122404761599, ++STORE, 140122387976192, 140122396368895, ++STORE, 140122167181312, 140122301399039, ++SNULL, 140122234290175, 140122301399039, ++STORE, 140122167181312, 140122234290175, ++STORE, 140122234290176, 140122301399039, ++ERASE, 140122234290176, 140122301399039, ++SNULL, 140122167316479, 140122234290175, ++STORE, 140122167181312, 140122167316479, ++STORE, 140122167316480, 140122234290175, ++STORE, 140122379583488, 140122396368895, ++STORE, 140122371190784, 140122396368895, ++STORE, 140122167316480, 140122301399039, ++STORE, 140122158788608, 140122167181311, ++SNULL, 140122371190784, 140122387976191, ++STORE, 140122387976192, 140122396368895, ++STORE, 140122371190784, 140122387976191, ++SNULL, 140122387980287, 140122396368895, ++STORE, 140122387976192, 140122387980287, ++STORE, 140122387980288, 140122396368895, ++SNULL, 140122167316480, 140122234290175, ++STORE, 140122234290176, 140122301399039, ++STORE, 140122167316480, 140122234290175, ++SNULL, 140122234425343, 140122301399039, ++STORE, 140122234290176, 140122234425343, ++STORE, 140122234425344, 140122301399039, ++STORE, 140122024570880, 140122158788607, ++SNULL, 140122024570880, 140122032963583, ++STORE, 140122032963584, 140122158788607, ++STORE, 140122024570880, 140122032963583, ++ERASE, 140122024570880, 140122032963583, ++STORE, 140121898745856, 140122158788607, ++STORE, 140121890353152, 140121898745855, ++SNULL, 140122100072447, 140122158788607, ++STORE, 140121898745856, 140122100072447, ++STORE, 140122100072448, 140122158788607, ++ERASE, 140122100072448, 140122158788607, ++SNULL, 140121965854719, 140122100072447, ++STORE, 140121898745856, 140121965854719, ++STORE, 140121965854720, 140122100072447, ++SNULL, 140121965854720, 140122032963583, ++STORE, 140122032963584, 140122100072447, ++STORE, 140121965854720, 140122032963583, ++ERASE, 140121965854720, 140122032963583, ++SNULL, 140121898881023, 140121965854719, ++STORE, 140121898745856, 140121898881023, ++STORE, 140121898881024, 140121965854719, ++SNULL, 140121890357247, 140121898745855, ++STORE, 140121890353152, 140121890357247, ++STORE, 140121890357248, 140121898745855, ++SNULL, 140122371190784, 140122379583487, ++STORE, 140122379583488, 140122387976191, ++STORE, 140122371190784, 140122379583487, ++SNULL, 140122379587583, 140122387976191, ++STORE, 140122379583488, 140122379587583, ++STORE, 140122379587584, 140122387976191, ++SNULL, 140122033098751, 140122100072447, ++STORE, 140122032963584, 140122033098751, ++STORE, 140122033098752, 140122100072447, ++SNULL, 140122158792703, 140122167181311, ++STORE, 140122158788608, 140122158792703, ++STORE, 140122158792704, 140122167181311, ++STORE, 140122150395904, 140122158788607, ++STORE, 140122142003200, 140122158788607, ++SNULL, 140122142007295, 140122158788607, ++STORE, 140122142003200, 140122142007295, ++STORE, 140122142007296, 140122158788607, ++SNULL, 140122371194879, 140122379583487, ++STORE, 140122371190784, 140122371194879, ++STORE, 140122371194880, 140122379583487, ++SNULL, 140122142007296, 140122150395903, ++STORE, 140122150395904, 140122158788607, ++STORE, 140122142007296, 140122150395903, ++SNULL, 140122150399999, 140122158788607, ++STORE, 140122150395904, 140122150399999, ++STORE, 140122150400000, 140122158788607, ++STORE, 140122133610496, 140122142003199, ++STORE, 140122125217792, 140122142003199, ++STORE, 140122116825088, 140122142003199, ++SNULL, 140122116829183, 140122142003199, ++STORE, 140122116825088, 140122116829183, ++STORE, 140122116829184, 140122142003199, ++SNULL, 140122116829184, 140122133610495, ++STORE, 140122133610496, 140122142003199, ++STORE, 140122116829184, 140122133610495, ++SNULL, 140122133614591, 140122142003199, ++STORE, 140122133610496, 140122133614591, ++STORE, 140122133614592, 140122142003199, ++SNULL, 140122116829184, 140122125217791, ++STORE, 140122125217792, 140122133610495, ++STORE, 140122116829184, 140122125217791, ++SNULL, 140122125221887, 140122133610495, ++STORE, 140122125217792, 140122125221887, ++STORE, 140122125221888, 140122133610495, ++STORE, 140122108432384, 140122116825087, ++SNULL, 140122108436479, 140122116825087, ++STORE, 140122108432384, 140122108436479, ++STORE, 140122108436480, 140122116825087, ++STORE, 140122024570880, 140122032963583, ++STORE, 140122016178176, 140122032963583, ++SNULL, 140122016182271, 140122032963583, ++STORE, 140122016178176, 140122016182271, ++STORE, 140122016182272, 140122032963583, ++SNULL, 140122016182272, 140122024570879, ++STORE, 140122024570880, 140122032963583, ++STORE, 140122016182272, 140122024570879, ++SNULL, 140122024574975, 140122032963583, ++STORE, 140122024570880, 140122024574975, ++STORE, 140122024574976, 140122032963583, ++STORE, 140122007785472, 140122016178175, ++SNULL, 140122007789567, 140122016178175, ++STORE, 140122007785472, 140122007789567, ++STORE, 140122007789568, 140122016178175, ++STORE, 140121999392768, 140122007785471, ++STORE, 140121991000064, 140122007785471, ++SNULL, 140121991004159, 140122007785471, ++STORE, 140121991000064, 140121991004159, ++STORE, 140121991004160, 140122007785471, ++SNULL, 140121991004160, 140121999392767, ++STORE, 140121999392768, 140122007785471, ++STORE, 140121991004160, 140121999392767, ++SNULL, 140121999396863, 140122007785471, ++STORE, 140121999392768, 140121999396863, ++STORE, 140121999396864, 140122007785471, ++STORE, 140121982607360, 140121991000063, ++STORE, 140121823244288, 140121890353151, ++ERASE, 140121823244288, 140121890353151, ++STORE, 140121756135424, 140121890353151, ++SNULL, 140121756135424, 140121764528127, ++STORE, 140121764528128, 140121890353151, ++STORE, 140121756135424, 140121764528127, ++ERASE, 140121756135424, 140121764528127, ++SNULL, 140121831636991, 140121890353151, ++STORE, 140121764528128, 140121831636991, ++STORE, 140121831636992, 140121890353151, ++ERASE, 140121831636992, 140121890353151, ++STORE, 140121974214656, 140121991000063, ++STORE, 140121630310400, 140121831636991, ++SNULL, 140121697419263, 140121831636991, ++STORE, 140121630310400, 140121697419263, ++STORE, 140121697419264, 140121831636991, ++SNULL, 140121697419264, 140121764528127, ++STORE, 140121764528128, 140121831636991, ++STORE, 140121697419264, 140121764528127, ++ERASE, 140121697419264, 140121764528127, ++STORE, 140121881960448, 140121890353151, ++STORE, 140121630310400, 140121831636991, ++STORE, 140121873567744, 140121890353151, ++SNULL, 140121630310400, 140121697419263, ++STORE, 140121697419264, 140121831636991, ++STORE, 140121630310400, 140121697419263, ++SNULL, 140121697554431, 140121831636991, ++STORE, 140121697419264, 140121697554431, ++STORE, 140121697554432, 140121831636991, ++STORE, 140121865175040, 140121890353151, ++STORE, 140121856782336, 140121890353151, ++STORE, 140121848389632, 140121890353151, ++STORE, 140121839996928, 140121890353151, ++STORE, 140121496092672, 140121697419263, ++STORE, 140121487699968, 140121496092671, ++STORE, 140121420591104, 140121487699967, ++STORE, 140121412198400, 140121420591103, ++ERASE, 140121420591104, 140121487699967, ++STORE, 140121479307264, 140121496092671, ++STORE, 140121277980672, 140121412198399, ++SNULL, 140121277980672, 140121294766079, ++STORE, 140121294766080, 140121412198399, ++STORE, 140121277980672, 140121294766079, ++ERASE, 140121277980672, 140121294766079, ++STORE, 140121470914560, 140121496092671, ++STORE, 140121462521856, 140121496092671, ++STORE, 140121160548352, 140121412198399, ++STORE, 140121454129152, 140121496092671, ++SNULL, 140121227657215, 140121412198399, ++STORE, 140121160548352, 140121227657215, ++STORE, 140121227657216, 140121412198399, ++SNULL, 140121227657216, 140121294766079, ++STORE, 140121294766080, 140121412198399, ++STORE, 140121227657216, 140121294766079, ++ERASE, 140121227657216, 140121294766079, ++STORE, 140121445736448, 140121496092671, ++STORE, 140121437343744, 140121496092671, ++SNULL, 140121437343744, 140121445736447, ++STORE, 140121445736448, 140121496092671, ++STORE, 140121437343744, 140121445736447, ++SNULL, 140121445740543, 140121496092671, ++STORE, 140121445736448, 140121445740543, ++STORE, 140121445740544, 140121496092671, ++SNULL, 140121697554432, 140121764528127, ++STORE, 140121764528128, 140121831636991, ++STORE, 140121697554432, 140121764528127, ++SNULL, 140121764663295, 140121831636991, ++STORE, 140121764528128, 140121764663295, ++STORE, 140121764663296, 140121831636991, ++SNULL, 140121496092672, 140121630310399, ++STORE, 140121630310400, 140121697419263, ++STORE, 140121496092672, 140121630310399, ++SNULL, 140121630445567, 140121697419263, ++STORE, 140121630310400, 140121630445567, ++STORE, 140121630445568, 140121697419263, ++SNULL, 140121445740544, 140121454129151, ++STORE, 140121454129152, 140121496092671, ++STORE, 140121445740544, 140121454129151, ++SNULL, 140121454133247, 140121496092671, ++STORE, 140121454129152, 140121454133247, ++STORE, 140121454133248, 140121496092671, ++STORE, 140121026330624, 140121227657215, ++SNULL, 140121093439487, 140121227657215, ++STORE, 140121026330624, 140121093439487, ++STORE, 140121093439488, 140121227657215, ++SNULL, 140121093439488, 140121160548351, ++STORE, 140121160548352, 140121227657215, ++STORE, 140121093439488, 140121160548351, ++ERASE, 140121093439488, 140121160548351, ++SNULL, 140121563201535, 140121630310399, ++STORE, 140121496092672, 140121563201535, ++STORE, 140121563201536, 140121630310399, ++ERASE, 140121563201536, 140121630310399, ++STORE, 140120892112896, 140121093439487, ++SNULL, 140120959221759, 140121093439487, ++STORE, 140120892112896, 140120959221759, ++STORE, 140120959221760, 140121093439487, ++SNULL, 140120959221760, 140121026330623, ++STORE, 140121026330624, 140121093439487, ++STORE, 140120959221760, 140121026330623, ++ERASE, 140120959221760, 140121026330623, ++STORE, 140120757895168, 140120959221759, ++SNULL, 140121361874943, 140121412198399, ++STORE, 140121294766080, 140121361874943, ++STORE, 140121361874944, 140121412198399, ++ERASE, 140121361874944, 140121412198399, ++SNULL, 140121294901247, 140121361874943, ++STORE, 140121294766080, 140121294901247, ++STORE, 140121294901248, 140121361874943, ++STORE, 140120623677440, 140120959221759, ++SNULL, 140120690786303, 140120959221759, ++STORE, 140120623677440, 140120690786303, ++STORE, 140120690786304, 140120959221759, ++SNULL, 140120690786304, 140120757895167, ++STORE, 140120757895168, 140120959221759, ++STORE, 140120690786304, 140120757895167, ++ERASE, 140120690786304, 140120757895167, ++SNULL, 140121160683519, 140121227657215, ++STORE, 140121160548352, 140121160683519, ++STORE, 140121160683520, 140121227657215, ++SNULL, 140121974214656, 140121982607359, ++STORE, 140121982607360, 140121991000063, ++STORE, 140121974214656, 140121982607359, ++SNULL, 140121982611455, 140121991000063, ++STORE, 140121982607360, 140121982611455, ++STORE, 140121982611456, 140121991000063, ++SNULL, 140121839996928, 140121873567743, ++STORE, 140121873567744, 140121890353151, ++STORE, 140121839996928, 140121873567743, ++SNULL, 140121873571839, 140121890353151, ++STORE, 140121873567744, 140121873571839, ++STORE, 140121873571840, 140121890353151, ++SNULL, 140121873571840, 140121881960447, ++STORE, 140121881960448, 140121890353151, ++STORE, 140121873571840, 140121881960447, ++SNULL, 140121881964543, 140121890353151, ++STORE, 140121881960448, 140121881964543, ++STORE, 140121881964544, 140121890353151, ++SNULL, 140121840001023, 140121873567743, ++STORE, 140121839996928, 140121840001023, ++STORE, 140121840001024, 140121873567743, ++SNULL, 140121840001024, 140121865175039, ++STORE, 140121865175040, 140121873567743, ++STORE, 140121840001024, 140121865175039, ++SNULL, 140121865179135, 140121873567743, ++STORE, 140121865175040, 140121865179135, ++STORE, 140121865179136, 140121873567743, ++SNULL, 140121437347839, 140121445736447, ++STORE, 140121437343744, 140121437347839, ++STORE, 140121437347840, 140121445736447, ++STORE, 140121621917696, 140121630310399, ++STORE, 140121613524992, 140121630310399, ++SNULL, 140121026465791, 140121093439487, ++STORE, 140121026330624, 140121026465791, ++STORE, 140121026465792, 140121093439487, ++SNULL, 140121496227839, 140121563201535, ++STORE, 140121496092672, 140121496227839, ++STORE, 140121496227840, 140121563201535, ++SNULL, 140120757895168, 140120892112895, ++STORE, 140120892112896, 140120959221759, ++STORE, 140120757895168, 140120892112895, ++SNULL, 140120892248063, 140120959221759, ++STORE, 140120892112896, 140120892248063, ++STORE, 140120892248064, 140120959221759, ++SNULL, 140120825004031, 140120892112895, ++STORE, 140120757895168, 140120825004031, ++STORE, 140120825004032, 140120892112895, ++ERASE, 140120825004032, 140120892112895, ++SNULL, 140120623812607, 140120690786303, ++STORE, 140120623677440, 140120623812607, ++STORE, 140120623812608, 140120690786303, ++SNULL, 140120758030335, 140120825004031, ++STORE, 140120757895168, 140120758030335, ++STORE, 140120758030336, 140120825004031, ++SNULL, 140121454133248, 140121462521855, ++STORE, 140121462521856, 140121496092671, ++STORE, 140121454133248, 140121462521855, ++SNULL, 140121462525951, 140121496092671, ++STORE, 140121462521856, 140121462525951, ++STORE, 140121462525952, 140121496092671, ++STORE, 140121605132288, 140121630310399, ++SNULL, 140121605136383, 140121630310399, ++STORE, 140121605132288, 140121605136383, ++STORE, 140121605136384, 140121630310399, ++STORE, 140121596739584, 140121605132287, ++SNULL, 140121605136384, 140121621917695, ++STORE, 140121621917696, 140121630310399, ++STORE, 140121605136384, 140121621917695, ++SNULL, 140121621921791, 140121630310399, ++STORE, 140121621917696, 140121621921791, ++STORE, 140121621921792, 140121630310399, ++STORE, 140121588346880, 140121605132287, ++STORE, 140121579954176, 140121605132287, ++SNULL, 140121412202495, 140121420591103, ++STORE, 140121412198400, 140121412202495, ++STORE, 140121412202496, 140121420591103, ++SNULL, 140121974218751, 140121982607359, ++STORE, 140121974214656, 140121974218751, ++STORE, 140121974218752, 140121982607359, ++SNULL, 140121462525952, 140121479307263, ++STORE, 140121479307264, 140121496092671, ++STORE, 140121462525952, 140121479307263, ++SNULL, 140121479311359, 140121496092671, ++STORE, 140121479307264, 140121479311359, ++STORE, 140121479311360, 140121496092671, ++STORE, 140121571561472, 140121605132287, ++SNULL, 140121571565567, 140121605132287, ++STORE, 140121571561472, 140121571565567, ++STORE, 140121571565568, 140121605132287, ++STORE, 140121428951040, 140121437343743, ++SNULL, 140121428955135, 140121437343743, ++STORE, 140121428951040, 140121428955135, ++STORE, 140121428955136, 140121437343743, ++SNULL, 140121840001024, 140121856782335, ++STORE, 140121856782336, 140121865175039, ++STORE, 140121840001024, 140121856782335, ++SNULL, 140121856786431, 140121865175039, ++STORE, 140121856782336, 140121856786431, ++STORE, 140121856786432, 140121865175039, ++STORE, 140121403805696, 140121412198399, ++SNULL, 140121840001024, 140121848389631, ++STORE, 140121848389632, 140121856782335, ++STORE, 140121840001024, 140121848389631, ++SNULL, 140121848393727, 140121856782335, ++STORE, 140121848389632, 140121848393727, ++STORE, 140121848393728, 140121856782335, ++SNULL, 140121479311360, 140121487699967, ++STORE, 140121487699968, 140121496092671, ++STORE, 140121479311360, 140121487699967, ++SNULL, 140121487704063, 140121496092671, ++STORE, 140121487699968, 140121487704063, ++STORE, 140121487704064, 140121496092671, ++STORE, 140121395412992, 140121412198399, ++STORE, 140121387020288, 140121412198399, ++SNULL, 140121387024383, 140121412198399, ++STORE, 140121387020288, 140121387024383, ++STORE, 140121387024384, 140121412198399, ++SNULL, 140121605136384, 140121613524991, ++STORE, 140121613524992, 140121621917695, ++STORE, 140121605136384, 140121613524991, ++SNULL, 140121613529087, 140121621917695, ++STORE, 140121613524992, 140121613529087, ++STORE, 140121613529088, 140121621917695, ++SNULL, 140121462525952, 140121470914559, ++STORE, 140121470914560, 140121479307263, ++STORE, 140121462525952, 140121470914559, ++SNULL, 140121470918655, 140121479307263, ++STORE, 140121470914560, 140121470918655, ++STORE, 140121470918656, 140121479307263, ++STORE, 140121378627584, 140121387020287, ++SNULL, 140121378631679, 140121387020287, ++STORE, 140121378627584, 140121378631679, ++STORE, 140121378631680, 140121387020287, ++SNULL, 140121571565568, 140121596739583, ++STORE, 140121596739584, 140121605132287, ++STORE, 140121571565568, 140121596739583, ++SNULL, 140121596743679, 140121605132287, ++STORE, 140121596739584, 140121596743679, ++STORE, 140121596743680, 140121605132287, ++SNULL, 140121387024384, 140121403805695, ++STORE, 140121403805696, 140121412198399, ++STORE, 140121387024384, 140121403805695, ++SNULL, 140121403809791, 140121412198399, ++STORE, 140121403805696, 140121403809791, ++STORE, 140121403809792, 140121412198399, ++STORE, 140121370234880, 140121378627583, ++SNULL, 140121387024384, 140121395412991, ++STORE, 140121395412992, 140121403805695, ++STORE, 140121387024384, 140121395412991, ++SNULL, 140121395417087, 140121403805695, ++STORE, 140121395412992, 140121395417087, ++STORE, 140121395417088, 140121403805695, ++SNULL, 140121571565568, 140121588346879, ++STORE, 140121588346880, 140121596739583, ++STORE, 140121571565568, 140121588346879, ++SNULL, 140121588350975, 140121596739583, ++STORE, 140121588346880, 140121588350975, ++STORE, 140121588350976, 140121596739583, ++SNULL, 140121571565568, 140121579954175, ++STORE, 140121579954176, 140121588346879, ++STORE, 140121571565568, 140121579954175, ++SNULL, 140121579958271, 140121588346879, ++STORE, 140121579954176, 140121579958271, ++STORE, 140121579958272, 140121588346879, ++STORE, 140121286373376, 140121294766079, ++STORE, 140121277980672, 140121294766079, ++SNULL, 140121277980672, 140121286373375, ++STORE, 140121286373376, 140121294766079, ++STORE, 140121277980672, 140121286373375, ++SNULL, 140121286377471, 140121294766079, ++STORE, 140121286373376, 140121286377471, ++STORE, 140121286377472, 140121294766079, ++STORE, 140121269587968, 140121286373375, ++STORE, 140121261195264, 140121286373375, ++SNULL, 140121261195264, 140121269587967, ++STORE, 140121269587968, 140121286373375, ++STORE, 140121261195264, 140121269587967, ++SNULL, 140121269592063, 140121286373375, ++STORE, 140121269587968, 140121269592063, ++STORE, 140121269592064, 140121286373375, ++STORE, 140121252802560, 140121269587967, ++SNULL, 140121252806655, 140121269587967, ++STORE, 140121252802560, 140121252806655, ++STORE, 140121252806656, 140121269587967, ++STORE, 140121244409856, 140121252802559, ++STORE, 140121236017152, 140121252802559, ++SNULL, 140121236017152, 140121244409855, ++STORE, 140121244409856, 140121252802559, ++STORE, 140121236017152, 140121244409855, ++SNULL, 140121244413951, 140121252802559, ++STORE, 140121244409856, 140121244413951, ++STORE, 140121244413952, 140121252802559, ++SNULL, 140121370238975, 140121378627583, ++STORE, 140121370234880, 140121370238975, ++STORE, 140121370238976, 140121378627583, ++STORE, 140121152155648, 140121160548351, ++STORE, 140121143762944, 140121160548351, ++STORE, 140121135370240, 140121160548351, ++SNULL, 140121135374335, 140121160548351, ++STORE, 140121135370240, 140121135374335, ++STORE, 140121135374336, 140121160548351, ++STORE, 140121126977536, 140121135370239, ++STORE, 140121118584832, 140121135370239, ++STORE, 140121110192128, 140121135370239, ++SNULL, 140121110192128, 140121118584831, ++STORE, 140121118584832, 140121135370239, ++STORE, 140121110192128, 140121118584831, ++SNULL, 140121118588927, 140121135370239, ++STORE, 140121118584832, 140121118588927, ++STORE, 140121118588928, 140121135370239, ++STORE, 140121101799424, 140121118584831, ++STORE, 140121017937920, 140121026330623, ++STORE, 140121009545216, 140121026330623, ++SNULL, 140121009545216, 140121017937919, ++STORE, 140121017937920, 140121026330623, ++STORE, 140121009545216, 140121017937919, ++SNULL, 140121017942015, 140121026330623, ++STORE, 140121017937920, 140121017942015, ++STORE, 140121017942016, 140121026330623, ++SNULL, 140121269592064, 140121277980671, ++STORE, 140121277980672, 140121286373375, ++STORE, 140121269592064, 140121277980671, ++SNULL, 140121277984767, 140121286373375, ++STORE, 140121277980672, 140121277984767, ++STORE, 140121277984768, 140121286373375, ++STORE, 140121001152512, 140121017937919, ++SNULL, 140121252806656, 140121261195263, ++STORE, 140121261195264, 140121269587967, ++STORE, 140121252806656, 140121261195263, ++SNULL, 140121261199359, 140121269587967, ++STORE, 140121261195264, 140121261199359, ++STORE, 140121261199360, 140121269587967, ++SNULL, 140121135374336, 140121152155647, ++STORE, 140121152155648, 140121160548351, ++STORE, 140121135374336, 140121152155647, ++SNULL, 140121152159743, 140121160548351, ++STORE, 140121152155648, 140121152159743, ++STORE, 140121152159744, 140121160548351, ++STORE, 140120992759808, 140121017937919, ++STORE, 140120984367104, 140121017937919, ++STORE, 140120975974400, 140121017937919, ++SNULL, 140121101799424, 140121110192127, ++STORE, 140121110192128, 140121118584831, ++STORE, 140121101799424, 140121110192127, ++SNULL, 140121110196223, 140121118584831, ++STORE, 140121110192128, 140121110196223, ++STORE, 140121110196224, 140121118584831, ++SNULL, 140121118588928, 140121126977535, ++STORE, 140121126977536, 140121135370239, ++STORE, 140121118588928, 140121126977535, ++SNULL, 140121126981631, 140121135370239, ++STORE, 140121126977536, 140121126981631, ++STORE, 140121126981632, 140121135370239, ++STORE, 140120967581696, 140121017937919, ++STORE, 140120883720192, 140120892112895, ++SNULL, 140120883724287, 140120892112895, ++STORE, 140120883720192, 140120883724287, ++STORE, 140120883724288, 140120892112895, ++STORE, 140120875327488, 140120883720191, ++SNULL, 140121101803519, 140121110192127, ++STORE, 140121101799424, 140121101803519, ++STORE, 140121101803520, 140121110192127, ++SNULL, 140121135374336, 140121143762943, ++STORE, 140121143762944, 140121152155647, ++STORE, 140121135374336, 140121143762943, ++SNULL, 140121143767039, 140121152155647, ++STORE, 140121143762944, 140121143767039, ++STORE, 140121143767040, 140121152155647, ++STORE, 140120866934784, 140120883720191, ++SNULL, 140120967581696, 140120984367103, ++STORE, 140120984367104, 140121017937919, ++STORE, 140120967581696, 140120984367103, ++SNULL, 140120984371199, 140121017937919, ++STORE, 140120984367104, 140120984371199, ++STORE, 140120984371200, 140121017937919, ++STORE, 140120858542080, 140120883720191, ++SNULL, 140121236021247, 140121244409855, ++STORE, 140121236017152, 140121236021247, ++STORE, 140121236021248, 140121244409855, ++SNULL, 140120984371200, 140121009545215, ++STORE, 140121009545216, 140121017937919, ++STORE, 140120984371200, 140121009545215, ++SNULL, 140121009549311, 140121017937919, ++STORE, 140121009545216, 140121009549311, ++STORE, 140121009549312, 140121017937919, ++SNULL, 140120984371200, 140120992759807, ++STORE, 140120992759808, 140121009545215, ++STORE, 140120984371200, 140120992759807, ++SNULL, 140120992763903, 140121009545215, ++STORE, 140120992759808, 140120992763903, ++STORE, 140120992763904, 140121009545215, ++SNULL, 140120992763904, 140121001152511, ++STORE, 140121001152512, 140121009545215, ++STORE, 140120992763904, 140121001152511, ++SNULL, 140121001156607, 140121009545215, ++STORE, 140121001152512, 140121001156607, ++STORE, 140121001156608, 140121009545215, ++STORE, 140120850149376, 140120883720191, ++SNULL, 140120850153471, 140120883720191, ++STORE, 140120850149376, 140120850153471, ++STORE, 140120850153472, 140120883720191, ++SNULL, 140120967585791, 140120984367103, ++STORE, 140120967581696, 140120967585791, ++STORE, 140120967585792, 140120984367103, ++SNULL, 140120850153472, 140120866934783, ++STORE, 140120866934784, 140120883720191, ++STORE, 140120850153472, 140120866934783, ++SNULL, 140120866938879, 140120883720191, ++STORE, 140120866934784, 140120866938879, ++STORE, 140120866938880, 140120883720191, ++STORE, 140120841756672, 140120850149375, ++SNULL, 140120967585792, 140120975974399, ++STORE, 140120975974400, 140120984367103, ++STORE, 140120967585792, 140120975974399, ++SNULL, 140120975978495, 140120984367103, ++STORE, 140120975974400, 140120975978495, ++STORE, 140120975978496, 140120984367103, ++SNULL, 140120866938880, 140120875327487, ++STORE, 140120875327488, 140120883720191, ++STORE, 140120866938880, 140120875327487, ++SNULL, 140120875331583, 140120883720191, ++STORE, 140120875327488, 140120875331583, ++STORE, 140120875331584, 140120883720191, ++STORE, 140120833363968, 140120850149375, ++STORE, 140120749502464, 140120757895167, ++STORE, 140120741109760, 140120757895167, ++STORE, 140120732717056, 140120757895167, ++STORE, 140120724324352, 140120757895167, ++SNULL, 140120724324352, 140120732717055, ++STORE, 140120732717056, 140120757895167, ++STORE, 140120724324352, 140120732717055, ++SNULL, 140120732721151, 140120757895167, ++STORE, 140120732717056, 140120732721151, ++STORE, 140120732721152, 140120757895167, ++STORE, 140120715931648, 140120732717055, ++SNULL, 140120715935743, 140120732717055, ++STORE, 140120715931648, 140120715935743, ++STORE, 140120715935744, 140120732717055, ++SNULL, 140120850153472, 140120858542079, ++STORE, 140120858542080, 140120866934783, ++STORE, 140120850153472, 140120858542079, ++SNULL, 140120858546175, 140120866934783, ++STORE, 140120858542080, 140120858546175, ++STORE, 140120858546176, 140120866934783, ++STORE, 140120707538944, 140120715931647, ++SNULL, 140120707543039, 140120715931647, ++STORE, 140120707538944, 140120707543039, ++STORE, 140120707543040, 140120715931647, ++SNULL, 140120833368063, 140120850149375, ++STORE, 140120833363968, 140120833368063, ++STORE, 140120833368064, 140120850149375, ++SNULL, 140120833368064, 140120841756671, ++STORE, 140120841756672, 140120850149375, ++STORE, 140120833368064, 140120841756671, ++SNULL, 140120841760767, 140120850149375, ++STORE, 140120841756672, 140120841760767, ++STORE, 140120841760768, 140120850149375, ++STORE, 140120699146240, 140120707538943, ++SNULL, 140120715935744, 140120724324351, ++STORE, 140120724324352, 140120732717055, ++STORE, 140120715935744, 140120724324351, ++SNULL, 140120724328447, 140120732717055, ++STORE, 140120724324352, 140120724328447, ++STORE, 140120724328448, 140120732717055, ++SNULL, 140120732721152, 140120741109759, ++STORE, 140120741109760, 140120757895167, ++STORE, 140120732721152, 140120741109759, ++SNULL, 140120741113855, 140120757895167, ++STORE, 140120741109760, 140120741113855, ++STORE, 140120741113856, 140120757895167, ++SNULL, 140120741113856, 140120749502463, ++STORE, 140120749502464, 140120757895167, ++STORE, 140120741113856, 140120749502463, ++SNULL, 140120749506559, 140120757895167, ++STORE, 140120749502464, 140120749506559, ++STORE, 140120749506560, 140120757895167, ++SNULL, 140120699150335, 140120707538943, ++STORE, 140120699146240, 140120699150335, ++STORE, 140120699150336, 140120707538943, ++STORE, 140122446557184, 140122446585855, ++STORE, 140122368999424, 140122371190783, ++SNULL, 140122368999424, 140122369089535, ++STORE, 140122369089536, 140122371190783, ++STORE, 140122368999424, 140122369089535, ++SNULL, 140122371182591, 140122371190783, ++STORE, 140122369089536, 140122371182591, ++STORE, 140122371182592, 140122371190783, ++ERASE, 140122371182592, 140122371190783, ++STORE, 140122371182592, 140122371190783, ++SNULL, 140122371186687, 140122371190783, ++STORE, 140122371182592, 140122371186687, ++STORE, 140122371186688, 140122371190783, ++ERASE, 140122446557184, 140122446585855, ++ERASE, 140121445736448, 140121445740543, ++ERASE, 140121445740544, 140121454129151, ++ERASE, 140121621917696, 140121621921791, ++ERASE, 140121621921792, 140121630310399, ++ERASE, 140121579954176, 140121579958271, ++ERASE, 140121579958272, 140121588346879, ++ERASE, 140121261195264, 140121261199359, ++ERASE, 140121261199360, 140121269587967, ++ERASE, 140121454129152, 140121454133247, ++ERASE, 140121454133248, 140121462521855, ++ERASE, 140121588346880, 140121588350975, ++ERASE, 140121588350976, 140121596739583, ++ERASE, 140121135370240, 140121135374335, ++ERASE, 140121135374336, 140121143762943, ++ERASE, 140121881960448, 140121881964543, ++ERASE, 140121881964544, 140121890353151, ++ERASE, 140121428951040, 140121428955135, ++ERASE, 140121428955136, 140121437343743, ++ERASE, 140121387020288, 140121387024383, ++ERASE, 140121387024384, 140121395412991, ++ERASE, 140121487699968, 140121487704063, ++ERASE, 140121487704064, 140121496092671, ++ERASE, 140121437343744, 140121437347839, ++ERASE, 140121437347840, 140121445736447, ++ERASE, 140121613524992, 140121613529087, ++ERASE, 140121613529088, 140121621917695, ++ERASE, 140121856782336, 140121856786431, ++ERASE, 140121856786432, 140121865175039, ++ERASE, 140121252802560, 140121252806655, ++ERASE, 140121252806656, 140121261195263, ++ERASE, 140121839996928, 140121840001023, ++ERASE, 140121840001024, 140121848389631, ++ERASE, 140121596739584, 140121596743679, ++ERASE, 140121596743680, 140121605132287, ++ERASE, 140121009545216, 140121009549311, ++ERASE, 140121009549312, 140121017937919, ++ERASE, 140120724324352, 140120724328447, ++ERASE, 140120724328448, 140120732717055, ++ERASE, 140120883720192, 140120883724287, ++ERASE, 140120883724288, 140120892112895, ++ERASE, 140121982607360, 140121982611455, ++ERASE, 140121982611456, 140121991000063, ++ERASE, 140121571561472, 140121571565567, ++ERASE, 140121571565568, 140121579954175, ++ERASE, 140121286373376, 140121286377471, ++ERASE, 140121286377472, 140121294766079, ++ERASE, 140120875327488, 140120875331583, ++ERASE, 140120875331584, 140120883720191, ++ERASE, 140121848389632, 140121848393727, ++ERASE, 140121848393728, 140121856782335, ++ERASE, 140121370234880, 140121370238975, ++ERASE, 140121370238976, 140121378627583, ++ERASE, 140121143762944, 140121143767039, ++ERASE, 140121143767040, 140121152155647, ++ERASE, 140121118584832, 140121118588927, ++ERASE, 140121118588928, 140121126977535, ++ERASE, 140120866934784, 140120866938879, ++ERASE, 140120866938880, 140120875327487, ++ERASE, 140120741109760, 140120741113855, ++ERASE, 140120741113856, 140120749502463, ++ERASE, 140121865175040, 140121865179135, ++ERASE, 140121865179136, 140121873567743, ++ERASE, 140121403805696, 140121403809791, ++ERASE, 140121403809792, 140121412198399, ++ERASE, 140121236017152, 140121236021247, ++ERASE, 140121236021248, 140121244409855, ++ERASE, 140120732717056, 140120732721151, ++ERASE, 140120732721152, 140120741109759, ++ERASE, 140121017937920, 140121017942015, ++ERASE, 140121017942016, 140121026330623, ++ERASE, 140121873567744, 140121873571839, ++ERASE, 140121873571840, 140121881960447, ++ERASE, 140121470914560, 140121470918655, ++ERASE, 140121470918656, 140121479307263, ++ERASE, 140121126977536, 140121126981631, ++ERASE, 140121126981632, 140121135370239, ++ERASE, 140120850149376, 140120850153471, ++ERASE, 140120850153472, 140120858542079, ++ERASE, 140120707538944, 140120707543039, ++ERASE, 140120707543040, 140120715931647, ++ERASE, 140121479307264, 140121479311359, ++ERASE, 140121479311360, 140121487699967, ++ERASE, 140120967581696, 140120967585791, ++ERASE, 140120967585792, 140120975974399, ++ERASE, 140120841756672, 140120841760767, ++ERASE, 140120841760768, 140120850149375, ++ERASE, 140121412198400, 140121412202495, ++ERASE, 140121412202496, 140121420591103, ++ERASE, 140122158788608, 140122158792703, ++ERASE, 140122158792704, 140122167181311, ++ERASE, 140122142003200, 140122142007295, ++ERASE, 140122142007296, 140122150395903, ++ERASE, 140121101799424, 140121101803519, ++ERASE, 140121101803520, 140121110192127, ++ERASE, 140120858542080, 140120858546175, ++ERASE, 140120858546176, 140120866934783, ++ERASE, 140120833363968, 140120833368063, ++ERASE, 140120833368064, 140120841756671, ++ERASE, 140121277980672, 140121277984767, ++ERASE, 140121277984768, 140121286373375, ++ERASE, 140121001152512, 140121001156607, ++ERASE, 140121001156608, 140121009545215, ++ERASE, 140120749502464, 140120749506559, ++ERASE, 140120749506560, 140120757895167, ++ERASE, 140121605132288, 140121605136383, ++ERASE, 140121605136384, 140121613524991, ++ERASE, 140121378627584, 140121378631679, ++ERASE, 140121378631680, 140121387020287, ++ERASE, 140121110192128, 140121110196223, ++ERASE, 140121110196224, 140121118584831, ++ERASE, 140121462521856, 140121462525951, ++ERASE, 140121462525952, 140121470914559, ++ERASE, 140121395412992, 140121395417087, ++ERASE, 140121395417088, 140121403805695, ++ERASE, 140121152155648, 140121152159743, ++ERASE, 140121152159744, 140121160548351, ++ERASE, 140120992759808, 140120992763903, ++ERASE, 140120992763904, 140121001152511, ++ERASE, 140122387976192, 140122387980287, ++ERASE, 140122387980288, 140122396368895, ++ERASE, 140121890353152, 140121890357247, ++ERASE, 140121890357248, 140121898745855, ++ERASE, 140121269587968, 140121269592063, ++ERASE, 140121269592064, 140121277980671, ++ }; ++ unsigned long set37[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722404016128, 140737488351231, ++SNULL, 140722404020223, 140737488351231, ++STORE, 140722404016128, 140722404020223, ++STORE, 140722403885056, 140722404020223, ++STORE, 94637010001920, 94637012254719, ++SNULL, 94637010132991, 94637012254719, ++STORE, 94637010001920, 94637010132991, ++STORE, 94637010132992, 94637012254719, ++ERASE, 94637010132992, 94637012254719, ++STORE, 94637012226048, 94637012234239, ++STORE, 94637012234240, 94637012254719, ++STORE, 139760240594944, 139760242847743, ++SNULL, 139760240738303, 139760242847743, ++STORE, 139760240594944, 139760240738303, ++STORE, 139760240738304, 139760242847743, ++ERASE, 139760240738304, 139760242847743, ++STORE, 139760242835456, 139760242843647, ++STORE, 139760242843648, 139760242847743, ++STORE, 140722405232640, 140722405236735, ++STORE, 140722405220352, 140722405232639, ++STORE, 139760242806784, 139760242835455, ++STORE, 139760242798592, 139760242806783, ++STORE, 139760238379008, 139760240594943, ++SNULL, 139760238379008, 139760238477311, ++STORE, 139760238477312, 139760240594943, ++STORE, 139760238379008, 139760238477311, ++SNULL, 139760240570367, 139760240594943, ++STORE, 139760238477312, 139760240570367, ++STORE, 139760240570368, 139760240594943, ++SNULL, 139760240570368, 139760240578559, ++STORE, 139760240578560, 139760240594943, ++STORE, 139760240570368, 139760240578559, ++ERASE, 139760240570368, 139760240578559, ++STORE, 139760240570368, 139760240578559, ++ERASE, 139760240578560, 139760240594943, ++STORE, 139760240578560, 139760240594943, ++STORE, 139760234582016, 139760238379007, ++SNULL, 139760234582016, 139760236240895, ++STORE, 139760236240896, 139760238379007, ++STORE, 139760234582016, 139760236240895, ++SNULL, 139760238338047, 139760238379007, ++STORE, 139760236240896, 139760238338047, ++STORE, 139760238338048, 139760238379007, ++SNULL, 139760238338048, 139760238362623, ++STORE, 139760238362624, 139760238379007, ++STORE, 139760238338048, 139760238362623, ++ERASE, 139760238338048, 139760238362623, ++STORE, 139760238338048, 139760238362623, ++ERASE, 139760238362624, 139760238379007, ++STORE, 139760238362624, 139760238379007, ++STORE, 139760242790400, 139760242806783, ++SNULL, 139760238354431, 139760238362623, ++STORE, 139760238338048, 139760238354431, ++STORE, 139760238354432, 139760238362623, ++SNULL, 139760240574463, 139760240578559, ++STORE, 139760240570368, 139760240574463, ++STORE, 139760240574464, 139760240578559, ++SNULL, 94637012230143, 94637012234239, ++STORE, 94637012226048, 94637012230143, ++STORE, 94637012230144, 94637012234239, ++SNULL, 139760242839551, 139760242843647, ++STORE, 139760242835456, 139760242839551, ++STORE, 139760242839552, 139760242843647, ++ERASE, 139760242806784, 139760242835455, ++STORE, 94637033324544, 94637033459711, ++STORE, 139760226189312, 139760234582015, ++SNULL, 139760226193407, 139760234582015, ++STORE, 139760226189312, 139760226193407, ++STORE, 139760226193408, 139760234582015, ++STORE, 139760217796608, 139760226189311, ++STORE, 139760083578880, 139760217796607, ++SNULL, 139760083578880, 139760114860031, ++STORE, 139760114860032, 139760217796607, ++STORE, 139760083578880, 139760114860031, ++ERASE, 139760083578880, 139760114860031, ++SNULL, 139760181968895, 139760217796607, ++STORE, 139760114860032, 139760181968895, ++STORE, 139760181968896, 139760217796607, ++ERASE, 139760181968896, 139760217796607, ++SNULL, 139760114995199, 139760181968895, ++STORE, 139760114860032, 139760114995199, ++STORE, 139760114995200, 139760181968895, ++SNULL, 139760217800703, 139760226189311, ++STORE, 139760217796608, 139760217800703, ++STORE, 139760217800704, 139760226189311, ++STORE, 139760209403904, 139760217796607, ++SNULL, 139760209407999, 139760217796607, ++STORE, 139760209403904, 139760209407999, ++STORE, 139760209408000, 139760217796607, ++STORE, 139760201011200, 139760209403903, ++SNULL, 139760201015295, 139760209403903, ++STORE, 139760201011200, 139760201015295, ++STORE, 139760201015296, 139760209403903, ++STORE, 139760192618496, 139760201011199, ++SNULL, 139760192622591, 139760201011199, ++STORE, 139760192618496, 139760192622591, ++STORE, 139760192622592, 139760201011199, ++STORE, 139760184225792, 139760192618495, ++STORE, 139759980642304, 139760114860031, ++STORE, 139759972249600, 139759980642303, ++STORE, 139759963856896, 139759980642303, ++STORE, 139759955464192, 139759980642303, ++STORE, 139759888355328, 139759955464191, ++SNULL, 139760047751167, 139760114860031, ++STORE, 139759980642304, 139760047751167, ++STORE, 139760047751168, 139760114860031, ++ERASE, 139760047751168, 139760114860031, ++SNULL, 139759980777471, 139760047751167, ++STORE, 139759980642304, 139759980777471, ++STORE, 139759980777472, 139760047751167, ++STORE, 139759980777472, 139760114860031, ++SNULL, 139759980777472, 139760047751167, ++STORE, 139760047751168, 139760114860031, ++STORE, 139759980777472, 139760047751167, ++SNULL, 139760047886335, 139760114860031, ++STORE, 139760047751168, 139760047886335, ++STORE, 139760047886336, 139760114860031, ++STORE, 139759821246464, 139759955464191, ++SNULL, 139759821246464, 139759888355327, ++STORE, 139759888355328, 139759955464191, ++STORE, 139759821246464, 139759888355327, ++ERASE, 139759821246464, 139759888355327, ++ERASE, 139759888355328, 139759955464191, ++ }; ++ unsigned long set38[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140730666221568, 140737488351231, ++SNULL, 140730666225663, 140737488351231, ++STORE, 140730666221568, 140730666225663, ++STORE, 140730666090496, 140730666225663, ++STORE, 94177584803840, 94177587056639, ++SNULL, 94177584934911, 94177587056639, ++STORE, 94177584803840, 94177584934911, ++STORE, 94177584934912, 94177587056639, ++ERASE, 94177584934912, 94177587056639, ++STORE, 94177587027968, 94177587036159, ++STORE, 94177587036160, 94177587056639, ++STORE, 140614382714880, 140614384967679, ++SNULL, 140614382858239, 140614384967679, ++STORE, 140614382714880, 140614382858239, ++STORE, 140614382858240, 140614384967679, ++ERASE, 140614382858240, 140614384967679, ++STORE, 140614384955392, 140614384963583, ++STORE, 140614384963584, 140614384967679, ++STORE, 140730666315776, 140730666319871, ++STORE, 140730666303488, 140730666315775, ++STORE, 140614384926720, 140614384955391, ++STORE, 140614384918528, 140614384926719, ++STORE, 140614380498944, 140614382714879, ++SNULL, 140614380498944, 140614380597247, ++STORE, 140614380597248, 140614382714879, ++STORE, 140614380498944, 140614380597247, ++SNULL, 140614382690303, 140614382714879, ++STORE, 140614380597248, 140614382690303, ++STORE, 140614382690304, 140614382714879, ++SNULL, 140614382690304, 140614382698495, ++STORE, 140614382698496, 140614382714879, ++STORE, 140614382690304, 140614382698495, ++ERASE, 140614382690304, 140614382698495, ++STORE, 140614382690304, 140614382698495, ++ERASE, 140614382698496, 140614382714879, ++STORE, 140614382698496, 140614382714879, ++STORE, 140614376701952, 140614380498943, ++SNULL, 140614376701952, 140614378360831, ++STORE, 140614378360832, 140614380498943, ++STORE, 140614376701952, 140614378360831, ++SNULL, 140614380457983, 140614380498943, ++STORE, 140614378360832, 140614380457983, ++STORE, 140614380457984, 140614380498943, ++SNULL, 140614380457984, 140614380482559, ++STORE, 140614380482560, 140614380498943, ++STORE, 140614380457984, 140614380482559, ++ERASE, 140614380457984, 140614380482559, ++STORE, 140614380457984, 140614380482559, ++ERASE, 140614380482560, 140614380498943, ++STORE, 140614380482560, 140614380498943, ++STORE, 140614384910336, 140614384926719, ++SNULL, 140614380474367, 140614380482559, ++STORE, 140614380457984, 140614380474367, ++STORE, 140614380474368, 140614380482559, ++SNULL, 140614382694399, 140614382698495, ++STORE, 140614382690304, 140614382694399, ++STORE, 140614382694400, 140614382698495, ++SNULL, 94177587032063, 94177587036159, ++STORE, 94177587027968, 94177587032063, ++STORE, 94177587032064, 94177587036159, ++SNULL, 140614384959487, 140614384963583, ++STORE, 140614384955392, 140614384959487, ++STORE, 140614384959488, 140614384963583, ++ERASE, 140614384926720, 140614384955391, ++STORE, 94177619791872, 94177619927039, ++STORE, 140614368309248, 140614376701951, ++SNULL, 140614368313343, 140614376701951, ++STORE, 140614368309248, 140614368313343, ++STORE, 140614368313344, 140614376701951, ++STORE, 140614359916544, 140614368309247, ++STORE, 140614225698816, 140614359916543, ++SNULL, 140614225698816, 140614276481023, ++STORE, 140614276481024, 140614359916543, ++STORE, 140614225698816, 140614276481023, ++ERASE, 140614225698816, 140614276481023, ++SNULL, 140614343589887, 140614359916543, ++STORE, 140614276481024, 140614343589887, ++STORE, 140614343589888, 140614359916543, ++ERASE, 140614343589888, 140614359916543, ++SNULL, 140614276616191, 140614343589887, ++STORE, 140614276481024, 140614276616191, ++STORE, 140614276616192, 140614343589887, ++SNULL, 140614359920639, 140614368309247, ++STORE, 140614359916544, 140614359920639, ++STORE, 140614359920640, 140614368309247, ++STORE, 140614351523840, 140614359916543, ++SNULL, 140614351527935, 140614359916543, ++STORE, 140614351523840, 140614351527935, ++STORE, 140614351527936, 140614359916543, ++STORE, 140614268088320, 140614276481023, ++SNULL, 140614268092415, 140614276481023, ++STORE, 140614268088320, 140614268092415, ++STORE, 140614268092416, 140614276481023, ++STORE, 140614259695616, 140614268088319, ++SNULL, 140614259699711, 140614268088319, ++STORE, 140614259695616, 140614259699711, ++STORE, 140614259699712, 140614268088319, ++STORE, 140614251302912, 140614259695615, ++STORE, 140614242910208, 140614259695615, ++STORE, 140614108692480, 140614242910207, ++SNULL, 140614108692480, 140614142263295, ++STORE, 140614142263296, 140614242910207, ++STORE, 140614108692480, 140614142263295, ++ERASE, 140614108692480, 140614142263295, ++STORE, 140614133870592, 140614142263295, ++STORE, 140613999652864, 140614133870591, ++SNULL, 140613999652864, 140614008045567, ++STORE, 140614008045568, 140614133870591, ++STORE, 140613999652864, 140614008045567, ++ERASE, 140613999652864, 140614008045567, ++STORE, 140613999652864, 140614008045567, ++STORE, 140613865435136, 140613999652863, ++SNULL, 140613865435136, 140613873827839, ++STORE, 140613873827840, 140613999652863, ++STORE, 140613865435136, 140613873827839, ++ERASE, 140613865435136, 140613873827839, ++SNULL, 140614209372159, 140614242910207, ++STORE, 140614142263296, 140614209372159, ++STORE, 140614209372160, 140614242910207, ++ERASE, 140614209372160, 140614242910207, ++SNULL, 140614142398463, 140614209372159, ++STORE, 140614142263296, 140614142398463, ++STORE, 140614142398464, 140614209372159, ++SNULL, 140614075154431, 140614133870591, ++STORE, 140614008045568, 140614075154431, ++STORE, 140614075154432, 140614133870591, ++ERASE, 140614075154432, 140614133870591, ++SNULL, 140614008180735, 140614075154431, ++STORE, 140614008045568, 140614008180735, ++STORE, 140614008180736, 140614075154431, ++SNULL, 140613940936703, 140613999652863, ++STORE, 140613873827840, 140613940936703, ++STORE, 140613940936704, 140613999652863, ++ERASE, 140613940936704, 140613999652863, ++SNULL, 140614242914303, 140614259695615, ++STORE, 140614242910208, 140614242914303, ++STORE, 140614242914304, 140614259695615, ++STORE, 140613739610112, 140613940936703, ++STORE, 140614234517504, 140614242910207, ++SNULL, 140614242914304, 140614251302911, ++STORE, 140614251302912, 140614259695615, ++STORE, 140614242914304, 140614251302911, ++SNULL, 140614251307007, 140614259695615, ++STORE, 140614251302912, 140614251307007, ++STORE, 140614251307008, 140614259695615, ++SNULL, 140613739610112, 140613873827839, ++STORE, 140613873827840, 140613940936703, ++STORE, 140613739610112, 140613873827839, ++SNULL, 140613873963007, 140613940936703, ++STORE, 140613873827840, 140613873963007, ++STORE, 140613873963008, 140613940936703, ++SNULL, 140614133874687, 140614142263295, ++STORE, 140614133870592, 140614133874687, ++STORE, 140614133874688, 140614142263295, ++SNULL, 140613806718975, 140613873827839, ++STORE, 140613739610112, 140613806718975, ++STORE, 140613806718976, 140613873827839, ++ERASE, 140613806718976, 140613873827839, ++STORE, 140614226124800, 140614242910207, ++SNULL, 140613739745279, 140613806718975, ++STORE, 140613739610112, 140613739745279, ++STORE, 140613739745280, 140613806718975, ++SNULL, 140613999656959, 140614008045567, ++STORE, 140613999652864, 140613999656959, ++STORE, 140613999656960, 140614008045567, ++SNULL, 140614226124800, 140614234517503, ++STORE, 140614234517504, 140614242910207, ++STORE, 140614226124800, 140614234517503, ++SNULL, 140614234521599, 140614242910207, ++STORE, 140614234517504, 140614234521599, ++STORE, 140614234521600, 140614242910207, ++STORE, 140614217732096, 140614234517503, ++STORE, 140614125477888, 140614133870591, ++SNULL, 140614125481983, 140614133870591, ++STORE, 140614125477888, 140614125481983, ++STORE, 140614125481984, 140614133870591, ++STORE, 140614117085184, 140614125477887, ++SNULL, 140614217736191, 140614234517503, ++STORE, 140614217732096, 140614217736191, ++STORE, 140614217736192, 140614234517503, ++SNULL, 140614117089279, 140614125477887, ++STORE, 140614117085184, 140614117089279, ++STORE, 140614117089280, 140614125477887, ++SNULL, 140614217736192, 140614226124799, ++STORE, 140614226124800, 140614234517503, ++STORE, 140614217736192, 140614226124799, ++SNULL, 140614226128895, 140614234517503, ++STORE, 140614226124800, 140614226128895, ++STORE, 140614226128896, 140614234517503, ++STORE, 140614108692480, 140614117085183, ++STORE, 140614100299776, 140614117085183, ++STORE, 140614091907072, 140614117085183, ++SNULL, 140614091907072, 140614108692479, ++STORE, 140614108692480, 140614117085183, ++STORE, 140614091907072, 140614108692479, ++SNULL, 140614108696575, 140614117085183, ++STORE, 140614108692480, 140614108696575, ++STORE, 140614108696576, 140614117085183, ++SNULL, 140614091907072, 140614100299775, ++STORE, 140614100299776, 140614108692479, ++STORE, 140614091907072, 140614100299775, ++SNULL, 140614100303871, 140614108692479, ++STORE, 140614100299776, 140614100303871, ++STORE, 140614100303872, 140614108692479, ++STORE, 140614083514368, 140614100299775, ++SNULL, 140614083518463, 140614100299775, ++STORE, 140614083514368, 140614083518463, ++STORE, 140614083518464, 140614100299775, ++STORE, 140613991260160, 140613999652863, ++SNULL, 140614083518464, 140614091907071, ++STORE, 140614091907072, 140614100299775, ++STORE, 140614083518464, 140614091907071, ++SNULL, 140614091911167, 140614100299775, ++STORE, 140614091907072, 140614091911167, ++STORE, 140614091911168, 140614100299775, ++SNULL, 140613991264255, 140613999652863, ++STORE, 140613991260160, 140613991264255, ++STORE, 140613991264256, 140613999652863, ++STORE, 140613982867456, 140613991260159, ++SNULL, 140613982871551, 140613991260159, ++STORE, 140613982867456, 140613982871551, ++STORE, 140613982871552, 140613991260159, ++STORE, 140613974474752, 140613982867455, ++SNULL, 140613974478847, 140613982867455, ++STORE, 140613974474752, 140613974478847, ++STORE, 140613974478848, 140613982867455, ++STORE, 140613966082048, 140613974474751, ++STORE, 140613739745280, 140613873827839, ++SNULL, 140613739745280, 140613806718975, ++STORE, 140613806718976, 140613873827839, ++STORE, 140613739745280, 140613806718975, ++SNULL, 140613806854143, 140613873827839, ++STORE, 140613806718976, 140613806854143, ++STORE, 140613806854144, 140613873827839, ++SNULL, 140613966086143, 140613974474751, ++STORE, 140613966082048, 140613966086143, ++STORE, 140613966086144, 140613974474751, ++STORE, 140613957689344, 140613966082047, ++STORE, 140613605392384, 140613739610111, ++STORE, 140613949296640, 140613966082047, ++STORE, 140613596999680, 140613605392383, ++STORE, 140613529890816, 140613596999679, ++STORE, 140613521498112, 140613529890815, ++STORE, 140613513105408, 140613529890815, ++STORE, 140613378887680, 140613513105407, ++SNULL, 140613378887680, 140613404065791, ++STORE, 140613404065792, 140613513105407, ++STORE, 140613378887680, 140613404065791, ++ERASE, 140613378887680, 140613404065791, ++STORE, 140613395673088, 140613404065791, ++STORE, 140613261455360, 140613395673087, ++SNULL, 140613261455360, 140613269848063, ++STORE, 140613269848064, 140613395673087, ++STORE, 140613261455360, 140613269848063, ++ERASE, 140613261455360, 140613269848063, ++STORE, 140613261455360, 140613269848063, ++STORE, 140613253062656, 140613269848063, ++STORE, 140613118844928, 140613253062655, ++STORE, 140613110452224, 140613118844927, ++SNULL, 140613118844928, 140613135630335, ++STORE, 140613135630336, 140613253062655, ++STORE, 140613118844928, 140613135630335, ++ERASE, 140613118844928, 140613135630335, ++STORE, 140613127237632, 140613135630335, ++STORE, 140613110452224, 140613135630335, ++STORE, 140612976234496, 140613110452223, ++STORE, 140612967841792, 140612976234495, ++STORE, 140612833624064, 140612967841791, ++STORE, 140612825231360, 140612833624063, ++STORE, 140612816838656, 140612833624063, ++STORE, 140612682620928, 140612816838655, ++STORE, 140612674228224, 140612682620927, ++SNULL, 140612682620928, 140612732977151, ++STORE, 140612732977152, 140612816838655, ++STORE, 140612682620928, 140612732977151, ++ERASE, 140612682620928, 140612732977151, ++SNULL, 140613672501247, 140613739610111, ++STORE, 140613605392384, 140613672501247, ++STORE, 140613672501248, 140613739610111, ++ERASE, 140613672501248, 140613739610111, ++SNULL, 140613605527551, 140613672501247, ++STORE, 140613605392384, 140613605527551, ++STORE, 140613605527552, 140613672501247, ++ERASE, 140613529890816, 140613596999679, ++STORE, 140612540010496, 140612674228223, ++SNULL, 140612540010496, 140612598759423, ++STORE, 140612598759424, 140612674228223, ++STORE, 140612540010496, 140612598759423, ++ERASE, 140612540010496, 140612598759423, ++SNULL, 140613471174655, 140613513105407, ++STORE, 140613404065792, 140613471174655, ++STORE, 140613471174656, 140613513105407, ++ERASE, 140613471174656, 140613513105407, ++SNULL, 140613404200959, 140613471174655, ++STORE, 140613404065792, 140613404200959, ++STORE, 140613404200960, 140613471174655, ++SNULL, 140613336956927, 140613395673087, ++STORE, 140613269848064, 140613336956927, ++STORE, 140613336956928, 140613395673087, ++ERASE, 140613336956928, 140613395673087, ++SNULL, 140612833624064, 140612867194879, ++STORE, 140612867194880, 140612967841791, ++STORE, 140612833624064, 140612867194879, ++ERASE, 140612833624064, 140612867194879, ++SNULL, 140612976234496, 140613001412607, ++STORE, 140613001412608, 140613110452223, ++STORE, 140612976234496, 140613001412607, ++ERASE, 140612976234496, 140613001412607, ++SNULL, 140613202739199, 140613253062655, ++STORE, 140613135630336, 140613202739199, ++STORE, 140613202739200, 140613253062655, ++ERASE, 140613202739200, 140613253062655, ++SNULL, 140613135765503, 140613202739199, ++STORE, 140613135630336, 140613135765503, ++STORE, 140613135765504, 140613202739199, ++SNULL, 140612816842751, 140612833624063, ++STORE, 140612816838656, 140612816842751, ++STORE, 140612816842752, 140612833624063, ++SNULL, 140613110456319, 140613135630335, ++STORE, 140613110452224, 140613110456319, ++STORE, 140613110456320, 140613135630335, ++SNULL, 140613949300735, 140613966082047, ++STORE, 140613949296640, 140613949300735, ++STORE, 140613949300736, 140613966082047, ++SNULL, 140613110456320, 140613118844927, ++STORE, 140613118844928, 140613135630335, ++STORE, 140613110456320, 140613118844927, ++SNULL, 140613118849023, 140613135630335, ++STORE, 140613118844928, 140613118849023, ++STORE, 140613118849024, 140613135630335, ++SNULL, 140612800086015, 140612816838655, ++STORE, 140612732977152, 140612800086015, ++STORE, 140612800086016, 140612816838655, ++ERASE, 140612800086016, 140612816838655, ++SNULL, 140613253062656, 140613261455359, ++STORE, 140613261455360, 140613269848063, ++STORE, 140613253062656, 140613261455359, ++SNULL, 140613261459455, 140613269848063, ++STORE, 140613261455360, 140613261459455, ++STORE, 140613261459456, 140613269848063, ++SNULL, 140612674232319, 140612682620927, ++STORE, 140612674228224, 140612674232319, ++STORE, 140612674232320, 140612682620927, ++STORE, 140613731217408, 140613739610111, ++STORE, 140613722824704, 140613739610111, ++SNULL, 140613949300736, 140613957689343, ++STORE, 140613957689344, 140613966082047, ++STORE, 140613949300736, 140613957689343, ++SNULL, 140613957693439, 140613966082047, ++STORE, 140613957689344, 140613957693439, ++STORE, 140613957693440, 140613966082047, ++STORE, 140612464541696, 140612674228223, ++SNULL, 140612531650559, 140612674228223, ++STORE, 140612464541696, 140612531650559, ++STORE, 140612531650560, 140612674228223, ++SNULL, 140612531650560, 140612598759423, ++STORE, 140612598759424, 140612674228223, ++STORE, 140612531650560, 140612598759423, ++ERASE, 140612531650560, 140612598759423, ++SNULL, 140612665868287, 140612674228223, ++STORE, 140612598759424, 140612665868287, ++STORE, 140612665868288, 140612674228223, ++ERASE, 140612665868288, 140612674228223, ++SNULL, 140613269983231, 140613336956927, ++STORE, 140613269848064, 140613269983231, ++STORE, 140613269983232, 140613336956927, ++SNULL, 140612934303743, 140612967841791, ++STORE, 140612867194880, 140612934303743, ++STORE, 140612934303744, 140612967841791, ++ERASE, 140612934303744, 140612967841791, ++SNULL, 140613068521471, 140613110452223, ++STORE, 140613001412608, 140613068521471, ++STORE, 140613068521472, 140613110452223, ++ERASE, 140613068521472, 140613110452223, ++STORE, 140613714432000, 140613739610111, ++SNULL, 140613001547775, 140613068521471, ++STORE, 140613001412608, 140613001547775, ++STORE, 140613001547776, 140613068521471, ++SNULL, 140612733112319, 140612800086015, ++STORE, 140612732977152, 140612733112319, ++STORE, 140612733112320, 140612800086015, ++SNULL, 140613513109503, 140613529890815, ++STORE, 140613513105408, 140613513109503, ++STORE, 140613513109504, 140613529890815, ++STORE, 140613706039296, 140613739610111, ++STORE, 140613697646592, 140613739610111, ++STORE, 140613689253888, 140613739610111, ++SNULL, 140613689257983, 140613739610111, ++STORE, 140613689253888, 140613689257983, ++STORE, 140613689257984, 140613739610111, ++SNULL, 140613253066751, 140613261455359, ++STORE, 140613253062656, 140613253066751, ++STORE, 140613253066752, 140613261455359, ++STORE, 140613680861184, 140613689253887, ++STORE, 140613588606976, 140613605392383, ++SNULL, 140613689257984, 140613731217407, ++STORE, 140613731217408, 140613739610111, ++STORE, 140613689257984, 140613731217407, ++SNULL, 140613731221503, 140613739610111, ++STORE, 140613731217408, 140613731221503, ++STORE, 140613731221504, 140613739610111, ++STORE, 140613580214272, 140613605392383, ++SNULL, 140612464676863, 140612531650559, ++STORE, 140612464541696, 140612464676863, ++STORE, 140612464676864, 140612531650559, ++SNULL, 140612598894591, 140612665868287, ++STORE, 140612598759424, 140612598894591, ++STORE, 140612598894592, 140612665868287, ++SNULL, 140612867330047, 140612934303743, ++STORE, 140612867194880, 140612867330047, ++STORE, 140612867330048, 140612934303743, ++STORE, 140613571821568, 140613605392383, ++SNULL, 140613571825663, 140613605392383, ++STORE, 140613571821568, 140613571825663, ++STORE, 140613571825664, 140613605392383, ++SNULL, 140613689257984, 140613722824703, ++STORE, 140613722824704, 140613731217407, ++STORE, 140613689257984, 140613722824703, ++SNULL, 140613722828799, 140613731217407, ++STORE, 140613722824704, 140613722828799, ++STORE, 140613722828800, 140613731217407, ++SNULL, 140613689257984, 140613714431999, ++STORE, 140613714432000, 140613722824703, ++STORE, 140613689257984, 140613714431999, ++SNULL, 140613714436095, 140613722824703, ++STORE, 140613714432000, 140613714436095, ++STORE, 140613714436096, 140613722824703, ++SNULL, 140612816842752, 140612825231359, ++STORE, 140612825231360, 140612833624063, ++STORE, 140612816842752, 140612825231359, ++SNULL, 140612825235455, 140612833624063, ++STORE, 140612825231360, 140612825235455, ++STORE, 140612825235456, 140612833624063, ++SNULL, 140613395677183, 140613404065791, ++STORE, 140613395673088, 140613395677183, ++STORE, 140613395677184, 140613404065791, ++SNULL, 140613689257984, 140613706039295, ++STORE, 140613706039296, 140613714431999, ++STORE, 140613689257984, 140613706039295, ++SNULL, 140613706043391, 140613714431999, ++STORE, 140613706039296, 140613706043391, ++STORE, 140613706043392, 140613714431999, ++SNULL, 140613118849024, 140613127237631, ++STORE, 140613127237632, 140613135630335, ++STORE, 140613118849024, 140613127237631, ++SNULL, 140613127241727, 140613135630335, ++STORE, 140613127237632, 140613127241727, ++STORE, 140613127241728, 140613135630335, ++SNULL, 140613571825664, 140613580214271, ++STORE, 140613580214272, 140613605392383, ++STORE, 140613571825664, 140613580214271, ++SNULL, 140613580218367, 140613605392383, ++STORE, 140613580214272, 140613580218367, ++STORE, 140613580218368, 140613605392383, ++SNULL, 140613689257984, 140613697646591, ++STORE, 140613697646592, 140613706039295, ++STORE, 140613689257984, 140613697646591, ++SNULL, 140613697650687, 140613706039295, ++STORE, 140613697646592, 140613697650687, ++STORE, 140613697650688, 140613706039295, ++SNULL, 140613680865279, 140613689253887, ++STORE, 140613680861184, 140613680865279, ++STORE, 140613680865280, 140613689253887, ++STORE, 140613563428864, 140613571821567, ++SNULL, 140613563432959, 140613571821567, ++STORE, 140613563428864, 140613563432959, ++STORE, 140613563432960, 140613571821567, ++SNULL, 140613580218368, 140613588606975, ++STORE, 140613588606976, 140613605392383, ++STORE, 140613580218368, 140613588606975, ++SNULL, 140613588611071, 140613605392383, ++STORE, 140613588606976, 140613588611071, ++STORE, 140613588611072, 140613605392383, ++SNULL, 140613513109504, 140613521498111, ++STORE, 140613521498112, 140613529890815, ++STORE, 140613513109504, 140613521498111, ++SNULL, 140613521502207, 140613529890815, ++STORE, 140613521498112, 140613521502207, ++STORE, 140613521502208, 140613529890815, ++SNULL, 140613588611072, 140613596999679, ++STORE, 140613596999680, 140613605392383, ++STORE, 140613588611072, 140613596999679, ++SNULL, 140613597003775, 140613605392383, ++STORE, 140613596999680, 140613597003775, ++STORE, 140613597003776, 140613605392383, ++STORE, 140613555036160, 140613563428863, ++SNULL, 140613555040255, 140613563428863, ++STORE, 140613555036160, 140613555040255, ++STORE, 140613555040256, 140613563428863, ++STORE, 140613546643456, 140613555036159, ++STORE, 140613538250752, 140613555036159, ++SNULL, 140613538250752, 140613546643455, ++STORE, 140613546643456, 140613555036159, ++STORE, 140613538250752, 140613546643455, ++SNULL, 140613546647551, 140613555036159, ++STORE, 140613546643456, 140613546647551, ++STORE, 140613546647552, 140613555036159, ++STORE, 140613504712704, 140613513105407, ++STORE, 140613496320000, 140613513105407, ++SNULL, 140613496324095, 140613513105407, ++STORE, 140613496320000, 140613496324095, ++STORE, 140613496324096, 140613513105407, ++STORE, 140613487927296, 140613496319999, ++SNULL, 140613487931391, 140613496319999, ++STORE, 140613487927296, 140613487931391, ++STORE, 140613487931392, 140613496319999, ++STORE, 140613479534592, 140613487927295, ++SNULL, 140612967845887, 140612976234495, ++STORE, 140612967841792, 140612967845887, ++STORE, 140612967845888, 140612976234495, ++STORE, 140613387280384, 140613395673087, ++STORE, 140613378887680, 140613395673087, ++SNULL, 140613378887680, 140613387280383, ++STORE, 140613387280384, 140613395673087, ++STORE, 140613378887680, 140613387280383, ++SNULL, 140613387284479, 140613395673087, ++STORE, 140613387280384, 140613387284479, ++STORE, 140613387284480, 140613395673087, ++STORE, 140613370494976, 140613387280383, ++STORE, 140613362102272, 140613387280383, ++SNULL, 140613479538687, 140613487927295, ++STORE, 140613479534592, 140613479538687, ++STORE, 140613479538688, 140613487927295, ++STORE, 140613353709568, 140613387280383, ++STORE, 140613345316864, 140613387280383, ++STORE, 140613244669952, 140613253062655, ++SNULL, 140613345320959, 140613387280383, ++STORE, 140613345316864, 140613345320959, ++STORE, 140613345320960, 140613387280383, ++SNULL, 140613538254847, 140613546643455, ++STORE, 140613538250752, 140613538254847, ++STORE, 140613538254848, 140613546643455, ++STORE, 140613236277248, 140613253062655, ++STORE, 140613227884544, 140613253062655, ++STORE, 140613219491840, 140613253062655, ++STORE, 140613211099136, 140613253062655, ++SNULL, 140613211103231, 140613253062655, ++STORE, 140613211099136, 140613211103231, ++STORE, 140613211103232, 140613253062655, ++STORE, 140613102059520, 140613110452223, ++STORE, 140613093666816, 140613110452223, ++SNULL, 140613093670911, 140613110452223, ++STORE, 140613093666816, 140613093670911, ++STORE, 140613093670912, 140613110452223, ++STORE, 140613085274112, 140613093666815, ++SNULL, 140613496324096, 140613504712703, ++STORE, 140613504712704, 140613513105407, ++STORE, 140613496324096, 140613504712703, ++SNULL, 140613504716799, 140613513105407, ++STORE, 140613504712704, 140613504716799, ++STORE, 140613504716800, 140613513105407, ++SNULL, 140613345320960, 140613378887679, ++STORE, 140613378887680, 140613387280383, ++STORE, 140613345320960, 140613378887679, ++SNULL, 140613378891775, 140613387280383, ++STORE, 140613378887680, 140613378891775, ++STORE, 140613378891776, 140613387280383, ++SNULL, 140613345320960, 140613362102271, ++STORE, 140613362102272, 140613378887679, ++STORE, 140613345320960, 140613362102271, ++SNULL, 140613362106367, 140613378887679, ++STORE, 140613362102272, 140613362106367, ++STORE, 140613362106368, 140613378887679, ++SNULL, 140613362106368, 140613370494975, ++STORE, 140613370494976, 140613378887679, ++STORE, 140613362106368, 140613370494975, ++SNULL, 140613370499071, 140613378887679, ++STORE, 140613370494976, 140613370499071, ++STORE, 140613370499072, 140613378887679, ++STORE, 140613076881408, 140613093666815, ++STORE, 140612993019904, 140613001412607, ++SNULL, 140613076885503, 140613093666815, ++STORE, 140613076881408, 140613076885503, ++STORE, 140613076885504, 140613093666815, ++SNULL, 140613093670912, 140613102059519, ++STORE, 140613102059520, 140613110452223, ++STORE, 140613093670912, 140613102059519, ++SNULL, 140613102063615, 140613110452223, ++STORE, 140613102059520, 140613102063615, ++STORE, 140613102063616, 140613110452223, ++SNULL, 140613076885504, 140613085274111, ++STORE, 140613085274112, 140613093666815, ++STORE, 140613076885504, 140613085274111, ++SNULL, 140613085278207, 140613093666815, ++STORE, 140613085274112, 140613085278207, ++STORE, 140613085278208, 140613093666815, ++STORE, 140612984627200, 140613001412607, ++STORE, 140612967845888, 140612984627199, ++SNULL, 140613211103232, 140613219491839, ++STORE, 140613219491840, 140613253062655, ++STORE, 140613211103232, 140613219491839, ++SNULL, 140613219495935, 140613253062655, ++STORE, 140613219491840, 140613219495935, ++STORE, 140613219495936, 140613253062655, ++STORE, 140612959449088, 140612967841791, ++STORE, 140612951056384, 140612967841791, ++SNULL, 140612951060479, 140612967841791, ++STORE, 140612951056384, 140612951060479, ++STORE, 140612951060480, 140612967841791, ++SNULL, 140613345320960, 140613353709567, ++STORE, 140613353709568, 140613362102271, ++STORE, 140613345320960, 140613353709567, ++SNULL, 140613353713663, 140613362102271, ++STORE, 140613353709568, 140613353713663, ++STORE, 140613353713664, 140613362102271, ++SNULL, 140613219495936, 140613244669951, ++STORE, 140613244669952, 140613253062655, ++STORE, 140613219495936, 140613244669951, ++SNULL, 140613244674047, 140613253062655, ++STORE, 140613244669952, 140613244674047, ++STORE, 140613244674048, 140613253062655, ++STORE, 140612942663680, 140612951056383, ++SNULL, 140613219495936, 140613236277247, ++STORE, 140613236277248, 140613244669951, ++STORE, 140613219495936, 140613236277247, ++SNULL, 140613236281343, 140613244669951, ++STORE, 140613236277248, 140613236281343, ++STORE, 140613236281344, 140613244669951, ++SNULL, 140613219495936, 140613227884543, ++STORE, 140613227884544, 140613236277247, ++STORE, 140613219495936, 140613227884543, ++SNULL, 140613227888639, 140613236277247, ++STORE, 140613227884544, 140613227888639, ++STORE, 140613227888640, 140613236277247, ++SNULL, 140612984627200, 140612993019903, ++STORE, 140612993019904, 140613001412607, ++STORE, 140612984627200, 140612993019903, ++SNULL, 140612993023999, 140613001412607, ++STORE, 140612993019904, 140612993023999, ++STORE, 140612993024000, 140613001412607, ++STORE, 140612858802176, 140612867194879, ++STORE, 140612850409472, 140612867194879, ++SNULL, 140612951060480, 140612959449087, ++STORE, 140612959449088, 140612967841791, ++STORE, 140612951060480, 140612959449087, ++SNULL, 140612959453183, 140612967841791, ++STORE, 140612959449088, 140612959453183, ++STORE, 140612959453184, 140612967841791, ++SNULL, 140612967845888, 140612976234495, ++STORE, 140612976234496, 140612984627199, ++STORE, 140612967845888, 140612976234495, ++SNULL, 140612976238591, 140612984627199, ++STORE, 140612976234496, 140612976238591, ++STORE, 140612976238592, 140612984627199, ++STORE, 140612842016768, 140612867194879, ++SNULL, 140612842020863, 140612867194879, ++STORE, 140612842016768, 140612842020863, ++STORE, 140612842020864, 140612867194879, ++SNULL, 140612984631295, 140612993019903, ++STORE, 140612984627200, 140612984631295, ++STORE, 140612984631296, 140612993019903, ++STORE, 140612825235456, 140612842016767, ++STORE, 140612808445952, 140612816838655, ++SNULL, 140612942667775, 140612951056383, ++STORE, 140612942663680, 140612942667775, ++STORE, 140612942667776, 140612951056383, ++STORE, 140612724584448, 140612732977151, ++SNULL, 140612724588543, 140612732977151, ++STORE, 140612724584448, 140612724588543, ++STORE, 140612724588544, 140612732977151, ++STORE, 140612716191744, 140612724584447, ++SNULL, 140612842020864, 140612850409471, ++STORE, 140612850409472, 140612867194879, ++STORE, 140612842020864, 140612850409471, ++SNULL, 140612850413567, 140612867194879, ++STORE, 140612850409472, 140612850413567, ++STORE, 140612850413568, 140612867194879, ++SNULL, 140612850413568, 140612858802175, ++STORE, 140612858802176, 140612867194879, ++STORE, 140612850413568, 140612858802175, ++SNULL, 140612858806271, 140612867194879, ++STORE, 140612858802176, 140612858806271, ++STORE, 140612858806272, 140612867194879, ++STORE, 140612707799040, 140612724584447, ++SNULL, 140612707803135, 140612724584447, ++STORE, 140612707799040, 140612707803135, ++STORE, 140612707803136, 140612724584447, ++SNULL, 140612707803136, 140612716191743, ++STORE, 140612716191744, 140612724584447, ++STORE, 140612707803136, 140612716191743, ++SNULL, 140612716195839, 140612724584447, ++STORE, 140612716191744, 140612716195839, ++STORE, 140612716195840, 140612724584447, ++SNULL, 140612808450047, 140612816838655, ++STORE, 140612808445952, 140612808450047, ++STORE, 140612808450048, 140612816838655, ++SNULL, 140612825235456, 140612833624063, ++STORE, 140612833624064, 140612842016767, ++STORE, 140612825235456, 140612833624063, ++SNULL, 140612833628159, 140612842016767, ++STORE, 140612833624064, 140612833628159, ++STORE, 140612833628160, 140612842016767, ++STORE, 140612699406336, 140612707799039, ++SNULL, 140612699410431, 140612707799039, ++STORE, 140612699406336, 140612699410431, ++STORE, 140612699410432, 140612707799039, ++STORE, 140614384926720, 140614384955391, ++STORE, 140614349332480, 140614351523839, ++SNULL, 140614349332480, 140614349422591, ++STORE, 140614349422592, 140614351523839, ++STORE, 140614349332480, 140614349422591, ++SNULL, 140614351515647, 140614351523839, ++STORE, 140614349422592, 140614351515647, ++STORE, 140614351515648, 140614351523839, ++ERASE, 140614351515648, 140614351523839, ++STORE, 140614351515648, 140614351523839, ++SNULL, 140614351519743, 140614351523839, ++STORE, 140614351515648, 140614351519743, ++STORE, 140614351519744, 140614351523839, ++ERASE, 140614384926720, 140614384955391, ++ERASE, 140613949296640, 140613949300735, ++ERASE, 140613949300736, 140613957689343, ++ERASE, 140613689253888, 140613689257983, ++ERASE, 140613689257984, 140613697646591, ++ERASE, 140613563428864, 140613563432959, ++ERASE, 140613563432960, 140613571821567, ++ERASE, 140613211099136, 140613211103231, ++ERASE, 140613211103232, 140613219491839, ++ERASE, 140614133870592, 140614133874687, ++ERASE, 140614133874688, 140614142263295, ++ERASE, 140612967841792, 140612967845887, ++ERASE, 140612967845888, 140612976234495, ++ERASE, 140613076881408, 140613076885503, ++ERASE, 140613076885504, 140613085274111, ++ERASE, 140612850409472, 140612850413567, ++ERASE, 140612850413568, 140612858802175, ++ERASE, 140613110452224, 140613110456319, ++ERASE, 140613110456320, 140613118844927, ++ERASE, 140613706039296, 140613706043391, ++ERASE, 140613706043392, 140613714431999, ++ERASE, 140613521498112, 140613521502207, ++ERASE, 140613521502208, 140613529890815, ++ERASE, 140613362102272, 140613362106367, ++ERASE, 140613362106368, 140613370494975, ++ERASE, 140613253062656, 140613253066751, ++ERASE, 140613253066752, 140613261455359, ++ERASE, 140612816838656, 140612816842751, ++ERASE, 140612816842752, 140612825231359, ++ERASE, 140613261455360, 140613261459455, ++ERASE, 140613261459456, 140613269848063, ++ERASE, 140613118844928, 140613118849023, ++ERASE, 140613118849024, 140613127237631, ++ERASE, 140613714432000, 140613714436095, ++ERASE, 140613714436096, 140613722824703, ++ERASE, 140613496320000, 140613496324095, ++ERASE, 140613496324096, 140613504712703, ++ERASE, 140613513105408, 140613513109503, ++ERASE, 140613513109504, 140613521498111, ++ERASE, 140613697646592, 140613697650687, ++ERASE, 140613697650688, 140613706039295, ++ERASE, 140613093666816, 140613093670911, ++ERASE, 140613093670912, 140613102059519, ++ERASE, 140612993019904, 140612993023999, ++ERASE, 140612993024000, 140613001412607, ++ERASE, 140613127237632, 140613127241727, ++ERASE, 140613127241728, 140613135630335, ++ERASE, 140613957689344, 140613957693439, ++ERASE, 140613957693440, 140613966082047, ++ERASE, 140613571821568, 140613571825663, ++ERASE, 140613571825664, 140613580214271, ++ERASE, 140613479534592, 140613479538687, ++ERASE, 140613479538688, 140613487927295, ++ERASE, 140612984627200, 140612984631295, ++ERASE, 140612984631296, 140612993019903, ++ERASE, 140613588606976, 140613588611071, ++ERASE, 140613588611072, 140613596999679, ++ERASE, 140613680861184, 140613680865279, ++ERASE, 140613680865280, 140613689253887, ++ERASE, 140613345316864, 140613345320959, ++ERASE, 140613345320960, 140613353709567, ++ERASE, 140613596999680, 140613597003775, ++ERASE, 140613597003776, 140613605392383, ++ERASE, 140613966082048, 140613966086143, ++ERASE, 140613966086144, 140613974474751, ++ERASE, 140613731217408, 140613731221503, ++ERASE, 140613731221504, 140613739610111, ++ERASE, 140613395673088, 140613395677183, ++ERASE, 140613395677184, 140613404065791, ++ERASE, 140612825231360, 140612825235455, ++ERASE, 140612825235456, 140612833624063, ++ERASE, 140612674228224, 140612674232319, ++ERASE, 140612674232320, 140612682620927, ++ERASE, 140613722824704, 140613722828799, ++ERASE, 140613722828800, 140613731217407, ++ERASE, 140613487927296, 140613487931391, ++ERASE, 140613487931392, 140613496319999, ++ERASE, 140613102059520, 140613102063615, ++ERASE, 140613102063616, 140613110452223, ++ERASE, 140614242910208, 140614242914303, ++ERASE, 140614242914304, 140614251302911, ++ERASE, 140612808445952, 140612808450047, ++ERASE, 140612808450048, 140612816838655, ++ERASE, 140613236277248, 140613236281343, ++ERASE, 140613236281344, 140613244669951, ++ERASE, 140613580214272, 140613580218367, ++ERASE, 140613580218368, 140613588606975, ++ERASE, 140613370494976, 140613370499071, ++ERASE, 140613370499072, 140613378887679, ++ERASE, 140613244669952, 140613244674047, ++ERASE, 140613244674048, 140613253062655, ++ERASE, 140612724584448, 140612724588543, ++ERASE, 140612724588544, 140612732977151, ++ERASE, 140612707799040, 140612707803135, ++ERASE, 140612707803136, 140612716191743, ++ERASE, 140613504712704, 140613504716799, ++ERASE, 140613504716800, 140613513105407, ++ }; ++ ++ unsigned long set39[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140736271417344, 140737488351231, ++SNULL, 140736271421439, 140737488351231, ++STORE, 140736271417344, 140736271421439, ++STORE, 140736271286272, 140736271421439, ++STORE, 94412930822144, 94412933074943, ++SNULL, 94412930953215, 94412933074943, ++STORE, 94412930822144, 94412930953215, ++STORE, 94412930953216, 94412933074943, ++ERASE, 94412930953216, 94412933074943, ++STORE, 94412933046272, 94412933054463, ++STORE, 94412933054464, 94412933074943, ++STORE, 140326136901632, 140326139154431, ++SNULL, 140326137044991, 140326139154431, ++STORE, 140326136901632, 140326137044991, ++STORE, 140326137044992, 140326139154431, ++ERASE, 140326137044992, 140326139154431, ++STORE, 140326139142144, 140326139150335, ++STORE, 140326139150336, 140326139154431, ++STORE, 140736271585280, 140736271589375, ++STORE, 140736271572992, 140736271585279, ++STORE, 140326139113472, 140326139142143, ++STORE, 140326139105280, 140326139113471, ++STORE, 140326134685696, 140326136901631, ++SNULL, 140326134685696, 140326134783999, ++STORE, 140326134784000, 140326136901631, ++STORE, 140326134685696, 140326134783999, ++SNULL, 140326136877055, 140326136901631, ++STORE, 140326134784000, 140326136877055, ++STORE, 140326136877056, 140326136901631, ++SNULL, 140326136877056, 140326136885247, ++STORE, 140326136885248, 140326136901631, ++STORE, 140326136877056, 140326136885247, ++ERASE, 140326136877056, 140326136885247, ++STORE, 140326136877056, 140326136885247, ++ERASE, 140326136885248, 140326136901631, ++STORE, 140326136885248, 140326136901631, ++STORE, 140326130888704, 140326134685695, ++SNULL, 140326130888704, 140326132547583, ++STORE, 140326132547584, 140326134685695, ++STORE, 140326130888704, 140326132547583, ++SNULL, 140326134644735, 140326134685695, ++STORE, 140326132547584, 140326134644735, ++STORE, 140326134644736, 140326134685695, ++SNULL, 140326134644736, 140326134669311, ++STORE, 140326134669312, 140326134685695, ++STORE, 140326134644736, 140326134669311, ++ERASE, 140326134644736, 140326134669311, ++STORE, 140326134644736, 140326134669311, ++ERASE, 140326134669312, 140326134685695, ++STORE, 140326134669312, 140326134685695, ++STORE, 140326139097088, 140326139113471, ++SNULL, 140326134661119, 140326134669311, ++STORE, 140326134644736, 140326134661119, ++STORE, 140326134661120, 140326134669311, ++SNULL, 140326136881151, 140326136885247, ++STORE, 140326136877056, 140326136881151, ++STORE, 140326136881152, 140326136885247, ++SNULL, 94412933050367, 94412933054463, ++STORE, 94412933046272, 94412933050367, ++STORE, 94412933050368, 94412933054463, ++SNULL, 140326139146239, 140326139150335, ++STORE, 140326139142144, 140326139146239, ++STORE, 140326139146240, 140326139150335, ++ERASE, 140326139113472, 140326139142143, ++STORE, 94412939493376, 94412939628543, ++STORE, 140326122496000, 140326130888703, ++SNULL, 140326122500095, 140326130888703, ++STORE, 140326122496000, 140326122500095, ++STORE, 140326122500096, 140326130888703, ++STORE, 140326114103296, 140326122495999, ++STORE, 140325979885568, 140326114103295, ++SNULL, 140325979885568, 140326043910143, ++STORE, 140326043910144, 140326114103295, ++STORE, 140325979885568, 140326043910143, ++ERASE, 140325979885568, 140326043910143, ++SNULL, 140326111019007, 140326114103295, ++STORE, 140326043910144, 140326111019007, ++STORE, 140326111019008, 140326114103295, ++ERASE, 140326111019008, 140326114103295, ++SNULL, 140326044045311, 140326111019007, ++STORE, 140326043910144, 140326044045311, ++STORE, 140326044045312, 140326111019007, ++SNULL, 140326114107391, 140326122495999, ++STORE, 140326114103296, 140326114107391, ++STORE, 140326114107392, 140326122495999, ++STORE, 140326035517440, 140326043910143, ++SNULL, 140326035521535, 140326043910143, ++STORE, 140326035517440, 140326035521535, ++STORE, 140326035521536, 140326043910143, ++STORE, 140326027124736, 140326035517439, ++SNULL, 140326027128831, 140326035517439, ++STORE, 140326027124736, 140326027128831, ++STORE, 140326027128832, 140326035517439, ++STORE, 140326018732032, 140326027124735, ++SNULL, 140326018736127, 140326027124735, ++STORE, 140326018732032, 140326018736127, ++STORE, 140326018736128, 140326027124735, ++STORE, 140326010339328, 140326018732031, ++STORE, 140326001946624, 140326018732031, ++STORE, 140325993553920, 140326018732031, ++STORE, 140325859336192, 140325993553919, ++SNULL, 140325859336192, 140325909692415, ++STORE, 140325909692416, 140325993553919, ++STORE, 140325859336192, 140325909692415, ++ERASE, 140325859336192, 140325909692415, ++SNULL, 140325976801279, 140325993553919, ++STORE, 140325909692416, 140325976801279, ++STORE, 140325976801280, 140325993553919, ++ERASE, 140325976801280, 140325993553919, ++STORE, 140325985161216, 140326018732031, ++STORE, 140325775474688, 140325976801279, ++STORE, 140325708365824, 140325976801279, ++SNULL, 140325708500991, 140325976801279, ++STORE, 140325708365824, 140325708500991, ++STORE, 140325708500992, 140325976801279, ++SNULL, 140325708500992, 140325909692415, ++STORE, 140325909692416, 140325976801279, ++STORE, 140325708500992, 140325909692415, ++SNULL, 140325909827583, 140325976801279, ++STORE, 140325909692416, 140325909827583, ++STORE, 140325909827584, 140325976801279, ++SNULL, 140325842583551, 140325909692415, ++STORE, 140325708500992, 140325842583551, ++STORE, 140325842583552, 140325909692415, ++ERASE, 140325842583552, 140325909692415, ++SNULL, 140325708500992, 140325775474687, ++STORE, 140325775474688, 140325842583551, ++STORE, 140325708500992, 140325775474687, ++SNULL, 140325775609855, 140325842583551, ++STORE, 140325775474688, 140325775609855, ++STORE, 140325775609856, 140325842583551, ++STORE, 140325775609856, 140325909692415, ++SNULL, 140325775609856, 140325842583551, ++STORE, 140325842583552, 140325909692415, ++STORE, 140325775609856, 140325842583551, ++SNULL, 140325842718719, 140325909692415, ++STORE, 140325842583552, 140325842718719, ++STORE, 140325842718720, 140325909692415, ++SNULL, 140325985161216, 140325993553919, ++STORE, 140325993553920, 140326018732031, ++STORE, 140325985161216, 140325993553919, ++SNULL, 140325993558015, 140326018732031, ++STORE, 140325993553920, 140325993558015, ++STORE, 140325993558016, 140326018732031, ++SNULL, 140325985165311, 140325993553919, ++STORE, 140325985161216, 140325985165311, ++STORE, 140325985165312, 140325993553919, ++SNULL, 140325993558016, 140326001946623, ++STORE, 140326001946624, 140326018732031, ++STORE, 140325993558016, 140326001946623, ++SNULL, 140326001950719, 140326018732031, ++STORE, 140326001946624, 140326001950719, ++STORE, 140326001950720, 140326018732031, ++SNULL, 140326001950720, 140326010339327, ++STORE, 140326010339328, 140326018732031, ++STORE, 140326001950720, 140326010339327, ++SNULL, 140326010343423, 140326018732031, ++STORE, 140326010339328, 140326010343423, ++STORE, 140326010343424, 140326018732031, ++STORE, 140325699973120, 140325708365823, ++STORE, 140325691580416, 140325708365823, ++STORE, 140325683187712, 140325708365823, ++SNULL, 140325683191807, 140325708365823, ++STORE, 140325683187712, 140325683191807, ++STORE, 140325683191808, 140325708365823, ++SNULL, 140325683191808, 140325699973119, ++STORE, 140325699973120, 140325708365823, ++STORE, 140325683191808, 140325699973119, ++SNULL, 140325699977215, 140325708365823, ++STORE, 140325699973120, 140325699977215, ++STORE, 140325699977216, 140325708365823, ++STORE, 140325674795008, 140325683187711, ++STORE, 140325666402304, 140325683187711, ++STORE, 140325658009600, 140325683187711, ++SNULL, 140325658009600, 140325666402303, ++STORE, 140325666402304, 140325683187711, ++STORE, 140325658009600, 140325666402303, ++SNULL, 140325666406399, 140325683187711, ++STORE, 140325666402304, 140325666406399, ++STORE, 140325666406400, 140325683187711, ++SNULL, 140325683191808, 140325691580415, ++STORE, 140325691580416, 140325699973119, ++STORE, 140325683191808, 140325691580415, ++SNULL, 140325691584511, 140325699973119, ++STORE, 140325691580416, 140325691584511, ++STORE, 140325691584512, 140325699973119, ++SNULL, 140325666406400, 140325674795007, ++STORE, 140325674795008, 140325683187711, ++STORE, 140325666406400, 140325674795007, ++SNULL, 140325674799103, 140325683187711, ++STORE, 140325674795008, 140325674799103, ++STORE, 140325674799104, 140325683187711, ++STORE, 140325649616896, 140325666402303, ++SNULL, 140325649616896, 140325658009599, ++STORE, 140325658009600, 140325666402303, ++STORE, 140325649616896, 140325658009599, ++SNULL, 140325658013695, 140325666402303, ++STORE, 140325658009600, 140325658013695, ++STORE, 140325658013696, 140325666402303, ++SNULL, 140325649620991, 140325658009599, ++STORE, 140325649616896, 140325649620991, ++STORE, 140325649620992, 140325658009599, ++STORE, 140325641224192, 140325649616895, ++STORE, 140325632831488, 140325649616895, ++SNULL, 140325632835583, 140325649616895, ++STORE, 140325632831488, 140325632835583, ++STORE, 140325632835584, 140325649616895, ++STORE, 140325624438784, 140325632831487, ++SNULL, 140325624442879, 140325632831487, ++STORE, 140325624438784, 140325624442879, ++STORE, 140325624442880, 140325632831487, ++SNULL, 140325632835584, 140325641224191, ++STORE, 140325641224192, 140325649616895, ++STORE, 140325632835584, 140325641224191, ++SNULL, 140325641228287, 140325649616895, ++STORE, 140325641224192, 140325641228287, ++STORE, 140325641228288, 140325649616895, ++STORE, 140325616046080, 140325624438783, ++SNULL, 140325616050175, 140325624438783, ++STORE, 140325616046080, 140325616050175, ++STORE, 140325616050176, 140325624438783, ++STORE, 140325607653376, 140325616046079, ++SNULL, 140325607657471, 140325616046079, ++STORE, 140325607653376, 140325607657471, ++STORE, 140325607657472, 140325616046079, ++STORE, 140325599260672, 140325607653375, ++STORE, 140325590867968, 140325607653375, ++STORE, 140325456650240, 140325590867967, ++SNULL, 140325456650240, 140325507039231, ++STORE, 140325507039232, 140325590867967, ++STORE, 140325456650240, 140325507039231, ++ERASE, 140325456650240, 140325507039231, ++STORE, 140325498646528, 140325507039231, ++STORE, 140325364428800, 140325498646527, ++SNULL, 140325364428800, 140325372821503, ++STORE, 140325372821504, 140325498646527, ++STORE, 140325364428800, 140325372821503, ++ERASE, 140325364428800, 140325372821503, ++STORE, 140325364428800, 140325372821503, ++STORE, 140325356036096, 140325372821503, ++STORE, 140325221818368, 140325356036095, ++SNULL, 140325221818368, 140325238603775, ++STORE, 140325238603776, 140325356036095, ++STORE, 140325221818368, 140325238603775, ++ERASE, 140325221818368, 140325238603775, ++STORE, 140325230211072, 140325238603775, ++STORE, 140325221818368, 140325238603775, ++STORE, 140325087600640, 140325221818367, ++STORE, 140325079207936, 140325087600639, ++SNULL, 140325087600640, 140325104386047, ++STORE, 140325104386048, 140325221818367, ++STORE, 140325087600640, 140325104386047, ++ERASE, 140325087600640, 140325104386047, ++STORE, 140325095993344, 140325104386047, ++STORE, 140325079207936, 140325104386047, ++STORE, 140324944990208, 140325079207935, ++SNULL, 140324944990208, 140324970168319, ++STORE, 140324970168320, 140325079207935, ++STORE, 140324944990208, 140324970168319, ++ERASE, 140324944990208, 140324970168319, ++STORE, 140324961775616, 140324970168319, ++STORE, 140324953382912, 140324970168319, ++STORE, 140324819165184, 140324953382911, ++STORE, 140324684947456, 140324953382911, ++STORE, 140324676554752, 140324684947455, ++STORE, 140324668162048, 140324684947455, ++STORE, 140324533944320, 140324668162047, ++STORE, 140324525551616, 140324533944319, ++SNULL, 140324533944320, 140324567515135, ++STORE, 140324567515136, 140324668162047, ++STORE, 140324533944320, 140324567515135, ++ERASE, 140324533944320, 140324567515135, ++STORE, 140324559122432, 140324567515135, ++STORE, 140324391333888, 140324525551615, ++SNULL, 140325574148095, 140325590867967, ++STORE, 140325507039232, 140325574148095, ++STORE, 140325574148096, 140325590867967, ++ERASE, 140325574148096, 140325590867967, ++SNULL, 140325439930367, 140325498646527, ++STORE, 140325372821504, 140325439930367, ++STORE, 140325439930368, 140325498646527, ++ERASE, 140325439930368, 140325498646527, ++SNULL, 140325305712639, 140325356036095, ++STORE, 140325238603776, 140325305712639, ++STORE, 140325305712640, 140325356036095, ++ERASE, 140325305712640, 140325356036095, ++SNULL, 140325171494911, 140325221818367, ++STORE, 140325104386048, 140325171494911, ++STORE, 140325171494912, 140325221818367, ++ERASE, 140325171494912, 140325221818367, ++SNULL, 140325104521215, 140325171494911, ++STORE, 140325104386048, 140325104521215, ++STORE, 140325104521216, 140325171494911, ++STORE, 140324257116160, 140324525551615, ++SNULL, 140324257116160, 140324299079679, ++STORE, 140324299079680, 140324525551615, ++STORE, 140324257116160, 140324299079679, ++ERASE, 140324257116160, 140324299079679, ++SNULL, 140325037277183, 140325079207935, ++STORE, 140324970168320, 140325037277183, ++STORE, 140325037277184, 140325079207935, ++ERASE, 140325037277184, 140325079207935, ++SNULL, 140324819165183, 140324953382911, ++STORE, 140324684947456, 140324819165183, ++STORE, 140324819165184, 140324953382911, ++SNULL, 140324819165184, 140324835950591, ++STORE, 140324835950592, 140324953382911, ++STORE, 140324819165184, 140324835950591, ++ERASE, 140324819165184, 140324835950591, ++SNULL, 140324903059455, 140324953382911, ++STORE, 140324835950592, 140324903059455, ++STORE, 140324903059456, 140324953382911, ++ERASE, 140324903059456, 140324953382911, ++SNULL, 140324684947456, 140324701732863, ++STORE, 140324701732864, 140324819165183, ++STORE, 140324684947456, 140324701732863, ++ERASE, 140324684947456, 140324701732863, ++SNULL, 140324768841727, 140324819165183, ++STORE, 140324701732864, 140324768841727, ++STORE, 140324768841728, 140324819165183, ++ERASE, 140324768841728, 140324819165183, ++SNULL, 140324634623999, 140324668162047, ++STORE, 140324567515136, 140324634623999, ++STORE, 140324634624000, 140324668162047, ++ERASE, 140324634624000, 140324668162047, ++SNULL, 140324391333887, 140324525551615, ++STORE, 140324299079680, 140324391333887, ++STORE, 140324391333888, 140324525551615, ++SNULL, 140324391333888, 140324433297407, ++STORE, 140324433297408, 140324525551615, ++STORE, 140324391333888, 140324433297407, ++ERASE, 140324391333888, 140324433297407, ++SNULL, 140325507174399, 140325574148095, ++STORE, 140325507039232, 140325507174399, ++STORE, 140325507174400, 140325574148095, ++SNULL, 140325590867968, 140325599260671, ++STORE, 140325599260672, 140325607653375, ++STORE, 140325590867968, 140325599260671, ++SNULL, 140325599264767, 140325607653375, ++STORE, 140325599260672, 140325599264767, ++STORE, 140325599264768, 140325607653375, ++SNULL, 140325372956671, 140325439930367, ++STORE, 140325372821504, 140325372956671, ++STORE, 140325372956672, 140325439930367, ++SNULL, 140324668166143, 140324684947455, ++STORE, 140324668162048, 140324668166143, ++STORE, 140324668166144, 140324684947455, ++SNULL, 140324525555711, 140324533944319, ++STORE, 140324525551616, 140324525555711, ++STORE, 140324525555712, 140324533944319, ++SNULL, 140324953382912, 140324961775615, ++STORE, 140324961775616, 140324970168319, ++STORE, 140324953382912, 140324961775615, ++SNULL, 140324961779711, 140324970168319, ++STORE, 140324961775616, 140324961779711, ++STORE, 140324961779712, 140324970168319, ++SNULL, 140325079212031, 140325104386047, ++STORE, 140325079207936, 140325079212031, ++STORE, 140325079212032, 140325104386047, ++SNULL, 140325221818368, 140325230211071, ++STORE, 140325230211072, 140325238603775, ++STORE, 140325221818368, 140325230211071, ++SNULL, 140325230215167, 140325238603775, ++STORE, 140325230211072, 140325230215167, ++STORE, 140325230215168, 140325238603775, ++SNULL, 140325356036096, 140325364428799, ++STORE, 140325364428800, 140325372821503, ++STORE, 140325356036096, 140325364428799, ++SNULL, 140325364432895, 140325372821503, ++ }; ++ unsigned long set40[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734309167104, 140737488351231, ++SNULL, 140734309171199, 140737488351231, ++STORE, 140734309167104, 140734309171199, ++STORE, 140734309036032, 140734309171199, ++STORE, 94270500081664, 94270502334463, ++SNULL, 94270500212735, 94270502334463, ++STORE, 94270500081664, 94270500212735, ++STORE, 94270500212736, 94270502334463, ++ERASE, 94270500212736, 94270502334463, ++STORE, 94270502305792, 94270502313983, ++STORE, 94270502313984, 94270502334463, ++STORE, 140321935110144, 140321937362943, ++SNULL, 140321935253503, 140321937362943, ++STORE, 140321935110144, 140321935253503, ++STORE, 140321935253504, 140321937362943, ++ERASE, 140321935253504, 140321937362943, ++STORE, 140321937350656, 140321937358847, ++STORE, 140321937358848, 140321937362943, ++STORE, 140734309625856, 140734309629951, ++STORE, 140734309613568, 140734309625855, ++STORE, 140321937321984, 140321937350655, ++STORE, 140321937313792, 140321937321983, ++STORE, 140321932894208, 140321935110143, ++SNULL, 140321932894208, 140321932992511, ++STORE, 140321932992512, 140321935110143, ++STORE, 140321932894208, 140321932992511, ++SNULL, 140321935085567, 140321935110143, ++STORE, 140321932992512, 140321935085567, ++STORE, 140321935085568, 140321935110143, ++SNULL, 140321935085568, 140321935093759, ++STORE, 140321935093760, 140321935110143, ++STORE, 140321935085568, 140321935093759, ++ERASE, 140321935085568, 140321935093759, ++STORE, 140321935085568, 140321935093759, ++ERASE, 140321935093760, 140321935110143, ++STORE, 140321935093760, 140321935110143, ++STORE, 140321929097216, 140321932894207, ++SNULL, 140321929097216, 140321930756095, ++STORE, 140321930756096, 140321932894207, ++STORE, 140321929097216, 140321930756095, ++SNULL, 140321932853247, 140321932894207, ++STORE, 140321930756096, 140321932853247, ++STORE, 140321932853248, 140321932894207, ++SNULL, 140321932853248, 140321932877823, ++STORE, 140321932877824, 140321932894207, ++STORE, 140321932853248, 140321932877823, ++ERASE, 140321932853248, 140321932877823, ++STORE, 140321932853248, 140321932877823, ++ERASE, 140321932877824, 140321932894207, ++STORE, 140321932877824, 140321932894207, ++STORE, 140321937305600, 140321937321983, ++SNULL, 140321932869631, 140321932877823, ++STORE, 140321932853248, 140321932869631, ++STORE, 140321932869632, 140321932877823, ++SNULL, 140321935089663, 140321935093759, ++STORE, 140321935085568, 140321935089663, ++STORE, 140321935089664, 140321935093759, ++SNULL, 94270502309887, 94270502313983, ++STORE, 94270502305792, 94270502309887, ++STORE, 94270502309888, 94270502313983, ++SNULL, 140321937354751, 140321937358847, ++STORE, 140321937350656, 140321937354751, ++STORE, 140321937354752, 140321937358847, ++ERASE, 140321937321984, 140321937350655, ++STORE, 94270507364352, 94270507499519, ++STORE, 140321920704512, 140321929097215, ++SNULL, 140321920708607, 140321929097215, ++STORE, 140321920704512, 140321920708607, ++STORE, 140321920708608, 140321929097215, ++STORE, 140321912311808, 140321920704511, ++STORE, 140321778094080, 140321912311807, ++SNULL, 140321778094080, 140321816051711, ++STORE, 140321816051712, 140321912311807, ++STORE, 140321778094080, 140321816051711, ++ERASE, 140321778094080, 140321816051711, ++SNULL, 140321883160575, 140321912311807, ++STORE, 140321816051712, 140321883160575, ++STORE, 140321883160576, 140321912311807, ++ERASE, 140321883160576, 140321912311807, ++SNULL, 140321816186879, 140321883160575, ++STORE, 140321816051712, 140321816186879, ++STORE, 140321816186880, 140321883160575, ++SNULL, 140321912315903, 140321920704511, ++STORE, 140321912311808, 140321912315903, ++STORE, 140321912315904, 140321920704511, ++STORE, 140321903919104, 140321912311807, ++SNULL, 140321903923199, 140321912311807, ++STORE, 140321903919104, 140321903923199, ++STORE, 140321903923200, 140321912311807, ++STORE, 140321895526400, 140321903919103, ++SNULL, 140321895530495, 140321903919103, ++STORE, 140321895526400, 140321895530495, ++STORE, 140321895530496, 140321903919103, ++STORE, 140321887133696, 140321895526399, ++SNULL, 140321887137791, 140321895526399, ++STORE, 140321887133696, 140321887137791, ++STORE, 140321887137792, 140321895526399, ++STORE, 140321807659008, 140321816051711, ++STORE, 140321673441280, 140321807659007, ++SNULL, 140321673441280, 140321681833983, ++STORE, 140321681833984, 140321807659007, ++STORE, 140321673441280, 140321681833983, ++ERASE, 140321673441280, 140321681833983, ++SNULL, 140321748942847, 140321807659007, ++STORE, 140321681833984, 140321748942847, ++STORE, 140321748942848, 140321807659007, ++ERASE, 140321748942848, 140321807659007, ++STORE, 140321799266304, 140321816051711, ++STORE, 140321790873600, 140321816051711, ++STORE, 140321782480896, 140321816051711, ++STORE, 140321547616256, 140321748942847, ++SNULL, 140321614725119, 140321748942847, ++STORE, 140321547616256, 140321614725119, ++STORE, 140321614725120, 140321748942847, ++SNULL, 140321614725120, 140321681833983, ++STORE, 140321681833984, 140321748942847, ++STORE, 140321614725120, 140321681833983, ++ERASE, 140321614725120, 140321681833983, ++SNULL, 140321681969151, 140321748942847, ++STORE, 140321681833984, 140321681969151, ++STORE, 140321681969152, 140321748942847, ++STORE, 140321547616256, 140321681833983, ++SNULL, 140321547616256, 140321614725119, ++STORE, 140321614725120, 140321681833983, ++STORE, 140321547616256, 140321614725119, ++SNULL, 140321614860287, 140321681833983, ++STORE, 140321614725120, 140321614860287, ++STORE, 140321614860288, 140321681833983, ++SNULL, 140321547751423, 140321614725119, ++STORE, 140321547616256, 140321547751423, ++STORE, 140321547751424, 140321614725119, ++STORE, 140321480507392, 140321547616255, ++SNULL, 140321782480896, 140321799266303, ++STORE, 140321799266304, 140321816051711, ++STORE, 140321782480896, 140321799266303, ++SNULL, 140321799270399, 140321816051711, ++STORE, 140321799266304, 140321799270399, ++STORE, 140321799270400, 140321816051711, ++STORE, 140321774088192, 140321799266303, ++SNULL, 140321774088192, 140321790873599, ++STORE, 140321790873600, 140321799266303, ++STORE, 140321774088192, 140321790873599, ++SNULL, 140321790877695, 140321799266303, ++STORE, 140321790873600, 140321790877695, ++STORE, 140321790877696, 140321799266303, ++SNULL, 140321480642559, 140321547616255, ++STORE, 140321480507392, 140321480642559, ++STORE, 140321480642560, 140321547616255, ++SNULL, 140321774088192, 140321782480895, ++STORE, 140321782480896, 140321790873599, ++STORE, 140321774088192, 140321782480895, ++SNULL, 140321782484991, 140321790873599, ++STORE, 140321782480896, 140321782484991, ++STORE, 140321782484992, 140321790873599, ++SNULL, 140321799270400, 140321807659007, ++STORE, 140321807659008, 140321816051711, ++STORE, 140321799270400, 140321807659007, ++SNULL, 140321807663103, 140321816051711, ++STORE, 140321807659008, 140321807663103, ++STORE, 140321807663104, 140321816051711, ++STORE, 140321765695488, 140321782480895, ++STORE, 140321757302784, 140321782480895, ++SNULL, 140321757306879, 140321782480895, ++STORE, 140321757302784, 140321757306879, ++STORE, 140321757306880, 140321782480895, ++STORE, 140321472114688, 140321480507391, ++STORE, 140321463721984, 140321480507391, ++SNULL, 140321463726079, 140321480507391, ++STORE, 140321463721984, 140321463726079, ++STORE, 140321463726080, 140321480507391, ++SNULL, 140321757306880, 140321774088191, ++STORE, 140321774088192, 140321782480895, ++STORE, 140321757306880, 140321774088191, ++SNULL, 140321774092287, 140321782480895, ++STORE, 140321774088192, 140321774092287, ++STORE, 140321774092288, 140321782480895, ++SNULL, 140321463726080, 140321472114687, ++STORE, 140321472114688, 140321480507391, ++STORE, 140321463726080, 140321472114687, ++SNULL, 140321472118783, 140321480507391, ++STORE, 140321472114688, 140321472118783, ++STORE, 140321472118784, 140321480507391, ++SNULL, 140321757306880, 140321765695487, ++STORE, 140321765695488, 140321774088191, ++STORE, 140321757306880, 140321765695487, ++SNULL, 140321765699583, 140321774088191, ++STORE, 140321765695488, 140321765699583, ++STORE, 140321765699584, 140321774088191, ++STORE, 140321455329280, 140321463721983, ++SNULL, 140321455333375, 140321463721983, ++STORE, 140321455329280, 140321455333375, ++STORE, 140321455333376, 140321463721983, ++STORE, 140321446936576, 140321455329279, ++STORE, 140321438543872, 140321455329279, ++STORE, 140321430151168, 140321455329279, ++SNULL, 140321430155263, 140321455329279, ++STORE, 140321430151168, 140321430155263, ++STORE, 140321430155264, 140321455329279, ++SNULL, 140321430155264, 140321446936575, ++STORE, 140321446936576, 140321455329279, ++STORE, 140321430155264, 140321446936575, ++SNULL, 140321446940671, 140321455329279, ++STORE, 140321446936576, 140321446940671, ++STORE, 140321446940672, 140321455329279, ++SNULL, 140321430155264, 140321438543871, ++STORE, 140321438543872, 140321446936575, ++STORE, 140321430155264, 140321438543871, ++SNULL, 140321438547967, 140321446936575, ++STORE, 140321438543872, 140321438547967, ++STORE, 140321438547968, 140321446936575, ++STORE, 140321421758464, 140321430151167, ++SNULL, 140321421762559, 140321430151167, ++STORE, 140321421758464, 140321421762559, ++STORE, 140321421762560, 140321430151167, ++STORE, 140321413365760, 140321421758463, ++SNULL, 140321413369855, 140321421758463, ++STORE, 140321413365760, 140321413369855, ++STORE, 140321413369856, 140321421758463, ++STORE, 140321404973056, 140321413365759, ++SNULL, 140321404977151, 140321413365759, ++STORE, 140321404973056, 140321404977151, ++STORE, 140321404977152, 140321413365759, ++STORE, 140321396580352, 140321404973055, ++STORE, 140321388187648, 140321404973055, ++STORE, 140321253969920, 140321388187647, ++SNULL, 140321253969920, 140321279180799, ++STORE, 140321279180800, 140321388187647, ++STORE, 140321253969920, 140321279180799, ++ERASE, 140321253969920, 140321279180799, ++SNULL, 140321346289663, 140321388187647, ++STORE, 140321279180800, 140321346289663, ++STORE, 140321346289664, 140321388187647, ++ERASE, 140321346289664, 140321388187647, ++STORE, 140321144963072, 140321346289663, ++STORE, 140321379794944, 140321404973055, ++STORE, 140321371402240, 140321404973055, ++STORE, 140321010745344, 140321346289663, ++STORE, 140321363009536, 140321404973055, ++SNULL, 140321077854207, 140321346289663, ++STORE, 140321010745344, 140321077854207, ++STORE, 140321077854208, 140321346289663, ++SNULL, 140321077854208, 140321144963071, ++STORE, 140321144963072, 140321346289663, ++STORE, 140321077854208, 140321144963071, ++ERASE, 140321077854208, 140321144963071, ++STORE, 140321354616832, 140321404973055, ++STORE, 140321136570368, 140321144963071, ++STORE, 140320943636480, 140321077854207, ++STORE, 140320876527616, 140321077854207, ++STORE, 140321128177664, 140321144963071, ++SNULL, 140320876662783, 140321077854207, ++STORE, 140320876527616, 140320876662783, ++STORE, 140320876662784, 140321077854207, ++STORE, 140321119784960, 140321144963071, ++STORE, 140321111392256, 140321144963071, ++STORE, 140320742309888, 140320876527615, ++STORE, 140321102999552, 140321144963071, ++STORE, 140320608092160, 140320876527615, ++SNULL, 140320675201023, 140320876527615, ++STORE, 140320608092160, 140320675201023, ++STORE, 140320675201024, 140320876527615, ++SNULL, 140320675201024, 140320742309887, ++STORE, 140320742309888, 140320876527615, ++STORE, 140320675201024, 140320742309887, ++ERASE, 140320675201024, 140320742309887, ++STORE, 140321094606848, 140321144963071, ++STORE, 140321086214144, 140321144963071, ++STORE, 140320608092160, 140320876527615, ++SNULL, 140320608092160, 140320675201023, ++STORE, 140320675201024, 140320876527615, ++STORE, 140320608092160, 140320675201023, ++SNULL, 140320675336191, 140320876527615, ++STORE, 140320675201024, 140320675336191, ++STORE, 140320675336192, 140320876527615, ++STORE, 140320599699456, 140320608092159, ++STORE, 140320591306752, 140320608092159, ++STORE, 140320457089024, 140320591306751, ++STORE, 140320448696320, 140320457089023, ++STORE, 140320314478592, 140320448696319, ++SNULL, 140321144963072, 140321279180799, ++STORE, 140321279180800, 140321346289663, ++STORE, 140321144963072, 140321279180799, ++SNULL, 140321279315967, 140321346289663, ++STORE, 140321279180800, 140321279315967, ++STORE, 140321279315968, 140321346289663, ++SNULL, 140321086214144, 140321136570367, ++STORE, 140321136570368, 140321144963071, ++STORE, 140321086214144, 140321136570367, ++SNULL, 140321136574463, 140321144963071, ++STORE, 140321136570368, 140321136574463, ++STORE, 140321136574464, 140321144963071, ++SNULL, 140321212071935, 140321279180799, ++STORE, 140321144963072, 140321212071935, ++STORE, 140321212071936, 140321279180799, ++ERASE, 140321212071936, 140321279180799, ++SNULL, 140321145098239, 140321212071935, ++STORE, 140321144963072, 140321145098239, ++STORE, 140321145098240, 140321212071935, ++SNULL, 140320876662784, 140321010745343, ++STORE, 140321010745344, 140321077854207, ++STORE, 140320876662784, 140321010745343, ++SNULL, 140321010880511, 140321077854207, ++STORE, 140321010745344, 140321010880511, ++STORE, 140321010880512, 140321077854207, ++SNULL, 140321354616832, 140321379794943, ++STORE, 140321379794944, 140321404973055, ++STORE, 140321354616832, 140321379794943, ++SNULL, 140321379799039, 140321404973055, ++STORE, 140321379794944, 140321379799039, ++STORE, 140321379799040, 140321404973055, ++SNULL, 140320876662784, 140320943636479, ++STORE, 140320943636480, 140321010745343, ++STORE, 140320876662784, 140320943636479, ++SNULL, 140320943771647, 140321010745343, ++STORE, 140320943636480, 140320943771647, ++STORE, 140320943771648, 140321010745343, ++SNULL, 140320809418751, 140320876527615, ++STORE, 140320675336192, 140320809418751, ++STORE, 140320809418752, 140320876527615, ++ERASE, 140320809418752, 140320876527615, ++SNULL, 140320675336192, 140320742309887, ++STORE, 140320742309888, 140320809418751, ++STORE, 140320675336192, 140320742309887, ++SNULL, 140320742445055, 140320809418751, ++STORE, 140320742309888, 140320742445055, ++STORE, 140320742445056, 140320809418751, ++SNULL, 140320608227327, 140320675201023, ++STORE, 140320608092160, 140320608227327, ++STORE, 140320608227328, 140320675201023, ++SNULL, 140320457089024, 140320473874431, ++STORE, 140320473874432, 140320591306751, ++STORE, 140320457089024, 140320473874431, ++ERASE, 140320457089024, 140320473874431, ++SNULL, 140320540983295, 140320591306751, ++STORE, 140320473874432, 140320540983295, ++STORE, 140320540983296, 140320591306751, ++ERASE, 140320540983296, 140320591306751, ++SNULL, 140320314478592, 140320339656703, ++STORE, 140320339656704, 140320448696319, ++STORE, 140320314478592, 140320339656703, ++ERASE, 140320314478592, 140320339656703, ++SNULL, 140321086214144, 140321128177663, ++STORE, 140321128177664, 140321136570367, ++STORE, 140321086214144, 140321128177663, ++SNULL, 140321128181759, 140321136570367, ++STORE, 140321128177664, 140321128181759, ++STORE, 140321128181760, 140321136570367, ++SNULL, 140321354616832, 140321371402239, ++STORE, 140321371402240, 140321379794943, ++STORE, 140321354616832, 140321371402239, ++SNULL, 140321371406335, 140321379794943, ++STORE, 140321371402240, 140321371406335, ++STORE, 140321371406336, 140321379794943, ++SNULL, 140320591310847, 140320608092159, ++STORE, 140320591306752, 140320591310847, ++STORE, 140320591310848, 140320608092159, ++SNULL, 140321354616832, 140321363009535, ++STORE, 140321363009536, 140321371402239, ++STORE, 140321354616832, 140321363009535, ++SNULL, 140321363013631, 140321371402239, ++STORE, 140321363009536, 140321363013631, ++STORE, 140321363013632, 140321371402239, ++SNULL, 140321086214144, 140321119784959, ++STORE, 140321119784960, 140321128177663, ++STORE, 140321086214144, 140321119784959, ++SNULL, 140321119789055, 140321128177663, ++STORE, 140321119784960, 140321119789055, ++STORE, 140321119789056, 140321128177663, ++SNULL, 140321086218239, 140321119784959, ++STORE, 140321086214144, 140321086218239, ++STORE, 140321086218240, 140321119784959, ++SNULL, 140321086218240, 140321094606847, ++STORE, 140321094606848, 140321119784959, ++STORE, 140321086218240, 140321094606847, ++SNULL, 140321094610943, 140321119784959, ++STORE, 140321094606848, 140321094610943, ++STORE, 140321094610944, 140321119784959, ++SNULL, 140320474009599, 140320540983295, ++STORE, 140320473874432, 140320474009599, ++STORE, 140320474009600, 140320540983295, ++SNULL, 140320406765567, 140320448696319, ++STORE, 140320339656704, 140320406765567, ++STORE, 140320406765568, 140320448696319, ++ERASE, 140320406765568, 140320448696319, ++SNULL, 140320339791871, 140320406765567, ++STORE, 140320339656704, 140320339791871, ++STORE, 140320339791872, 140320406765567, ++STORE, 140321270788096, 140321279180799, ++STORE, 140321262395392, 140321279180799, ++STORE, 140321254002688, 140321279180799, ++SNULL, 140321254002688, 140321262395391, ++STORE, 140321262395392, 140321279180799, ++STORE, 140321254002688, 140321262395391, ++SNULL, 140321262399487, 140321279180799, ++STORE, 140321262395392, 140321262399487, ++STORE, 140321262399488, 140321279180799, ++STORE, 140321245609984, 140321262395391, ++STORE, 140321237217280, 140321262395391, ++SNULL, 140321237217280, 140321245609983, ++STORE, 140321245609984, 140321262395391, ++STORE, 140321237217280, 140321245609983, ++SNULL, 140321245614079, 140321262395391, ++STORE, 140321245609984, 140321245614079, ++STORE, 140321245614080, 140321262395391, ++SNULL, 140321379799040, 140321388187647, ++STORE, 140321388187648, 140321404973055, ++STORE, 140321379799040, 140321388187647, ++SNULL, 140321388191743, 140321404973055, ++STORE, 140321388187648, 140321388191743, ++STORE, 140321388191744, 140321404973055, ++SNULL, 140321354620927, 140321363009535, ++STORE, 140321354616832, 140321354620927, ++STORE, 140321354620928, 140321363009535, ++SNULL, 140321388191744, 140321396580351, ++STORE, 140321396580352, 140321404973055, ++STORE, 140321388191744, 140321396580351, ++SNULL, 140321396584447, 140321404973055, ++STORE, 140321396580352, 140321396584447, ++STORE, 140321396584448, 140321404973055, ++SNULL, 140321094610944, 140321111392255, ++STORE, 140321111392256, 140321119784959, ++STORE, 140321094610944, 140321111392255, ++SNULL, 140321111396351, 140321119784959, ++STORE, 140321111392256, 140321111396351, ++STORE, 140321111396352, 140321119784959, ++STORE, 140321228824576, 140321245609983, ++SNULL, 140321094610944, 140321102999551, ++STORE, 140321102999552, 140321111392255, ++STORE, 140321094610944, 140321102999551, ++SNULL, 140321103003647, 140321111392255, ++STORE, 140321102999552, 140321103003647, ++STORE, 140321103003648, 140321111392255, ++STORE, 140321220431872, 140321245609983, ++SNULL, 140321220435967, 140321245609983, ++STORE, 140321220431872, 140321220435967, ++STORE, 140321220435968, 140321245609983, ++STORE, 140320868134912, 140320876527615, ++SNULL, 140320868139007, 140320876527615, ++STORE, 140320868134912, 140320868139007, ++STORE, 140320868139008, 140320876527615, ++SNULL, 140320591310848, 140320599699455, ++STORE, 140320599699456, 140320608092159, ++STORE, 140320591310848, 140320599699455, ++SNULL, 140320599703551, 140320608092159, ++STORE, 140320599699456, 140320599703551, ++STORE, 140320599703552, 140320608092159, ++STORE, 140320859742208, 140320868134911, ++SNULL, 140321262399488, 140321270788095, ++STORE, 140321270788096, 140321279180799, ++STORE, 140321262399488, 140321270788095, ++SNULL, 140321270792191, 140321279180799, ++STORE, 140321270788096, 140321270792191, ++STORE, 140321270792192, 140321279180799, ++STORE, 140320851349504, 140320868134911, ++STORE, 140320842956800, 140320868134911, ++STORE, 140320834564096, 140320868134911, ++STORE, 140320826171392, 140320868134911, ++SNULL, 140320826171392, 140320834564095, ++STORE, 140320834564096, 140320868134911, ++STORE, 140320826171392, 140320834564095, ++SNULL, 140320834568191, 140320868134911, ++STORE, 140320834564096, 140320834568191, ++STORE, 140320834568192, 140320868134911, ++SNULL, 140321220435968, 140321228824575, ++STORE, 140321228824576, 140321245609983, ++STORE, 140321220435968, 140321228824575, ++SNULL, 140321228828671, 140321245609983, ++STORE, 140321228824576, 140321228828671, ++STORE, 140321228828672, 140321245609983, ++STORE, 140320817778688, 140320834564095, ++SNULL, 140320817782783, 140320834564095, ++STORE, 140320817778688, 140320817782783, ++STORE, 140320817782784, 140320834564095, ++STORE, 140320582914048, 140320591306751, ++SNULL, 140321228828672, 140321237217279, ++STORE, 140321237217280, 140321245609983, ++STORE, 140321228828672, 140321237217279, ++SNULL, 140321237221375, 140321245609983, ++STORE, 140321237217280, 140321237221375, ++STORE, 140321237221376, 140321245609983, ++SNULL, 140320448700415, 140320457089023, ++STORE, 140320448696320, 140320448700415, ++STORE, 140320448700416, 140320457089023, ++SNULL, 140321245614080, 140321254002687, ++STORE, 140321254002688, 140321262395391, ++STORE, 140321245614080, 140321254002687, ++SNULL, 140321254006783, 140321262395391, ++STORE, 140321254002688, 140321254006783, ++STORE, 140321254006784, 140321262395391, ++STORE, 140320574521344, 140320591306751, ++SNULL, 140320574525439, 140320591306751, ++STORE, 140320574521344, 140320574525439, ++STORE, 140320574525440, 140320591306751, ++STORE, 140320566128640, 140320574521343, ++SNULL, 140320566132735, 140320574521343, ++STORE, 140320566128640, 140320566132735, ++STORE, 140320566132736, 140320574521343, ++SNULL, 140320574525440, 140320582914047, ++STORE, 140320582914048, 140320591306751, ++STORE, 140320574525440, 140320582914047, ++SNULL, 140320582918143, 140320591306751, ++STORE, 140320582914048, 140320582918143, ++STORE, 140320582918144, 140320591306751, ++STORE, 140320557735936, 140320566128639, ++SNULL, 140320557740031, 140320566128639, ++STORE, 140320557735936, 140320557740031, ++STORE, 140320557740032, 140320566128639, ++STORE, 140320549343232, 140320557735935, ++STORE, 140320465481728, 140320473874431, ++STORE, 140320448700416, 140320473874431, ++SNULL, 140320834568192, 140320859742207, ++STORE, 140320859742208, 140320868134911, ++STORE, 140320834568192, 140320859742207, ++SNULL, 140320859746303, 140320868134911, ++STORE, 140320859742208, 140320859746303, ++STORE, 140320859746304, 140320868134911, ++STORE, 140320440303616, 140320448696319, ++STORE, 140320431910912, 140320448696319, ++SNULL, 140320834568192, 140320851349503, ++STORE, 140320851349504, 140320859742207, ++STORE, 140320834568192, 140320851349503, ++SNULL, 140320851353599, 140320859742207, ++STORE, 140320851349504, 140320851353599, ++STORE, 140320851353600, 140320859742207, ++SNULL, 140320817782784, 140320826171391, ++STORE, 140320826171392, 140320834564095, ++STORE, 140320817782784, 140320826171391, ++SNULL, 140320826175487, 140320834564095, ++STORE, 140320826171392, 140320826175487, ++STORE, 140320826175488, 140320834564095, ++SNULL, 140320834568192, 140320842956799, ++STORE, 140320842956800, 140320851349503, ++STORE, 140320834568192, 140320842956799, ++SNULL, 140320842960895, 140320851349503, ++STORE, 140320842956800, 140320842960895, ++STORE, 140320842960896, 140320851349503, ++STORE, 140320423518208, 140320448696319, ++SNULL, 140320423522303, 140320448696319, ++STORE, 140320423518208, 140320423522303, ++STORE, 140320423522304, 140320448696319, ++STORE, 140320415125504, 140320423518207, ++STORE, 140320331264000, 140320339656703, ++STORE, 140320322871296, 140320339656703, ++STORE, 140320314478592, 140320339656703, ++SNULL, 140320314482687, 140320339656703, ++STORE, 140320314478592, 140320314482687, ++STORE, 140320314482688, 140320339656703, ++STORE, 140320306085888, 140320314478591, ++SNULL, 140320306089983, 140320314478591, ++STORE, 140320306085888, 140320306089983, ++STORE, 140320306089984, 140320314478591, ++STORE, 140320297693184, 140320306085887, ++SNULL, 140320297697279, 140320306085887, ++STORE, 140320297693184, 140320297697279, ++STORE, 140320297697280, 140320306085887, ++STORE, 140320289300480, 140320297693183, ++STORE, 140320280907776, 140320297693183, ++SNULL, 140320280911871, 140320297693183, ++STORE, 140320280907776, 140320280911871, ++STORE, 140320280911872, 140320297693183, ++SNULL, 140320423522304, 140320431910911, ++STORE, 140320431910912, 140320448696319, ++STORE, 140320423522304, 140320431910911, ++SNULL, 140320431915007, 140320448696319, ++STORE, 140320431910912, 140320431915007, ++STORE, 140320431915008, 140320448696319, ++SNULL, 140320549347327, 140320557735935, ++STORE, 140320549343232, 140320549347327, ++STORE, 140320549347328, 140320557735935, ++STORE, 140320272515072, 140320280907775, ++SNULL, 140320448700416, 140320457089023, ++STORE, 140320457089024, 140320473874431, ++STORE, 140320448700416, 140320457089023, ++SNULL, 140320457093119, 140320473874431, ++STORE, 140320457089024, 140320457093119, ++STORE, 140320457093120, 140320473874431, ++STORE, 140320264122368, 140320280907775, ++SNULL, 140320457093120, 140320465481727, ++STORE, 140320465481728, 140320473874431, ++STORE, 140320457093120, 140320465481727, ++SNULL, 140320465485823, 140320473874431, ++STORE, 140320465481728, 140320465485823, ++STORE, 140320465485824, 140320473874431, ++SNULL, 140320431915008, 140320440303615, ++STORE, 140320440303616, 140320448696319, ++STORE, 140320431915008, 140320440303615, ++SNULL, 140320440307711, 140320448696319, ++STORE, 140320440303616, 140320440307711, ++STORE, 140320440307712, 140320448696319, ++STORE, 140320255729664, 140320280907775, ++STORE, 140320247336960, 140320280907775, ++SNULL, 140320247341055, 140320280907775, ++STORE, 140320247336960, 140320247341055, ++STORE, 140320247341056, 140320280907775, ++STORE, 140320238944256, 140320247336959, ++STORE, 140320230551552, 140320247336959, ++SNULL, 140320230551552, 140320238944255, ++STORE, 140320238944256, 140320247336959, ++STORE, 140320230551552, 140320238944255, ++SNULL, 140320238948351, 140320247336959, ++STORE, 140320238944256, 140320238948351, ++STORE, 140320238948352, 140320247336959, ++SNULL, 140320314482688, 140320331263999, ++STORE, 140320331264000, 140320339656703, ++STORE, 140320314482688, 140320331263999, ++SNULL, 140320331268095, 140320339656703, ++STORE, 140320331264000, 140320331268095, ++STORE, 140320331268096, 140320339656703, ++SNULL, 140320280911872, 140320289300479, ++STORE, 140320289300480, 140320297693183, ++STORE, 140320280911872, 140320289300479, ++SNULL, 140320289304575, 140320297693183, ++STORE, 140320289300480, 140320289304575, ++STORE, 140320289304576, 140320297693183, ++SNULL, 140320415129599, 140320423518207, ++STORE, 140320415125504, 140320415129599, ++STORE, 140320415129600, 140320423518207, ++STORE, 140320222158848, 140320238944255, ++STORE, 140320213766144, 140320238944255, ++STORE, 140320205373440, 140320238944255, ++SNULL, 140320205377535, 140320238944255, ++STORE, 140320205373440, 140320205377535, ++STORE, 140320205377536, 140320238944255, ++SNULL, 140320314482688, 140320322871295, ++STORE, 140320322871296, 140320331263999, ++STORE, 140320314482688, 140320322871295, ++SNULL, 140320322875391, 140320331263999, ++STORE, 140320322871296, 140320322875391, ++STORE, 140320322875392, 140320331263999, ++SNULL, 140320247341056, 140320272515071, ++STORE, 140320272515072, 140320280907775, ++STORE, 140320247341056, 140320272515071, ++SNULL, 140320272519167, 140320280907775, ++STORE, 140320272515072, 140320272519167, ++STORE, 140320272519168, 140320280907775, ++SNULL, 140320247341056, 140320264122367, ++STORE, 140320264122368, 140320272515071, ++STORE, 140320247341056, 140320264122367, ++SNULL, 140320264126463, 140320272515071, ++STORE, 140320264122368, 140320264126463, ++STORE, 140320264126464, 140320272515071, ++SNULL, 140320205377536, 140320230551551, ++STORE, 140320230551552, 140320238944255, ++STORE, 140320205377536, 140320230551551, ++SNULL, 140320230555647, 140320238944255, ++STORE, 140320230551552, 140320230555647, ++STORE, 140320230555648, 140320238944255, ++STORE, 140320196980736, 140320205373439, ++SNULL, 140320196984831, 140320205373439, ++STORE, 140320196980736, 140320196984831, ++STORE, 140320196984832, 140320205373439, ++STORE, 140320188588032, 140320196980735, ++SNULL, 140320247341056, 140320255729663, ++STORE, 140320255729664, 140320264122367, ++STORE, 140320247341056, 140320255729663, ++SNULL, 140320255733759, 140320264122367, ++STORE, 140320255729664, 140320255733759, ++STORE, 140320255733760, 140320264122367, ++STORE, 140320180195328, 140320196980735, ++SNULL, 140320180199423, 140320196980735, ++STORE, 140320180195328, 140320180199423, ++STORE, 140320180199424, 140320196980735, ++STORE, 140320171802624, 140320180195327, ++STORE, 140320163409920, 140320180195327, ++SNULL, 140320163414015, 140320180195327, ++STORE, 140320163409920, 140320163414015, ++STORE, 140320163414016, 140320180195327, ++SNULL, 140320205377536, 140320222158847, ++STORE, 140320222158848, 140320230551551, ++STORE, 140320205377536, 140320222158847, ++SNULL, 140320222162943, 140320230551551, ++STORE, 140320222158848, 140320222162943, ++STORE, 140320222162944, 140320230551551, ++SNULL, 140320205377536, 140320213766143, ++STORE, 140320213766144, 140320222158847, ++STORE, 140320205377536, 140320213766143, ++SNULL, 140320213770239, 140320222158847, ++STORE, 140320213766144, 140320213770239, ++STORE, 140320213770240, 140320222158847, ++STORE, 140320155017216, 140320163409919, ++SNULL, 140320180199424, 140320188588031, ++STORE, 140320188588032, 140320196980735, ++STORE, 140320180199424, 140320188588031, ++SNULL, 140320188592127, 140320196980735, ++STORE, 140320188588032, 140320188592127, ++STORE, 140320188592128, 140320196980735, ++SNULL, 140320155021311, 140320163409919, ++STORE, 140320155017216, 140320155021311, ++STORE, 140320155021312, 140320163409919, ++SNULL, 140320163414016, 140320171802623, ++STORE, 140320171802624, 140320180195327, ++STORE, 140320163414016, 140320171802623, ++SNULL, 140320171806719, 140320180195327, ++STORE, 140320171802624, 140320171806719, ++STORE, 140320171806720, 140320180195327, ++STORE, 140320146624512, 140320155017215, ++SNULL, 140320146628607, 140320155017215, ++STORE, 140320146624512, 140320146628607, ++STORE, 140320146628608, 140320155017215, ++STORE, 140321937321984, 140321937350655, ++STORE, 140321884942336, 140321887133695, ++SNULL, 140321884942336, 140321885032447, ++STORE, 140321885032448, 140321887133695, ++STORE, 140321884942336, 140321885032447, ++SNULL, 140321887125503, 140321887133695, ++STORE, 140321885032448, 140321887125503, ++STORE, 140321887125504, 140321887133695, ++ERASE, 140321887125504, 140321887133695, ++STORE, 140321887125504, 140321887133695, ++SNULL, 140321887129599, 140321887133695, ++STORE, 140321887125504, 140321887129599, ++STORE, 140321887129600, 140321887133695, ++ERASE, 140321937321984, 140321937350655, ++ERASE, 140321086214144, 140321086218239, ++ERASE, 140321086218240, 140321094606847, ++ERASE, 140321119784960, 140321119789055, ++ERASE, 140321119789056, 140321128177663, ++ERASE, 140321245609984, 140321245614079, ++ERASE, 140321245614080, 140321254002687, ++ERASE, 140320574521344, 140320574525439, ++ERASE, 140320574525440, 140320582914047, ++ERASE, 140320297693184, 140320297697279, ++ERASE, 140320297697280, 140320306085887, ++ERASE, 140321354616832, 140321354620927, ++ERASE, 140321354620928, 140321363009535, ++ERASE, 140320834564096, 140320834568191, ++ERASE, 140320834568192, 140320842956799, ++ERASE, 140320591306752, 140320591310847, ++ERASE, 140320591310848, 140320599699455, ++ERASE, 140321136570368, 140321136574463, ++ERASE, 140321136574464, 140321144963071, ++ERASE, 140321237217280, 140321237221375, ++ERASE, 140321237221376, 140321245609983, ++ERASE, 140321363009536, 140321363013631, ++ERASE, 140321363013632, 140321371402239, ++ERASE, 140320599699456, 140320599703551, ++ERASE, 140320599703552, 140320608092159, ++ERASE, 140321396580352, 140321396584447, ++ERASE, 140321396584448, 140321404973055, ++ERASE, 140320566128640, 140320566132735, ++ERASE, 140320566132736, 140320574521343, ++ERASE, 140321094606848, 140321094610943, ++ERASE, 140321094610944, 140321102999551, ++ERASE, 140320582914048, 140320582918143, ++ERASE, 140320582918144, 140320591306751, ++ERASE, 140320289300480, 140320289304575, ++ERASE, 140320289304576, 140320297693183, ++ERASE, 140320163409920, 140320163414015, ++ }; ++ unsigned long set41[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140728157171712, 140737488351231, ++SNULL, 140728157175807, 140737488351231, ++STORE, 140728157171712, 140728157175807, ++STORE, 140728157040640, 140728157175807, ++STORE, 94376106364928, 94376108613631, ++SNULL, 94376106487807, 94376108613631, ++STORE, 94376106364928, 94376106487807, ++STORE, 94376106487808, 94376108613631, ++SNULL, 94376106487808, 94376108613631, ++STORE, 94376108584960, 94376108593151, ++STORE, 94376108593152, 94376108613631, ++STORE, 140113496432640, 140113498685439, ++SNULL, 140113496575999, 140113498685439, ++STORE, 140113496432640, 140113496575999, ++STORE, 140113496576000, 140113498685439, ++SNULL, 140113496576000, 140113498685439, ++STORE, 140113498673152, 140113498681343, ++STORE, 140113498681344, 140113498685439, ++STORE, 140728157609984, 140728157618175, ++STORE, 140728157593600, 140728157609983, ++STORE, 140113498636288, 140113498673151, ++STORE, 140113498628096, 140113498636287, ++STORE, 140113492635648, 140113496432639, ++SNULL, 140113492635648, 140113494294527, ++STORE, 140113494294528, 140113496432639, ++STORE, 140113492635648, 140113494294527, ++SNULL, 140113496391679, 140113496432639, ++STORE, 140113494294528, 140113496391679, ++STORE, 140113496391680, 140113496432639, ++SNULL, 140113496391680, 140113496416255, ++STORE, 140113496416256, 140113496432639, ++STORE, 140113496391680, 140113496416255, ++SNULL, 140113496391680, 140113496416255, ++STORE, 140113496391680, 140113496416255, ++SNULL, 140113496416256, 140113496432639, ++STORE, 140113496416256, 140113496432639, ++SNULL, 140113496408063, 140113496416255, ++STORE, 140113496391680, 140113496408063, ++STORE, 140113496408064, 140113496416255, ++SNULL, 94376108589055, 94376108593151, ++STORE, 94376108584960, 94376108589055, ++STORE, 94376108589056, 94376108593151, ++SNULL, 140113498677247, 140113498681343, ++STORE, 140113498673152, 140113498677247, ++STORE, 140113498677248, 140113498681343, ++SNULL, 140113498636288, 140113498673151, ++STORE, 94376135090176, 94376135094271, ++STORE, 94376135090176, 94376135098367, ++STORE, 94376139288576, 94376139292671, ++STORE, 94376143482880, 94376143486975, ++STORE, 94376147677184, 94376147681279, ++STORE, 94376151871488, 94376151875583, ++STORE, 94376156065792, 94376156069887, ++STORE, 94376160260096, 94376160264191, ++STORE, 94376164454400, 94376164458495, ++STORE, 94376168648704, 94376168652799, ++STORE, 94376172843008, 94376172847103, ++STORE, 94376177037312, 94376177041407, ++STORE, 94376181231616, 94376181235711, ++STORE, 94376185425920, 94376185430015, ++STORE, 94376189620224, 94376189624319, ++STORE, 94376193814528, 94376193818623, ++STORE, 94376198008832, 94376198012927, ++STORE, 94376202203136, 94376202207231, ++STORE, 94376206397440, 94376206401535, ++STORE, 94376210591744, 94376210595839, ++STORE, 94376214786048, 94376214790143, ++STORE, 94376218980352, 94376218984447, ++STORE, 94376223174656, 94376223178751, ++STORE, 94376227368960, 94376227373055, ++STORE, 94376231563264, 94376231567359, ++STORE, 94376235757568, 94376235761663, ++STORE, 94376239951872, 94376239955967, ++STORE, 94376244146176, 94376244150271, ++STORE, 94376248340480, 94376248344575, ++STORE, 94376252534784, 94376252538879, ++STORE, 94376256729088, 94376256733183, ++STORE, 94376260923392, 94376260927487, ++STORE, 94376265117696, 94376265121791, ++STORE, 94376269312000, 94376269316095, ++STORE, 94376273506304, 94376273510399, ++STORE, 94376277700608, 94376277704703, ++STORE, 94376281894912, 94376281899007, ++STORE, 94376286089216, 94376286093311, ++STORE, 94376290283520, 94376290287615, ++STORE, 94376294477824, 94376294481919, ++STORE, 94376298672128, 94376298676223, ++STORE, 94376302866432, 94376302870527, ++STORE, 94376307060736, 94376307064831, ++STORE, 94376311255040, 94376311259135, ++STORE, 94376315449344, 94376315453439, ++STORE, 94376319643648, 94376319647743, ++STORE, 94376323837952, 94376323842047, ++STORE, 94376328032256, 94376328036351, ++STORE, 94376332226560, 94376332230655, ++STORE, 94376336420864, 94376336424959, ++STORE, 94376340615168, 94376340619263, ++STORE, 94376344809472, 94376344813567, ++STORE, 94376349003776, 94376349007871, ++STORE, 94376353198080, 94376353202175, ++STORE, 94376357392384, 94376357396479, ++STORE, 94376361586688, 94376361590783, ++STORE, 94376365780992, 94376365785087, ++STORE, 94376369975296, 94376369979391, ++STORE, 94376374169600, 94376374173695, ++STORE, 94376378363904, 94376378367999, ++STORE, 94376382558208, 94376382562303, ++STORE, 94376386752512, 94376386756607, ++STORE, 94376390946816, 94376390950911, ++STORE, 94376395141120, 94376395145215, ++STORE, 94376399335424, 94376399339519, ++STORE, 94376403529728, 94376403533823, ++STORE, 94376407724032, 94376407728127, ++STORE, 94376411918336, 94376411922431, ++STORE, 94376416112640, 94376416116735, ++STORE, 94376420306944, 94376420311039, ++STORE, 94376424501248, 94376424505343, ++STORE, 94376428695552, 94376428699647, ++STORE, 94376432889856, 94376432893951, ++STORE, 94376437084160, 94376437088255, ++STORE, 94376441278464, 94376441282559, ++STORE, 94376445472768, 94376445476863, ++STORE, 94376449667072, 94376449671167, ++STORE, 94376453861376, 94376453865471, ++STORE, 94376458055680, 94376458059775, ++STORE, 94376462249984, 94376462254079, ++STORE, 94376466444288, 94376466448383, ++STORE, 94376470638592, 94376470642687, ++STORE, 94376474832896, 94376474836991, ++STORE, 94376479027200, 94376479031295, ++STORE, 94376483221504, 94376483225599, ++STORE, 94376487415808, 94376487419903, ++STORE, 94376491610112, 94376491614207, ++STORE, 94376495804416, 94376495808511, ++STORE, 94376499998720, 94376500002815, ++STORE, 94376504193024, 94376504197119, ++STORE, 94376508387328, 94376508391423, ++STORE, 94376512581632, 94376512585727, ++STORE, 94376516775936, 94376516780031, ++STORE, 94376520970240, 94376520974335, ++STORE, 94376525164544, 94376525168639, ++STORE, 94376529358848, 94376529362943, ++STORE, 94376533553152, 94376533557247, ++STORE, 94376537747456, 94376537751551, ++STORE, 94376541941760, 94376541945855, ++STORE, 94376546136064, 94376546140159, ++STORE, 94376550330368, 94376550334463, ++STORE, 94376554524672, 94376554528767, ++STORE, 94376558718976, 94376558723071, ++STORE, 94376562913280, 94376562917375, ++STORE, 94376567107584, 94376567111679, ++STORE, 94376571301888, 94376571305983, ++STORE, 94376575496192, 94376575500287, ++STORE, 94376579690496, 94376579694591, ++STORE, 94376583884800, 94376583888895, ++STORE, 94376588079104, 94376588083199, ++STORE, 94376592273408, 94376592277503, ++STORE, 94376596467712, 94376596471807, ++STORE, 94376600662016, 94376600666111, ++STORE, 94376604856320, 94376604860415, ++STORE, 94376609050624, 94376609054719, ++STORE, 94376613244928, 94376613249023, ++STORE, 94376617439232, 94376617443327, ++STORE, 94376621633536, 94376621637631, ++STORE, 94376625827840, 94376625831935, ++STORE, 94376630022144, 94376630026239, ++STORE, 94376634216448, 94376634220543, ++STORE, 94376638410752, 94376638414847, ++STORE, 94376642605056, 94376642609151, ++STORE, 94376646799360, 94376646803455, ++STORE, 94376650993664, 94376650997759, ++STORE, 94376655187968, 94376655192063, ++STORE, 94376659382272, 94376659386367, ++STORE, 94376663576576, 94376663580671, ++STORE, 94376667770880, 94376667774975, ++STORE, 94376671965184, 94376671969279, ++STORE, 94376676159488, 94376676163583, ++STORE, 94376680353792, 94376680357887, ++STORE, 94376684548096, 94376684552191, ++STORE, 94376688742400, 94376688746495, ++STORE, 94376692936704, 94376692940799, ++STORE, 94376697131008, 94376697135103, ++STORE, 94376701325312, 94376701329407, ++STORE, 94376705519616, 94376705523711, ++STORE, 94376709713920, 94376709718015, ++STORE, 94376713908224, 94376713912319, ++STORE, 94376718102528, 94376718106623, ++STORE, 94376722296832, 94376722300927, ++STORE, 94376726491136, 94376726495231, ++STORE, 94376730685440, 94376730689535, ++STORE, 94376734879744, 94376734883839, ++STORE, 94376739074048, 94376739078143, ++STORE, 94376743268352, 94376743272447, ++STORE, 94376747462656, 94376747466751, ++STORE, 94376751656960, 94376751661055, ++STORE, 94376755851264, 94376755855359, ++STORE, 94376760045568, 94376760049663, ++STORE, 94376764239872, 94376764243967, ++STORE, 94376768434176, 94376768438271, ++STORE, 94376772628480, 94376772632575, ++STORE, 94376776822784, 94376776826879, ++STORE, 94376781017088, 94376781021183, ++STORE, 94376785211392, 94376785215487, ++STORE, 94376789405696, 94376789409791, ++STORE, 94376793600000, 94376793604095, ++STORE, 94376797794304, 94376797798399, ++STORE, 94376801988608, 94376801992703, ++STORE, 94376806182912, 94376806187007, ++STORE, 94376810377216, 94376810381311, ++STORE, 94376814571520, 94376814575615, ++STORE, 94376818765824, 94376818769919, ++STORE, 94376822960128, 94376822964223, ++STORE, 94376827154432, 94376827158527, ++STORE, 94376831348736, 94376831352831, ++STORE, 94376835543040, 94376835547135, ++STORE, 94376839737344, 94376839741439, ++STORE, 94376843931648, 94376843935743, ++STORE, 94376848125952, 94376848130047, ++STORE, 94376852320256, 94376852324351, ++STORE, 94376856514560, 94376856518655, ++STORE, 94376860708864, 94376860712959, ++STORE, 94376864903168, 94376864907263, ++STORE, 94376869097472, 94376869101567, ++STORE, 94376873291776, 94376873295871, ++STORE, 94376877486080, 94376877490175, ++STORE, 94376881680384, 94376881684479, ++STORE, 94376885874688, 94376885878783, ++STORE, 94376890068992, 94376890073087, ++STORE, 94376894263296, 94376894267391, ++STORE, 94376898457600, 94376898461695, ++STORE, 94376902651904, 94376902655999, ++STORE, 94376906846208, 94376906850303, ++STORE, 94376911040512, 94376911044607, ++STORE, 94376915234816, 94376915238911, ++STORE, 94376919429120, 94376919433215, ++STORE, 94376923623424, 94376923627519, ++STORE, 94376927817728, 94376927821823, ++STORE, 94376932012032, 94376932016127, ++STORE, 94376936206336, 94376936210431, ++STORE, 94376940400640, 94376940404735, ++STORE, 94376944594944, 94376944599039, ++STORE, 94376948789248, 94376948793343, ++STORE, 94376952983552, 94376952987647, ++STORE, 94376957177856, 94376957181951, ++STORE, 94376961372160, 94376961376255, ++STORE, 94376965566464, 94376965570559, ++STORE, 94376969760768, 94376969764863, ++STORE, 94376973955072, 94376973959167, ++STORE, 94376978149376, 94376978153471, ++STORE, 94376982343680, 94376982347775, ++STORE, 94376986537984, 94376986542079, ++STORE, 94376990732288, 94376990736383, ++STORE, 94376994926592, 94376994930687, ++STORE, 94376999120896, 94376999124991, ++STORE, 94377003315200, 94377003319295, ++STORE, 94377007509504, 94377007513599, ++STORE, 94377011703808, 94377011707903, ++STORE, 94377015898112, 94377015902207, ++STORE, 94377020092416, 94377020096511, ++STORE, 94377024286720, 94377024290815, ++STORE, 94377028481024, 94377028485119, ++STORE, 94377032675328, 94377032679423, ++STORE, 94377036869632, 94377036873727, ++STORE, 94377041063936, 94377041068031, ++STORE, 94377045258240, 94377045262335, ++STORE, 94377049452544, 94377049456639, ++STORE, 94377053646848, 94377053650943, ++STORE, 94377057841152, 94377057845247, ++STORE, 94377062035456, 94377062039551, ++STORE, 94377066229760, 94377066233855, ++STORE, 94377070424064, 94377070428159, ++STORE, 94377074618368, 94377074622463, ++STORE, 94377078812672, 94377078816767, ++STORE, 94377083006976, 94377083011071, ++STORE, 94377087201280, 94377087205375, ++STORE, 94377091395584, 94377091399679, ++STORE, 94377095589888, 94377095593983, ++STORE, 94377099784192, 94377099788287, ++STORE, 94377103978496, 94377103982591, ++STORE, 94377108172800, 94377108176895, ++STORE, 94377112367104, 94377112371199, ++STORE, 94377116561408, 94377116565503, ++STORE, 94377120755712, 94377120759807, ++STORE, 94377124950016, 94377124954111, ++STORE, 94377129144320, 94377129148415, ++STORE, 94377133338624, 94377133342719, ++STORE, 94377137532928, 94377137537023, ++STORE, 94377141727232, 94377141731327, ++STORE, 94377145921536, 94377145925631, ++STORE, 94377150115840, 94377150119935, ++STORE, 94377154310144, 94377154314239, ++STORE, 94377158504448, 94377158508543, ++STORE, 94377162698752, 94377162702847, ++STORE, 94377166893056, 94377166897151, ++STORE, 94377171087360, 94377171091455, ++STORE, 94377175281664, 94377175285759, ++STORE, 94377179475968, 94377179480063, ++STORE, 94377183670272, 94377183674367, ++STORE, 94377187864576, 94377187868671, ++STORE, 94377192058880, 94377192062975, ++STORE, 94377196253184, 94377196257279, ++STORE, 94377200447488, 94377200451583, ++STORE, 94377204641792, 94377204645887, ++SNULL, 94376135094271, 94376135098367, ++STORE, 94376135090176, 94376135094271, ++STORE, 94376135094272, 94376135098367, ++SNULL, 94376135094272, 94377208836095, ++ }; ++ unsigned long set42[] = { ++STORE, 314572800, 1388314623, ++STORE, 1462157312, 1462169599, ++STORE, 1462169600, 1462185983, ++STORE, 1462185984, 1462190079, ++STORE, 1462190080, 1462194175, ++STORE, 1462194176, 1462198271, ++STORE, 1879986176, 1881800703, ++STORE, 1881800704, 1882034175, ++STORE, 1882034176, 1882193919, ++STORE, 1882193920, 1882406911, ++STORE, 1882406912, 1882451967, ++STORE, 1882451968, 1882996735, ++STORE, 1882996736, 1885892607, ++STORE, 1885892608, 1885896703, ++STORE, 1885896704, 1885904895, ++STORE, 1885904896, 1885908991, ++STORE, 1885908992, 1885913087, ++STORE, 1885913088, 1885966335, ++STORE, 1885966336, 1886232575, ++STORE, 1886232576, 1886236671, ++STORE, 1886236672, 1886240767, ++STORE, 1886240768, 1886244863, ++STORE, 1886244864, 1886248959, ++STORE, 1886248960, 1886294015, ++STORE, 1886294016, 1886494719, ++STORE, 1886494720, 1886498815, ++STORE, 1886498816, 1886502911, ++STORE, 1886502912, 1886507007, ++STORE, 1886507008, 1886511103, ++STORE, 1886511104, 1886556159, ++STORE, 1886556160, 1886629887, ++STORE, 1886629888, 1886633983, ++STORE, 1886633984, 1886638079, ++STORE, 1886638080, 1886642175, ++STORE, 1886642176, 1886646271, ++STORE, 1886646272, 1886666751, ++STORE, 1886666752, 1886670847, ++STORE, 1886670848, 1886674943, ++STORE, 1886674944, 1886679039, ++STORE, 1886679040, 1895419903, ++STORE, 1895419904, 1895550975, ++STORE, 1895550976, 1896148991, ++STORE, 1896148992, 1897189375, ++STORE, 1897189376, 1897701375, ++STORE, 1897701376, 1897803775, ++STORE, 1897803776, 1897816063, ++STORE, 1897816064, 1899913215, ++STORE, 1899913216, 1909379071, ++STORE, 1909379072, 1909387263, ++STORE, 1909387264, 1909391359, ++STORE, 1909391360, 1909432319, ++STORE, 1909432320, 1909436415, ++STORE, 1909436416, 1909440511, ++STORE, 1909440512, 1909460991, ++STORE, 1909460992, 1909547007, ++STORE, 1909547008, 1909551103, ++STORE, 1909551104, 1909555199, ++STORE, 1909555200, 1909559295, ++STORE, 1909559296, 1909563391, ++STORE, 1909563392, 1909739519, ++STORE, 1909739520, 1910566911, ++STORE, 1910566912, 1910571007, ++STORE, 1910571008, 1910575103, ++STORE, 1910575104, 1910579199, ++STORE, 1910579200, 1910583295, ++STORE, 1910583296, 1910587391, ++STORE, 1910587392, 1910620159, ++STORE, 1910620160, 1910624255, ++STORE, 1910624256, 1910628351, ++STORE, 1910628352, 1910632447, ++STORE, 1910632448, 1910652927, ++STORE, 1910652928, 1910657023, ++STORE, 1910657024, 1910661119, ++STORE, 1910661120, 1910665215, ++STORE, 1910665216, 1910669311, ++STORE, 1910669312, 1910677503, ++STORE, 1910677504, 1910681599, ++STORE, 1910681600, 1910685695, ++STORE, 1910685696, 1910689791, ++STORE, 1910689792, 1910697983, ++STORE, 1910697984, 1910702079, ++STORE, 1910702080, 1910706175, ++STORE, 1910706176, 1910710271, ++STORE, 1910710272, 1914093567, ++STORE, 1914093568, 1914097663, ++STORE, 1914097664, 1969434623, ++STORE, 1969434624, 1977819135, ++STORE, 3290435584, 3426750463, ++STORE, 3426750464, 3426754559, ++STORE, 3426754560, 3426762751, ++STORE, 3426762752, 3426766847, ++STORE, 3426766848, 3426770943, ++STORE, 3427037184, 3427061759, ++STORE, 3427061760, 3427135487, ++STORE, 3427135488, 3427143679, ++STORE, 3427143680, 3427147775, ++STORE, 3427147776, 3427209215, ++STORE, 3427319808, 3432116223, ++STORE, 3432116224, 3450130431, ++STORE, 3450130432, 3451027455, ++STORE, 3451027456, 3451031551, ++STORE, 3451031552, 3451461631, ++STORE, 3451736064, 3456688127, ++STORE, 3456688128, 3475222527, ++STORE, 3475222528, 3476119551, ++STORE, 3476119552, 3476127743, ++STORE, 3476127744, 3476553727, ++STORE, 3476631552, 3477315583, ++STORE, 3477315584, 3479949311, ++STORE, 3479949312, 3480002559, ++STORE, 3480002560, 3480006655, ++STORE, 3480006656, 3480432639, ++STORE, 3480539136, 3480543231, ++STORE, 3480543232, 3480547327, ++STORE, 3480547328, 3480555519, ++STORE, 3480854528, 3480903679, ++STORE, 3480903680, 3480969215, ++STORE, 3480969216, 3480977407, ++STORE, 3480977408, 3480981503, ++STORE, 3481030656, 3481092095, ++STORE, 3481092096, 3481235455, ++STORE, 3481235456, 3481243647, ++STORE, 3481243648, 3481247743, ++STORE, 3481436160, 3481444351, ++STORE, 3481444352, 3481456639, ++STORE, 3481456640, 3481460735, ++STORE, 3481460736, 3481464831, ++STORE, 3481587712, 3481645055, ++STORE, 3481645056, 3481772031, ++STORE, 3481772032, 3481776127, ++STORE, 3481776128, 3481780223, ++STORE, 3481874432, 3481935871, ++STORE, 3481935872, 3482030079, ++STORE, 3482030080, 3482038271, ++STORE, 3482038272, 3482042367, ++STORE, 3482198016, 3482230783, ++STORE, 3482230784, 3482271743, ++STORE, 3482271744, 3482279935, ++STORE, 3482279936, 3482284031, ++STORE, 3482562560, 3482566655, ++STORE, 3482566656, 3482570751, ++STORE, 3482570752, 3482574847, ++STORE, 3482636288, 3482689535, ++STORE, 3482689536, 3482746879, ++STORE, 3482746880, 3482755071, ++STORE, 3482755072, 3482759167, ++STORE, 3482972160, 3483062271, ++STORE, 3483062272, 3483242495, ++STORE, 3483242496, 3483246591, ++STORE, 3483246592, 3483250687, ++STORE, 3483398144, 3483688959, ++STORE, 3483688960, 3484114943, ++STORE, 3484114944, 3484131327, ++STORE, 3484131328, 3484135423, ++STORE, 3484135424, 3484143615, ++STORE, 3484184576, 3484475391, ++STORE, 3484475392, 3485028351, ++STORE, 3485028352, 3485057023, ++STORE, 3485057024, 3485061119, ++STORE, 3485360128, 3485364223, ++STORE, 3485364224, 3485368319, ++STORE, 3485368320, 3485372415, ++STORE, 3485589504, 3485593599, ++STORE, 3485593600, 3485597695, ++STORE, 3485597696, 3485601791, ++STORE, 3485913088, 3485937663, ++STORE, 3485937664, 3485974527, ++STORE, 3485974528, 3485982719, ++STORE, 3485982720, 3485986815, ++STORE, 3486052352, 3486056447, ++STORE, 3486056448, 3486064639, ++STORE, 3486064640, 3486068735, ++STORE, 3486068736, 3486072831, ++STORE, 3486294016, 3486302207, ++STORE, 3486302208, 3486306303, ++STORE, 3486306304, 3486310399, ++STORE, 3486310400, 3486314495, ++STORE, 3486670848, 3486679039, ++STORE, 3486679040, 3486683135, ++STORE, 3486683136, 3486687231, ++STORE, 3486687232, 3486691327, ++STORE, 3486863360, 3486871551, ++STORE, 3486871552, 3486875647, ++STORE, 3486875648, 3486879743, ++STORE, 3486879744, 3486883839, ++STORE, 3487584256, 3522543615, ++STORE, 3522543616, 3523321855, ++STORE, 3523321856, 3523342335, ++STORE, 3523342336, 3523387391, ++STORE, 3523387392, 3523391487, ++STORE, 3523391488, 3523395583, ++STORE, 3523477504, 3523686399, ++STORE, 3523686400, 3523981311, ++STORE, 3523981312, 3523997695, ++STORE, 3523997696, 3524001791, ++STORE, 3524177920, 3525013503, ++STORE, 3525013504, 3526582271, ++STORE, 3526582272, 3526606847, ++STORE, 3526606848, 3526610943, ++STORE, 3526610944, 3526615039, ++STORE, 3526672384, 3526746111, ++STORE, 3526746112, 3526860799, ++STORE, 3526860800, 3526868991, ++STORE, 3526868992, 3526873087, ++STORE, 3527000064, 3527475199, ++STORE, 3527475200, 3527479295, ++STORE, 3527479296, 3527573503, ++STORE, 3527573504, 3527581695, ++STORE, 3527581696, 3527585791, ++STORE, 3527585792, 3527606271, ++STORE, 3527909376, 3527913471, ++STORE, 3527913472, 3527917567, ++STORE, 3527917568, 3527921663, ++STORE, 3527950336, 3528011775, ++STORE, 3528011776, 3528093695, ++STORE, 3528093696, 3528101887, ++STORE, 3528101888, 3528105983, ++STORE, 3528228864, 3528241151, ++STORE, 3528241152, 3528261631, ++STORE, 3528261632, 3528265727, ++STORE, 3528273920, 3528593407, ++STORE, 3528593408, 3528609791, ++STORE, 3528609792, 3528638463, ++STORE, 3528638464, 3528642559, ++STORE, 3528642560, 3528646655, ++STORE, 3528880128, 3528912895, ++STORE, 3528912896, 3528962047, ++STORE, 3528962048, 3528966143, ++STORE, 3528966144, 3528970239, ++STORE, 3528982528, 3530293247, ++STORE, 3530366976, 3530825727, ++STORE, 3530825728, 3531317247, ++STORE, 3531317248, 3541041151, ++STORE, 3541041152, 3541303295, ++STORE, 3541430272, 3566206975, ++STORE, 3566206976, 3566993407, ++STORE, 3567239168, 3587571711, ++STORE, 3587571712, 3588284415, ++STORE, 3588284416, 3588661247, ++STORE, 3588661248, 3589066751, ++STORE, 3589066752, 3589574655, ++STORE, 3589574656, 3590078463, ++STORE, 3590078464, 3590373375, ++STORE, 3590373376, 3590668287, ++STORE, 3590668288, 3590963199, ++STORE, 3590963200, 3591294975, ++STORE, 3591294976, 3591602175, ++STORE, 3591602176, 3591933951, ++STORE, 3591933952, 3592241151, ++STORE, 3592241152, 3592572927, ++STORE, 3592572928, 3592876031, ++STORE, 3592876032, 3593211903, ++STORE, 3593211904, 3593547775, ++STORE, 3593547776, 3593650175, ++STORE, 3593650176, 3593928703, ++STORE, 3593928704, 3593936895, ++STORE, 3593936896, 3593940991, ++STORE, 3594006528, 3594301439, ++STORE, 3594301440, 3594739711, ++STORE, 3594739712, 3594756095, ++STORE, 3594756096, 3594760191, ++STORE, 3594760192, 3594768383, ++STORE, 3594952704, 3595051007, ++STORE, 3595051008, 3595223039, ++STORE, 3595223040, 3595227135, ++STORE, 3595227136, 3595235327, ++STORE, 3595431936, 3595775999, ++STORE, 3595776000, 3596701695, ++STORE, 3596701696, 3596742655, ++STORE, 3596742656, 3596746751, ++STORE, 3596746752, 3596750847, ++STORE, 3596767232, 3597070335, ++STORE, 3597070336, 3597402111, ++STORE, 3597402112, 3598188543, ++STORE, 3598262272, 3623428095, ++STORE, 3623428096, 3623432191, ++STORE, 3623432192, 3623436287, ++STORE, 3623436288, 3623440383, ++STORE, 3623616512, 3623878655, ++STORE, 3624169472, 3624300543, ++STORE, 3627524096, 3628523519, ++STORE, 3628523520, 3629522943, ++STORE, 3696631808, 3730186239, ++STORE, 3730186240, 3763740671, ++STORE, 3763740672, 3764027391, ++STORE, 3764027392, 3765133311, ++STORE, 3765133312, 3765145599, ++STORE, 3765145600, 3765149695, ++STORE, 3765178368, 3766022143, ++STORE, 3766022144, 3768791039, ++STORE, 3768791040, 3768840191, ++STORE, 3768840192, 3768844287, ++STORE, 3768897536, 3768913919, ++STORE, 3768913920, 3768934399, ++STORE, 3768934400, 3768938495, ++STORE, 3769016320, 3769147391, ++STORE, 3769147392, 3769233407, ++STORE, 3769233408, 3769356287, ++STORE, 3769356288, 3769360383, ++STORE, 3769360384, 3769368575, ++STORE, 3769376768, 3794542591, ++STORE, 3794542592, 3794599935, ++STORE, 3794599936, 3794731007, ++STORE, 3794731008, 3794735103, ++STORE, 3794735104, 3794743295, ++STORE, 3794849792, 3794980863, ++STORE, 3794980864, 3794984959, ++STORE, 3794984960, 3794989055, ++STORE, 3794989056, 3794993151, ++STORE, 3794993152, 3794997247, ++STORE, 3795103744, 3795128319, ++STORE, 3795128320, 3795165183, ++STORE, 3795165184, 3795169279, ++STORE, 3795169280, 3795173375, ++STORE, 3795210240, 3795357695, ++STORE, 3795357696, 3795365887, ++STORE, 3795365888, 3795374079, ++STORE, 3795374080, 3795378175, ++STORE, 3795378176, 3795382271, ++STORE, 3795406848, 3795738623, ++STORE, 3795738624, 3795742719, ++STORE, 3795742720, 3795755007, ++STORE, 3795755008, 3795759103, ++STORE, 3795763200, 3795894271, ++STORE, 3795894272, 3796041727, ++STORE, 3796041728, 3796054015, ++STORE, 3796054016, 3796066303, ++STORE, 3796066304, 3796070399, ++STORE, 3796176896, 3796205567, ++STORE, 3796205568, 3796250623, ++STORE, 3796250624, 3796254719, ++STORE, 3796254720, 3796258815, ++STORE, 3796262912, 3796393983, ++STORE, 3796393984, 3796516863, ++STORE, 3796516864, 3796873215, ++STORE, 3796873216, 3796885503, ++STORE, 3796885504, 3796889599, ++STORE, 3796963328, 3796967423, ++STORE, 3796967424, 3796975615, ++STORE, 3796975616, 3796979711, ++STORE, 3797000192, 3797307391, ++STORE, 3797307392, 3797311487, ++STORE, 3797311488, 3797315583, ++STORE, 3797315584, 3797323775, ++STORE, 3797327872, 3797450751, ++STORE, 3797450752, 3797458943, ++STORE, 3797458944, 3797471231, ++STORE, 3797471232, 3797475327, ++STORE, 3797577728, 3797700607, ++STORE, 3797700608, 3797721087, ++STORE, 3797721088, 3797733375, ++STORE, 3797733376, 3797741567, ++STORE, 3797741568, 3797864447, ++STORE, 3797864448, 3797995519, ++STORE, 3797995520, 3798048767, ++STORE, 3798048768, 3798179839, ++STORE, 3798179840, 3798188031, ++STORE, 3798188032, 3798192127, ++STORE, 3798290432, 3798302719, ++STORE, 3798302720, 3798323199, ++STORE, 3798323200, 3798327295, ++STORE, 3798327296, 3798331391, ++STORE, 3798429696, 3798433791, ++STORE, 3798433792, 3798552575, ++STORE, 3798552576, 3798556671, ++STORE, 3798556672, 3798568959, ++STORE, 3798568960, 3798573055, ++STORE, 3798573056, 3798581247, ++STORE, 3798618112, 3798749183, ++STORE, 3798749184, 3798855679, ++STORE, 3798855680, 3798966271, ++STORE, 3798966272, 3798982655, ++STORE, 3798982656, 3798986751, ++STORE, 3799101440, 3799171071, ++STORE, 3799171072, 3799240703, ++STORE, 3799240704, 3799248895, ++STORE, 3799248896, 3799252991, ++STORE, 3799326720, 3799650303, ++STORE, 3799650304, 3800629247, ++STORE, 3800629248, 3800641535, ++STORE, 3800641536, 3800645631, ++STORE, 3800645632, 3800649727, ++STORE, 3800649728, 3800903679, ++STORE, 3800903680, 3800936447, ++STORE, 3800936448, 3800969215, ++STORE, 3800969216, 3800981503, ++STORE, 3800981504, 3800985599, ++STORE, 3801001984, 3801133055, ++STORE, 3801133056, 3801202687, ++STORE, 3801202688, 3801591807, ++STORE, 3801591808, 3801599999, ++STORE, 3801600000, 3801604095, ++STORE, 3801604096, 3801608191, ++STORE, 3801608192, 3801739263, ++STORE, 3801739264, 3801755647, ++STORE, 3801755648, 3801796607, ++STORE, 3801796608, 3801804799, ++STORE, 3801804800, 3801808895, ++STORE, 3801878528, 3801944063, ++STORE, 3801944064, 3802116095, ++STORE, 3802116096, 3802124287, ++STORE, 3802124288, 3802128383, ++STORE, 3802136576, 3803447295, ++STORE, 3803492352, 3803553791, ++STORE, 3803553792, 3804233727, ++STORE, 3804233728, 3806068735, ++STORE, 3806121984, 3806253055, ++STORE, 3806253056, 3806674943, ++STORE, 3806674944, 3807117311, ++STORE, 3807117312, 3807379455, ++STORE, 3807379456, 3807432703, ++STORE, 3807432704, 3807563775, ++STORE, 3807563776, 3809202175, ++STORE, 3809202176, 3810250751, ++STORE, 3810250752, 3827027967, ++STORE, 3827027968, 3829125119, ++STORE, 3829125120, 3837513727, ++STORE, 3837513728, 3839610879, ++STORE, 3839610880, 3847999487, ++STORE, 3847999488, 3856392191, ++STORE, 3856392192, 3864784895, ++STORE, 3864784896, 3868983295, ++STORE, 3868983296, 3885760511, ++STORE, 3885760512, 3886809087, ++STORE, 3886809088, 3887857663, ++STORE, 3887857664, 3888119807, ++STORE, 3888144384, 3888148479, ++STORE, 3888148480, 3888218111, ++STORE, 3888218112, 3888222207, ++STORE, 3888222208, 3888353279, ++STORE, 3888353280, 3889172479, ++STORE, 3889172480, 3892314111, ++STORE, 3892314112, 3892576255, ++STORE, 3892588544, 3892637695, ++STORE, 3892637696, 3892686847, ++STORE, 3892686848, 3892744191, ++STORE, 3892748288, 3892785151, ++STORE, 3892785152, 3895459839, ++STORE, 3895459840, 3895721983, ++STORE, 3895738368, 3895885823, ++STORE, 3895885824, 3897081855, ++STORE, 3897081856, 3906482175, ++STORE, 3906482176, 3916144639, ++STORE, 3916144640, 3925766143, ++STORE, 3925766144, 3926974463, ++STORE, 3926974464, 3928367103, ++STORE, 3928367104, 3928911871, ++STORE, 3928911872, 3933995007, ++STORE, 3933995008, 3935830015, ++STORE, 3935830016, 3935846399, ++STORE, 3935879168, 3936010239, ++STORE, 3936010240, 3936026623, ++STORE, 3936026624, 3936034815, ++STORE, 3936034816, 3936051199, ++STORE, 3936051200, 3936055295, ++STORE, 3936071680, 3936137215, ++STORE, 3936137216, 3936202751, ++STORE, 3936202752, 3936219135, ++STORE, 3936235520, 3936251903, ++STORE, 3936268288, 3936276479, ++STORE, 3936276480, 3936284671, ++STORE, 3936284672, 3936288767, ++STORE, 3936288768, 3936292863, ++STORE, 3936296960, 3936354303, ++STORE, 3936354304, 3936616447, ++STORE, 3936628736, 3936669695, ++STORE, 3936669696, 3936747519, ++STORE, 3936747520, 3936870399, ++STORE, 3936870400, 3936874495, ++STORE, 3936874496, 3936878591, ++STORE, 3936882688, 3936903167, ++STORE, 3936911360, 3936948223, ++STORE, 3936948224, 3936964607, ++STORE, 3936964608, 3937103871, ++STORE, 3937103872, 3937107967, ++STORE, 3937132544, 3937161215, ++STORE, 3937189888, 3937255423, ++STORE, 3937255424, 3938512895, ++STORE, 3938512896, 3945435135, ++STORE, 3945435136, 3945476095, ++STORE, 3945476096, 3945484287, ++STORE, 3945484288, 3945496575, ++STORE, 3945500672, 3945541631, ++STORE, 3945558016, 3945566207, ++STORE, 3945566208, 3945594879, ++STORE, 3945594880, 3945598975, ++STORE, 3945598976, 3945603071, ++STORE, 3945611264, 3945742335, ++STORE, 3945742336, 3945844735, ++STORE, 3945844736, 3945848831, ++STORE, 3945848832, 3945861119, ++STORE, 3945861120, 3945865215, ++STORE, 3945869312, 3945897983, ++STORE, 3945897984, 3946303487, ++STORE, 3946303488, 3946397695, ++STORE, 3946397696, 3946569727, ++STORE, 3946569728, 3946573823, ++STORE, 3946573824, 3946594303, ++STORE, 3946594304, 3946663935, ++STORE, 3946663936, 3946708991, ++STORE, 3946708992, 3946823679, ++STORE, 3946823680, 3946827775, ++STORE, 3946827776, 3946831871, ++STORE, 3946831872, 3946860543, ++STORE, 3946893312, 3946897407, ++STORE, 3946897408, 3946905599, ++STORE, 3946905600, 3946909695, ++STORE, 3946909696, 3946913791, ++STORE, 3946913792, 3946930175, ++STORE, 3946930176, 3946967039, ++STORE, 3946967040, 3947102207, ++STORE, 3947102208, 3948412927, ++STORE, 3948441600, 3948556287, ++STORE, 3948556288, 3948576767, ++STORE, 3948576768, 3948597247, ++STORE, 3948597248, 3948605439, ++STORE, 3948605440, 3948609535, ++STORE, 3948609536, 3948654591, ++STORE, 3948654592, 3948781567, ++STORE, 3948781568, 3948822527, ++STORE, 3948822528, 3948904447, ++STORE, 3948904448, 3948908543, ++STORE, 3948908544, 3948912639, ++STORE, 3948945408, 3949043711, ++STORE, 3949043712, 3949174783, ++STORE, 3949174784, 3949191167, ++STORE, 3949191168, 3949195263, ++STORE, 3949207552, 3949252607, ++STORE, 3949252608, 3949256703, ++STORE, 3949256704, 3949363199, ++STORE, 3949363200, 3949367295, ++STORE, 3949367296, 3949379583, ++STORE, 3949379584, 3949383679, ++STORE, 3949383680, 3949400063, ++STORE, 3949400064, 3949404159, ++STORE, 3949416448, 3949481983, ++STORE, 3949481984, 3949486079, ++STORE, 3949486080, 3949592575, ++STORE, 3949592576, 3949596671, ++STORE, 3949596672, 3949621247, ++STORE, 3949621248, 3949662207, ++STORE, 3949662208, 3949666303, ++STORE, 3949694976, 3949727743, ++STORE, 3949727744, 3949731839, ++STORE, 3949731840, 3949838335, ++STORE, 3949838336, 3949842431, ++STORE, 3949842432, 3949846527, ++STORE, 3949846528, 3949854719, ++STORE, 3949854720, 3949858815, ++STORE, 3949858816, 3949862911, ++STORE, 3949867008, 3949891583, ++STORE, 3949891584, 3949928447, ++STORE, 3949928448, 3949993983, ++STORE, 3949993984, 3950043135, ++STORE, 3950043136, 3950059519, ++STORE, 3950059520, 3950096383, ++STORE, 3950096384, 3950100479, ++STORE, 3950100480, 3950104575, ++STORE, 3950104576, 3950157823, ++STORE, 3950157824, 3950292991, ++STORE, 3950292992, 3950346239, ++STORE, 3950346240, 3950477311, ++STORE, 3950477312, 3950485503, ++STORE, 3950485504, 3950489599, ++STORE, 3950493696, 3950510079, ++STORE, 3950510080, 3950661631, ++STORE, 3950661632, 3951005695, ++STORE, 3951005696, 3951026175, ++STORE, 3951026176, 3951030271, ++STORE, 3951030272, 3951054847, ++STORE, 3951054848, 3951116287, ++STORE, 3951116288, 3951144959, ++STORE, 3951144960, 3951149055, ++STORE, 3951149056, 3951194111, ++STORE, 3951194112, 3951202303, ++STORE, 3951202304, 3951206399, ++STORE, 3951210496, 3951226879, ++STORE, 3951226880, 3951329279, ++STORE, 3951329280, 3951366143, ++STORE, 3951366144, 3951411199, ++STORE, 3951411200, 3951415295, ++STORE, 3951415296, 3951419391, ++STORE, 3951419392, 3951452159, ++STORE, 3951452160, 3951566847, ++STORE, 3951566848, 3951812607, ++STORE, 3951812608, 3952173055, ++STORE, 3952173056, 3952214015, ++STORE, 3952214016, 3952218111, ++STORE, 3952222208, 3952250879, ++STORE, 3952250880, 3952369663, ++STORE, 3952369664, 3952488447, ++STORE, 3952488448, 3952627711, ++STORE, 3952627712, 3952635903, ++STORE, 3952635904, 3952639999, ++STORE, 3952652288, 3952668671, ++STORE, 3952668672, 3953000447, ++STORE, 3953000448, 3953004543, ++STORE, 3953004544, 3953008639, ++STORE, 3953008640, 3953012735, ++STORE, 3953012736, 3953037311, ++STORE, 3953037312, 3953151999, ++STORE, 3953152000, 3953291263, ++STORE, 3953291264, 3953324031, ++STORE, 3953324032, 3953364991, ++STORE, 3953364992, 3953373183, ++STORE, 3953373184, 3953377279, ++STORE, 3953381376, 3953410047, ++STORE, 3953410048, 3953491967, ++STORE, 3953491968, 3953643519, ++STORE, 3953643520, 3953651711, ++STORE, 3953651712, 3953655807, ++STORE, 3953659904, 3953766399, ++STORE, 3953766400, 3953774591, ++STORE, 3953774592, 3953786879, ++STORE, 3953786880, 3953790975, ++STORE, 3953790976, 3953823743, ++STORE, 3953823744, 3953963007, ++STORE, 3953963008, 3954024447, ++STORE, 3954024448, 3954118655, ++STORE, 3954118656, 3954122751, ++STORE, 3954122752, 3954126847, ++STORE, 3954130944, 3954184191, ++STORE, 3954184192, 3954294783, ++STORE, 3954294784, 3954323455, ++STORE, 3954323456, 3954393087, ++STORE, 3954393088, 3954397183, ++STORE, 3954397184, 3954401279, ++STORE, 3954401280, 3954405375, ++STORE, 3954409472, 3954528255, ++STORE, 3954528256, 3954737151, ++STORE, 3954737152, 3955052543, ++STORE, 3955052544, 3955060735, ++STORE, 3955060736, 3955064831, ++STORE, 3955068928, 3955105791, ++STORE, 3955105792, 3955167231, ++STORE, 3955167232, 3955277823, ++STORE, 3955277824, 3955310591, ++STORE, 3955310592, 3955351551, ++STORE, 3955351552, 3955359743, ++STORE, 3955359744, 3955363839, ++STORE, 3955363840, 3955392511, ++STORE, 3955392512, 3955453951, ++STORE, 3955453952, 3955601407, ++STORE, 3955601408, 3955777535, ++STORE, 3955777536, 3955982335, ++STORE, 3955982336, 3956011007, ++STORE, 3956011008, 3956015103, ++STORE, 3956023296, 3956039679, ++STORE, 3956039680, 3956125695, ++STORE, 3956125696, 3956129791, ++STORE, 3956129792, 3956133887, ++STORE, 3956133888, 3956137983, ++STORE, 3956142080, 3956449279, ++STORE, 3956449280, 3956543487, ++STORE, 3956543488, 3956719615, ++STORE, 3956719616, 3956731903, ++STORE, 3956731904, 3956735999, ++STORE, 3956744192, 3956793343, ++STORE, 3956793344, 3956887551, ++STORE, 3956887552, 3956953087, ++STORE, 3956953088, 3957035007, ++STORE, 3957035008, 3957039103, ++STORE, 3957039104, 3957047295, ++STORE, 3957047296, 3957071871, ++STORE, 3957071872, 3957231615, ++STORE, 3957231616, 3957563391, ++STORE, 3957563392, 3957579775, ++STORE, 3957579776, 3957583871, ++STORE, 3957592064, 3957608447, ++STORE, 3957608448, 3957878783, ++STORE, 3957878784, 3958591487, ++STORE, 3958591488, 3958599679, ++STORE, 3958599680, 3958607871, ++STORE, 3958607872, 3958620159, ++STORE, 3958620160, 3958624255, ++STORE, 3958624256, 3963199487, ++STORE, 3963199488, 3963285503, ++STORE, 3963285504, 3963371519, ++STORE, 3963371520, 3963428863, ++STORE, 3963428864, 3963555839, ++STORE, 3963555840, 3963559935, ++STORE, 3963559936, 3963564031, ++STORE, 3963568128, 3963596799, ++STORE, 3963596800, 3963682815, ++STORE, 3963682816, 3963695103, ++STORE, 3963695104, 3963711487, ++STORE, 3963711488, 3963715583, ++STORE, 3963719680, 3963752447, ++STORE, 3963752448, 3963846655, ++STORE, 3963846656, 3963932671, ++STORE, 3963932672, 3964444671, ++STORE, 3964444672, 3964448767, ++STORE, 3964448768, 3965808639, ++STORE, 3965808640, 3965845503, ++STORE, 3965845504, 3965849599, ++STORE, 3965853696, 3965935615, ++STORE, 3965935616, 3966017535, ++STORE, 3966017536, 3966103551, ++STORE, 3966103552, 3966685183, ++STORE, 3966685184, 3967705087, ++STORE, 3967705088, 3967758335, ++STORE, 3967758336, 3967762431, ++STORE, 3967762432, 3967770623, ++STORE, 3967770624, 3967799295, ++STORE, 3967799296, 3967848447, ++STORE, 3967848448, 3967868927, ++STORE, 3967868928, 3967901695, ++STORE, 3967901696, 3967905791, ++STORE, 3967905792, 3967909887, ++STORE, 3967909888, 3967995903, ++STORE, 3967995904, 3968077823, ++STORE, 3968077824, 3968159743, ++STORE, 3968159744, 3968167935, ++STORE, 3968167936, 3968172031, ++STORE, 3968172032, 3968192511, ++STORE, 3968192512, 3968196607, ++STORE, 3968196608, 3968200703, ++STORE, 3968208896, 3968516095, ++STORE, 3968516096, 3968528383, ++STORE, 3968528384, 3968552959, ++STORE, 3968552960, 3968557055, ++STORE, 3968561152, 3968593919, ++STORE, 3968593920, 3968626687, ++STORE, 3968626688, 3971153919, ++STORE, 3971153920, 3973754879, ++STORE, 3973754880, 3973804031, ++STORE, 3973804032, 3973820415, ++STORE, 3973820416, 3973832703, ++STORE, 3973840896, 3973873663, ++STORE, 3973873664, 3973967871, ++STORE, 3973967872, 3973976063, ++STORE, 3973976064, 3973984255, ++STORE, 3973984256, 3973988351, ++STORE, 3973988352, 3973992447, ++STORE, 3973996544, 3974008831, ++STORE, 3974008832, 3974045695, ++STORE, 3974045696, 3974139903, ++STORE, 3974139904, 3974254591, ++STORE, 3974254592, 3974275071, ++STORE, 3974275072, 3974291455, ++STORE, 3974291456, 3974295551, ++STORE, 3974295552, 3974373375, ++STORE, 3974373376, 3974524927, ++STORE, 3974524928, 3974529023, ++STORE, 3974529024, 3974537215, ++STORE, 3974537216, 3974541311, ++STORE, 3974541312, 3974545407, ++STORE, 3974545408, 3974627327, ++STORE, 3974627328, 3974680575, ++STORE, 3974680576, 3974811647, ++STORE, 3974811648, 3974819839, ++STORE, 3974819840, 3974823935, ++STORE, 3974832128, 3974918143, ++STORE, 3974918144, 3974963199, ++STORE, 3974963200, 3975077887, ++STORE, 3975077888, 3975090175, ++STORE, 3975090176, 3975094271, ++STORE, 3975094272, 3975102463, ++STORE, 3975102464, 3975114751, ++STORE, 3975114752, 3975266303, ++STORE, 3975266304, 3975274495, ++STORE, 3975274496, 3975286783, ++STORE, 3975286784, 3975290879, ++STORE, 3975290880, 3975299071, ++STORE, 3975299072, 3975315455, ++STORE, 3975315456, 3975430143, ++STORE, 3975430144, 3975536639, ++STORE, 3975536640, 3975651327, ++STORE, 3975651328, 3975655423, ++STORE, 3975655424, 3975659519, ++STORE, 3975659520, 3975770111, ++STORE, 3975770112, 3975778303, ++STORE, 3975778304, 3975790591, ++STORE, 3975790592, 3975794687, ++STORE, 3975794688, 3975798783, ++STORE, 3975798784, 3975831551, ++STORE, 3975831552, 3975872511, ++STORE, 3975872512, 3975987199, ++STORE, 3975987200, 3976134655, ++STORE, 3976134656, 3977175039, ++STORE, 3977175040, 3977183231, ++STORE, 3977183232, 3977191423, ++STORE, 3977191424, 3977195519, ++STORE, 3977199616, 3977248767, ++STORE, 3977248768, 3977539583, ++STORE, 3977539584, 3977965567, ++STORE, 3977965568, 3977981951, ++STORE, 3977981952, 3977986047, ++STORE, 3977986048, 3977994239, ++STORE, 3977994240, 3978002431, ++STORE, 3978002432, 3978084351, ++STORE, 3978084352, 3978125311, ++STORE, 3978125312, 3978174463, ++STORE, 3978174464, 3978178559, ++STORE, 3978178560, 3978182655, ++STORE, 3978182656, 3978207231, ++STORE, 3978207232, 3978297343, ++STORE, 3978297344, 3978301439, ++STORE, 3978301440, 3978305535, ++STORE, 3978305536, 3978309631, ++STORE, 3978309632, 3978317823, ++STORE, 3978317824, 3978625023, ++STORE, 3978625024, 3978657791, ++STORE, 3978657792, 3978727423, ++STORE, 3978727424, 3978735615, ++STORE, 3978735616, 3978739711, ++STORE, 3978739712, 3978760191, ++STORE, 3978760192, 3978842111, ++STORE, 3978842112, 3978850303, ++STORE, 3978850304, 3978858495, ++STORE, 3978858496, 3978862591, ++STORE, 3978862592, 3978895359, ++STORE, 3978895360, 3979014143, ++STORE, 3979014144, 3979132927, ++STORE, 3979132928, 3979288575, ++STORE, 3979288576, 3979481087, ++STORE, 3979481088, 3979489279, ++STORE, 3979489280, 3979493375, ++STORE, 3979497472, 3979583487, ++STORE, 3979583488, 3979673599, ++STORE, 3979673600, 3979718655, ++STORE, 3979718656, 3979829247, ++STORE, 3979829248, 3979841535, ++STORE, 3979841536, 3979882495, ++STORE, 3979882496, 3979964415, ++STORE, 3979964416, 3980013567, ++STORE, 3980013568, 3980148735, ++STORE, 3980148736, 3980152831, ++STORE, 3980152832, 3980320767, ++STORE, 3980320768, 3980337151, ++STORE, 3980337152, 3980341247, ++STORE, 3980345344, 3980365823, ++STORE, 3980365824, 3980423167, ++STORE, 3980423168, 3980460031, ++STORE, 3980460032, 3980500991, ++STORE, 3980500992, 3980509183, ++STORE, 3980509184, 3980513279, ++STORE, 3980513280, 3980546047, ++STORE, 3980546048, 3980660735, ++STORE, 3980660736, 3980951551, ++STORE, 3980951552, 3981500415, ++STORE, 3981500416, 3981529087, ++STORE, 3981529088, 3981533183, ++STORE, 3981537280, 3981549567, ++STORE, 3981549568, 3981598719, ++STORE, 3981598720, 3981717503, ++STORE, 3981717504, 3982127103, ++STORE, 3982127104, 3982675967, ++STORE, 3982675968, 3982733311, ++STORE, 3982733312, 3982737407, ++STORE, 3982741504, 3982860287, ++STORE, 3982860288, 3982905343, ++STORE, 3982905344, 3982966783, ++STORE, 3982966784, 3982974975, ++STORE, 3982974976, 3982979071, ++STORE, 3982979072, 3983032319, ++STORE, 3983032320, 3983085567, ++STORE, 3983085568, 3983208447, ++STORE, 3983208448, 3983212543, ++STORE, 3983212544, 3983220735, ++STORE, 3983220736, 3983224831, ++STORE, 3983224832, 3983237119, ++STORE, 3983237120, 3983351807, ++STORE, 3983351808, 3983376383, ++STORE, 3983376384, 3983392767, ++STORE, 3983392768, 3983396863, ++STORE, 3983396864, 3983400959, ++STORE, 3983400960, 3983417343, ++STORE, 3983417344, 3983753215, ++STORE, 3983753216, 3983757311, ++STORE, 3983757312, 3983761407, ++STORE, 3983761408, 3983765503, ++STORE, 3983765504, 3983769599, ++STORE, 3983769600, 3983880191, ++STORE, 3983880192, 3983892479, ++STORE, 3983892480, 3983900671, ++STORE, 3983900672, 3983904767, ++STORE, 3983904768, 3983908863, ++STORE, 3983908864, 3983941631, ++STORE, 3983941632, 3983990783, ++STORE, 3983990784, 3984097279, ++STORE, 3984097280, 3984105471, ++STORE, 3984105472, 3984117759, ++STORE, 3984117760, 3984121855, ++STORE, 3984121856, 3984125951, ++STORE, 3984125952, 3984134143, ++STORE, 3984134144, 3984150527, ++STORE, 3984150528, 3984416767, ++STORE, 3984416768, 3984470015, ++STORE, 3984470016, 3984564223, ++STORE, 3984564224, 3984568319, ++STORE, 3984572416, 3984629759, ++STORE, 3984629760, 3984805887, ++STORE, 3984805888, 3985096703, ++STORE, 3985096704, 3985104895, ++STORE, 3985104896, 3985108991, ++STORE, 3985113088, 3986862079, ++STORE, 3986862080, 3993640959, ++STORE, 3993640960, 3993739263, ++STORE, 3993739264, 3993743359, ++STORE, 3993743360, 3993759743, ++STORE, 3993759744, 3993780223, ++STORE, 3993780224, 3993784319, ++STORE, 3993784320, 3993792511, ++STORE, 3993792512, 3993796607, ++STORE, 3993796608, 3993800703, ++STORE, 3993804800, 3994214399, ++STORE, 3994214400, 3994218495, ++STORE, 3994218496, 3994222591, ++STORE, 3994222592, 3994226687, ++STORE, 3994230784, 3994243071, ++STORE, 3994243072, 3994255359, ++STORE, 3994255360, 3994304511, ++STORE, 3994304512, 3994386431, ++STORE, 3994386432, 3994509311, ++STORE, 3994509312, 3994521599, ++STORE, 3994521600, 3994525695, ++STORE, 3994529792, 3994542079, ++STORE, 3994542080, 3994660863, ++STORE, 3994660864, 3994705919, ++STORE, 3994705920, 3994796031, ++STORE, 3994796032, 3994800127, ++STORE, 3994800128, 3994804223, ++STORE, 3994804224, 3994812415, ++STORE, 3994812416, 3994845183, ++STORE, 3994845184, 3994898431, ++STORE, 3994898432, 3994902527, ++STORE, 3994902528, 3994906623, ++STORE, 3994910720, 3994931199, ++STORE, 3994931200, 3995181055, ++STORE, 3995181056, 3995222015, ++STORE, 3995222016, 3995275263, ++STORE, 3995275264, 3995279359, ++STORE, 3995279360, 3995283455, ++STORE, 3995283456, 3995291647, ++STORE, 3995291648, 3995324415, ++STORE, 3995324416, 3995451391, ++STORE, 3995451392, 3995697151, ++STORE, 3995697152, 3996078079, ++STORE, 3996078080, 3996086271, ++STORE, 3996086272, 3996090367, ++STORE, 3996094464, 3996119039, ++STORE, 3996119040, 3996200959, ++STORE, 3996200960, 3996229631, ++STORE, 3996229632, 3996233727, ++STORE, 3996233728, 3996282879, ++STORE, 3996282880, 3996291071, ++STORE, 3996291072, 3996295167, ++STORE, 3996299264, 3996311551, ++STORE, 3996311552, 3996430335, ++STORE, 3996430336, 3996467199, ++STORE, 3996467200, 3996504063, ++STORE, 3996504064, 3996512255, ++STORE, 3996512256, 3996516351, ++STORE, 3996516352, 3996540927, ++STORE, 3996540928, 3996671999, ++STORE, 3996672000, 3996676095, ++STORE, 3996676096, 3996684287, ++STORE, 3996684288, 3996688383, ++STORE, 3996688384, 3996692479, ++STORE, 3996692480, 3996717055, ++STORE, 3996717056, 3997048831, ++STORE, 3997048832, 3997057023, ++STORE, 3997057024, 3997073407, ++STORE, 3997073408, 3997077503, ++STORE, 3997077504, 3997081599, ++STORE, 3997081600, 3997097983, ++STORE, 3997097984, 3997179903, ++STORE, 3997179904, 3997356031, ++STORE, 3997356032, 3997650943, ++STORE, 3997650944, 3997675519, ++STORE, 3997675520, 3997679615, ++STORE, 3997683712, 3997700095, ++STORE, 3997700096, 3997745151, ++STORE, 3997745152, 3997802495, ++STORE, 3997802496, 3997810687, ++STORE, 3997810688, 3997814783, ++STORE, 3997814784, 3998064639, ++STORE, 3998064640, 3998081023, ++STORE, 3998081024, 3998085119, ++STORE, 3998085120, 3998130175, ++STORE, 3998130176, 3998134271, ++STORE, 3998134272, 3998142463, ++STORE, 3998142464, 3998179327, ++STORE, 3998179328, 3998212095, ++STORE, 3998212096, 3998326783, ++STORE, 3998326784, 3998351359, ++STORE, 3998351360, 3998392319, ++STORE, 3998392320, 3998396415, ++STORE, 3998396416, 3998400511, ++STORE, 3998400512, 3998433279, ++STORE, 3998433280, 3998466047, ++STORE, 3998466048, 3998613503, ++STORE, 3998613504, 3998666751, ++STORE, 3998666752, 3998724095, ++STORE, 3998724096, 3998732287, ++STORE, 3998732288, 3998736383, ++STORE, 3998736384, 3998760959, ++STORE, 3998760960, 3998777343, ++STORE, 3998777344, 3998822399, ++STORE, 3998822400, 3998826495, ++STORE, 3998826496, 3998830591, ++STORE, 3998830592, 3998863359, ++STORE, 3998863360, 3998900223, ++STORE, 3998900224, 3999043583, ++STORE, 3999043584, 3999121407, ++STORE, 3999121408, 3999215615, ++STORE, 3999215616, 3999223807, ++STORE, 3999223808, 3999227903, ++STORE, 3999227904, 3999236095, ++STORE, 3999236096, 3999268863, ++STORE, 3999268864, 3999301631, ++STORE, 3999301632, 3999354879, ++STORE, 3999354880, 3999428607, ++STORE, 3999428608, 3999436799, ++STORE, 3999436800, 3999440895, ++STORE, 3999444992, 3999461375, ++STORE, 3999461376, 3999584255, ++STORE, 3999584256, 3999760383, ++STORE, 3999760384, 4000219135, ++STORE, 4000219136, 4000235519, ++STORE, 4000235520, 4000251903, ++STORE, 4000251904, 4000501759, ++STORE, 4000501760, 4000505855, ++STORE, 4000505856, 4000509951, ++STORE, 4000509952, 4000518143, ++STORE, 4000518144, 4000522239, ++STORE, 4000522240, 4000587775, ++STORE, 4000587776, 4000645119, ++STORE, 4000645120, 4000813055, ++STORE, 4000813056, 4000817151, ++STORE, 4000821248, 4000837631, ++STORE, 4000837632, 4000870399, ++STORE, 4000870400, 4000874495, ++STORE, 4000874496, 4000878591, ++STORE, 4000878592, 4000882687, ++STORE, 4000882688, 4000886783, ++STORE, 4000886784, 4000890879, ++STORE, 4000890880, 4000907263, ++STORE, 4000907264, 4001214463, ++STORE, 4001214464, 4001558527, ++STORE, 4001558528, 4002484223, ++STORE, 4002484224, 4002525183, ++STORE, 4002525184, 4002529279, ++STORE, 4002529280, 4002533375, ++STORE, 4002533376, 4002537471, ++STORE, 4002537472, 4002660351, ++STORE, 4002660352, 4002779135, ++STORE, 4002779136, 4002791423, ++STORE, 4002791424, 4002799615, ++STORE, 4002799616, 4002807807, ++STORE, 4002807808, 4002811903, ++STORE, 4002811904, 4002828287, ++STORE, 4002828288, 4002910207, ++STORE, 4002910208, 4003028991, ++STORE, 4003028992, 4003037183, ++STORE, 4003037184, 4003045375, ++STORE, 4003045376, 4003049471, ++STORE, 4003049472, 4003053567, ++STORE, 4003053568, 4003057663, ++STORE, 4003057664, 4003065855, ++STORE, 4003065856, 4003135487, ++STORE, 4003135488, 4003446783, ++STORE, 4003446784, 4003450879, ++STORE, 4003450880, 4003454975, ++STORE, 4003454976, 4003459071, ++STORE, 4003459072, 4003463167, ++STORE, 4003463168, 4003495935, ++STORE, 4003495936, 4003569663, ++STORE, 4003569664, 4003573759, ++STORE, 4003573760, 4003704831, ++STORE, 4003704832, 4003708927, ++STORE, 4003708928, 4003713023, ++STORE, 4003713024, 4003737599, ++STORE, 4003737600, 4003770367, ++STORE, 4003770368, 4003876863, ++STORE, 4003876864, 4003880959, ++STORE, 4003880960, 4003885055, ++STORE, 4003885056, 4003889151, ++STORE, 4003889152, 4003893247, ++STORE, 4003893248, 4003897343, ++STORE, 4003897344, 4003962879, ++STORE, 4003962880, 4004069375, ++STORE, 4004069376, 4004093951, ++STORE, 4004093952, 4004118527, ++STORE, 4004118528, 4004122623, ++STORE, 4004122624, 4004126719, ++STORE, 4004126720, 4004155391, ++STORE, 4004155392, 4004286463, ++STORE, 4004286464, 4004384767, ++STORE, 4004384768, 4004388863, ++STORE, 4004388864, 4004646911, ++STORE, 4004646912, 4004655103, ++STORE, 4004655104, 4004659199, ++STORE, 4004659200, 4004667391, ++STORE, 4004667392, 4004683775, ++STORE, 4004683776, 4004814847, ++STORE, 4004814848, 4004818943, ++STORE, 4004818944, 4004823039, ++STORE, 4004823040, 4004827135, ++STORE, 4004827136, 4004835327, ++STORE, 4004835328, 4004954111, ++STORE, 4004954112, 4005085183, ++STORE, 4005085184, 4005306367, ++STORE, 4005306368, 4005765119, ++STORE, 4005765120, 4005789695, ++STORE, 4005789696, 4005793791, ++STORE, 4005793792, 4005801983, ++STORE, 4005801984, 4005920767, ++STORE, 4005920768, 4005945343, ++STORE, 4005945344, 4005949439, ++STORE, 4005949440, 4005986303, ++STORE, 4005986304, 4005990399, ++STORE, 4005990400, 4005994495, ++STORE, 4005994496, 4006002687, ++STORE, 4006002688, 4006109183, ++STORE, 4006109184, 4006117375, ++STORE, 4006117376, 4006121471, ++STORE, 4006121472, 4006133759, ++STORE, 4006133760, 4006137855, ++STORE, 4006137856, 4006141951, ++STORE, 4006141952, 4006150143, ++STORE, 4006150144, 4006391807, ++STORE, 4006391808, 4006445055, ++STORE, 4006445056, 4006563839, ++STORE, 4006563840, 4006572031, ++STORE, 4006572032, 4006576127, ++STORE, 4006576128, 4006584319, ++STORE, 4006584320, 4006694911, ++STORE, 4006694912, 4006739967, ++STORE, 4006739968, 4006776831, ++STORE, 4006776832, 4006785023, ++STORE, 4006785024, 4006789119, ++STORE, 4006789120, 4006797311, ++STORE, 4006797312, 4006813695, ++STORE, 4006813696, 4006846463, ++STORE, 4006846464, 4006977535, ++STORE, 4006977536, 4007006207, ++STORE, 4007006208, 4007010303, ++STORE, 4007010304, 4007067647, ++STORE, 4007067648, 4007075839, ++STORE, 4007075840, 4007084031, ++STORE, 4007084032, 4007100415, ++STORE, 4007100416, 4007116799, ++STORE, 4007116800, 4007133183, ++STORE, 4007133184, 4007153663, ++STORE, 4007153664, 4007178239, ++STORE, 4007178240, 4007202815, ++STORE, 4007202816, 4007206911, ++STORE, 4007206912, 4007272447, ++STORE, 4007272448, 4007276543, ++STORE, 4007276544, 4007280639, ++STORE, 4007280640, 4007284735, ++STORE, 4007284736, 4007292927, ++STORE, 4007292928, 4007423999, ++STORE, 4007424000, 4007448575, ++STORE, 4007448576, 4007452671, ++STORE, 4007452672, 4007505919, ++STORE, 4007505920, 4007510015, ++STORE, 4007510016, 4007514111, ++STORE, 4007514112, 4007645183, ++STORE, 4007645184, 4007776255, ++STORE, 4007776256, 4007780351, ++STORE, 4007780352, 4007784447, ++STORE, 4007784448, 4007788543, ++STORE, 4007788544, 4007809023, ++STORE, 4007809024, 4007829503, ++STORE, 4007829504, 4007960575, ++STORE, 4007960576, 4008091647, ++STORE, 4008091648, 4008296447, ++STORE, 4008296448, 4008890367, ++STORE, 4008890368, 4008898559, ++STORE, 4008898560, 4008902655, ++STORE, 4008902656, 4008996863, ++STORE, 4008996864, 4009041919, ++STORE, 4009041920, 4009082879, ++STORE, 4009082880, 4009091071, ++STORE, 4009091072, 4009107455, ++STORE, 4009107456, 4009349119, ++STORE, 4009349120, 4009373695, ++STORE, 4009373696, 4009414655, ++STORE, 4009414656, 4009422847, ++STORE, 4009422848, 4009426943, ++STORE, 4009426944, 4009447423, ++STORE, 4009447424, 4009471999, ++STORE, 4009472000, 4009512959, ++STORE, 4009512960, 4009594879, ++STORE, 4009594880, 4009598975, ++STORE, 4009598976, 4009697279, ++STORE, 4009697280, 4009713663, ++STORE, 4009713664, 4009717759, ++STORE, 4009717760, 4009721855, ++STORE, 4009721856, 4009730047, ++STORE, 4009730048, 4009861119, ++STORE, 4009861120, 4009951231, ++STORE, 4009951232, 4010131455, ++STORE, 4010131456, 4010135551, ++STORE, 4010135552, 4010139647, ++STORE, 4010139648, 4010143743, ++STORE, 4010143744, 4010164223, ++STORE, 4010164224, 4010295295, ++STORE, 4010295296, 4010299391, ++STORE, 4010299392, 4010491903, ++STORE, 4010491904, 4010495999, ++STORE, 4010496000, 4010668031, ++STORE, 4010668032, 4011028479, ++STORE, 4011028480, 4011053055, ++STORE, 4011053056, 4011057151, ++STORE, 4011057152, 4011118591, ++STORE, 4011118592, 4011126783, ++STORE, 4011126784, 4011130879, ++STORE, 4011130880, 4011143167, ++STORE, 4011143168, 4011147263, ++STORE, 4011147264, 4011167743, ++STORE, 4011167744, 4011171839, ++STORE, 4011171840, 4011360255, ++STORE, 4011360256, 4011364351, ++STORE, 4011364352, 4011626495, ++STORE, 4011626496, 4012216319, ++STORE, 4012216320, 4012228607, ++STORE, 4012228608, 4012232703, ++STORE, 4012232704, 4012236799, ++STORE, 4012236800, 4012240895, ++STORE, 4012240896, 4012261375, ++STORE, 4012261376, 4012392447, ++STORE, 4012392448, 4012466175, ++STORE, 4012466176, 4012597247, ++STORE, 4012597248, 4012601343, ++STORE, 4012601344, 4012605439, ++STORE, 4012605440, 4012609535, ++STORE, 4012609536, 4012679167, ++STORE, 4012679168, 4013563903, ++STORE, 4013563904, 4015366143, ++STORE, 4015366144, 4015411199, ++STORE, 4015411200, 4015415295, ++STORE, 4015415296, 4015419391, ++STORE, 4015419392, 4015542271, ++STORE, 4015542272, 4015550463, ++STORE, 4015550464, 4015558655, ++STORE, 4015558656, 4015562751, ++STORE, 4015562752, 4015583231, ++STORE, 4015583232, 4015587327, ++STORE, 4015587328, 4015603711, ++STORE, 4015665152, 4015669247, ++STORE, 4015669248, 4015812607, ++STORE, 4015812608, 4015816703, ++STORE, 4015816704, 4016111615, ++STORE, 4016111616, 4016467967, ++STORE, 4016467968, 4016508927, ++STORE, 4016508928, 4016517119, ++STORE, 4016517120, 4016525311, ++STORE, 4016525312, 4016586751, ++STORE, 4016586752, 4016664575, ++STORE, 4016664576, 4016697343, ++STORE, 4016697344, 4016742399, ++STORE, 4016742400, 4016746495, ++STORE, 4016746496, 4016750591, ++STORE, 4016750592, 4016758783, ++STORE, 4016799744, 4016844799, ++STORE, 4016844800, 4016902143, ++STORE, 4016902144, 4016992255, ++STORE, 4016992256, 4017000447, ++STORE, 4017000448, 4017004543, ++STORE, 4017004544, 4017008639, ++STORE, 4017008640, 4017016831, ++STORE, 4017016832, 4017020927, ++STORE, 4017020928, 4017127423, ++STORE, 4017127424, 4017131519, ++STORE, 4017131520, 4017229823, ++STORE, 4017229824, 4017422335, ++STORE, 4017422336, 4017438719, ++STORE, 4017438720, 4017442815, ++STORE, 4017442816, 4017446911, ++STORE, 4017446912, 4017455103, ++STORE, 4017455104, 4017766399, ++STORE, 4017766400, 4017909759, ++STORE, 4017909760, 4018081791, ++STORE, 4018081792, 4018089983, ++STORE, 4018089984, 4018094079, ++STORE, 4018094080, 4018098175, ++STORE, 4018098176, 4018327551, ++STORE, 4018327552, 4018331647, ++STORE, 4018331648, 4018339839, ++STORE, 4018339840, 4018348031, ++STORE, 4018348032, 4018610175, ++STORE, 4018610176, 4018626559, ++STORE, 4018626560, 4018647039, ++STORE, 4018647040, 4018651135, ++STORE, 4018651136, 4018749439, ++STORE, 4018749440, 4018761727, ++STORE, 4018761728, 4018802687, ++STORE, 4018802688, 4018806783, ++STORE, 4018806784, 4018810879, ++STORE, 4018810880, 4018814975, ++STORE, 4018814976, 4018823167, ++STORE, 4018823168, 4018954239, ++STORE, 4018954240, 4019007487, ++STORE, 4019007488, 4019068927, ++STORE, 4019068928, 4019077119, ++STORE, 4019077120, 4019081215, ++STORE, 4019081216, 4019093503, ++STORE, 4019093504, 4019208191, ++STORE, 4019208192, 4019232767, ++STORE, 4019232768, 4019265535, ++STORE, 4019265536, 4019269631, ++STORE, 4019269632, 4019277823, ++STORE, 4019277824, 4019458047, ++STORE, 4019458048, 4019519487, ++STORE, 4019519488, 4019613695, ++STORE, 4019613696, 4019621887, ++STORE, 4019621888, 4019625983, ++STORE, 4019625984, 4019630079, ++STORE, 4019630080, 4019744767, ++STORE, 4019744768, 4019822591, ++STORE, 4019822592, 4019929087, ++STORE, 4019929088, 4019941375, ++STORE, 4019941376, 4019945471, ++STORE, 4019945472, 4019961855, ++STORE, 4019961856, 4019994623, ++STORE, 4019994624, 4019998719, ++STORE, 4019998720, 4020002815, ++STORE, 4020002816, 4020006911, ++STORE, 4020006912, 4020011007, ++STORE, 4020011008, 4020256767, ++STORE, 4020256768, 4020326399, ++STORE, 4020326400, 4020457471, ++STORE, 4020457472, 4020469759, ++STORE, 4020469760, 4020473855, ++STORE, 4020473856, 4020482047, ++STORE, 4020482048, 4020711423, ++STORE, 4020711424, 4020715519, ++STORE, 4020715520, 4020719615, ++STORE, 4020719616, 4020723711, ++STORE, 4020723712, 4020805631, ++STORE, 4020805632, 4021051391, ++STORE, 4021051392, 4021460991, ++STORE, 4021460992, 4021469183, ++STORE, 4021469184, 4021473279, ++STORE, 4021473280, 4021571583, ++STORE, 4021571584, 4021633023, ++STORE, 4021633024, 4021727231, ++STORE, 4021727232, 4021735423, ++STORE, 4021735424, 4021739519, ++STORE, 4021739520, 4021747711, ++STORE, 4021747712, 4021829631, ++STORE, 4021829632, 4021866495, ++STORE, 4021866496, 4021919743, ++STORE, 4021919744, 4021927935, ++STORE, 4021927936, 4021932031, ++STORE, 4021932032, 4021944319, ++STORE, 4021944320, 4022157311, ++STORE, 4022157312, 4022161407, ++STORE, 4022161408, 4022173695, ++STORE, 4022173696, 4022177791, ++STORE, 4022177792, 4022472703, ++STORE, 4022472704, 4022509567, ++STORE, 4022509568, 4022583295, ++STORE, 4022583296, 4022587391, ++STORE, 4022587392, 4022591487, ++STORE, 4022591488, 4022607871, ++STORE, 4022607872, 4022657023, ++STORE, 4022657024, 4022722559, ++STORE, 4022722560, 4022730751, ++STORE, 4022730752, 4022734847, ++STORE, 4022734848, 4022865919, ++STORE, 4022865920, 4022943743, ++STORE, 4022943744, 4023062527, ++STORE, 4023062528, 4023074815, ++STORE, 4023074816, 4023078911, ++STORE, 4023078912, 4023128063, ++STORE, 4023128064, 4023218175, ++STORE, 4023218176, 4023361535, ++STORE, 4023361536, 4023373823, ++STORE, 4023373824, 4023377919, ++STORE, 4023377920, 4023558143, ++STORE, 4023558144, 4023631871, ++STORE, 4023631872, 4023816191, ++STORE, 4023816192, 4023820287, ++STORE, 4023820288, 4023824383, ++STORE, 4023824384, 4023832575, ++STORE, 4023832576, 4024078335, ++STORE, 4024078336, 4024197119, ++STORE, 4024197120, 4024389631, ++STORE, 4024389632, 4024406015, ++STORE, 4024406016, 4024410111, ++STORE, 4024410112, 4024422399, ++STORE, 4024422400, 4024619007, ++STORE, 4024619008, 4024639487, ++STORE, 4024639488, 4024655871, ++STORE, 4024655872, 4024664063, ++STORE, 4024664064, 4024668159, ++STORE, 4024668160, 4024676351, ++STORE, 4024676352, 4024905727, ++STORE, 4024905728, 4024909823, ++STORE, 4024909824, 4024918015, ++STORE, 4024918016, 4024922111, ++STORE, 4024922112, 4024930303, ++STORE, 4024930304, 4025110527, ++STORE, 4025110528, 4025176063, ++STORE, 4025176064, 4025208831, ++STORE, 4025208832, 4025212927, ++STORE, 4025212928, 4025217023, ++STORE, 4025217024, 4025348095, ++STORE, 4025348096, 4025372671, ++STORE, 4025372672, 4025458687, ++STORE, 4025458688, 4025466879, ++STORE, 4025466880, 4025565183, ++STORE, 4025565184, 4025757695, ++STORE, 4025757696, 4026249215, ++STORE, 4026249216, 4026261503, ++STORE, 4026261504, 4026265599, ++STORE, 4026265600, 4026269695, ++STORE, 4026269696, 4026302463, ++STORE, 4026302464, 4026306559, ++STORE, 4026306560, 4026314751, ++STORE, 4026314752, 4026318847, ++STORE, 4026318848, 4026322943, ++STORE, 4026322944, 4026327039, ++STORE, 4026327040, 4026654719, ++STORE, 4026654720, 4026671103, ++STORE, 4026671104, 4026720255, ++STORE, 4026720256, 4026724351, ++STORE, 4026724352, 4026728447, ++STORE, 4026728448, 4026732543, ++STORE, 4026732544, 4026863615, ++STORE, 4026863616, 4027027455, ++STORE, 4027027456, 4027031551, ++STORE, 4027031552, 4027514879, ++STORE, 4027514880, 4027531263, ++STORE, 4027531264, 4027535359, ++STORE, 4027535360, 4027539455, ++STORE, 4027539456, 4027785215, ++STORE, 4027785216, 4027789311, ++STORE, 4027789312, 4027793407, ++STORE, 4027793408, 4027797503, ++STORE, 4027797504, 4027863039, ++STORE, 4027863040, 4027899903, ++STORE, 4027899904, 4027949055, ++STORE, 4027949056, 4027957247, ++STORE, 4027957248, 4027961343, ++STORE, 4027961344, 4027965439, ++STORE, 4027965440, 4028194815, ++STORE, 4028194816, 4028252159, ++STORE, 4028252160, 4028338175, ++STORE, 4028338176, 4028350463, ++STORE, 4028350464, 4028354559, ++STORE, 4028354560, 4028452863, ++STORE, 4028452864, 4028489727, ++STORE, 4028489728, 4028530687, ++STORE, 4028530688, 4028538879, ++STORE, 4028538880, 4028542975, ++STORE, 4028542976, 4028551167, ++STORE, 4028551168, 4028665855, ++STORE, 4028665856, 4029349887, ++STORE, 4029349888, 4030468095, ++STORE, 4030468096, 4030513151, ++STORE, 4030513152, 4030517247, ++STORE, 4030517248, 4030525439, ++STORE, 4030525440, 4030529535, ++STORE, 4030529536, 4030758911, ++STORE, 4030758912, 4030828543, ++STORE, 4030828544, 4030943231, ++STORE, 4030943232, 4030951423, ++STORE, 4030951424, 4030955519, ++STORE, 4030955520, 4030967807, ++STORE, 4030967808, 4031131647, ++STORE, 4031131648, 4031135743, ++STORE, 4031135744, 4031139839, ++STORE, 4031139840, 4031148031, ++STORE, 4031148032, 4031152127, ++STORE, 4031152128, 4031160319, ++STORE, 4031160320, 4031504383, ++STORE, 4031504384, 4031598591, ++STORE, 4031598592, 4031754239, ++STORE, 4031754240, 4031766527, ++STORE, 4031766528, 4031770623, ++STORE, 4031770624, 4031774719, ++STORE, 4031774720, 4031782911, ++STORE, 4031782912, 4031799295, ++STORE, 4031799296, 4031856639, ++STORE, 4031856640, 4031983615, ++STORE, 4031983616, 4031987711, ++STORE, 4031987712, 4031991807, ++STORE, 4031991808, 4032270335, ++STORE, 4032270336, 4032274431, ++STORE, 4032274432, 4032282623, ++STORE, 4032282624, 4032286719, ++STORE, 4032286720, 4032290815, ++STORE, 4032290816, 4032389119, ++STORE, 4032389120, 4032397311, ++STORE, 4032397312, 4032405503, ++STORE, 4032405504, 4032413695, ++STORE, 4032413696, 4032417791, ++STORE, 4032417792, 4032565247, ++STORE, 4032565248, 4032593919, ++STORE, 4032593920, 4032737279, ++STORE, 4032737280, 4032741375, ++STORE, 4032741376, 4032745471, ++STORE, 4032745472, 4032770047, ++STORE, 4032770048, 4032933887, ++STORE, 4032933888, 4032999423, ++STORE, 4032999424, 4033032191, ++STORE, 4033032192, 4033036287, ++STORE, 4033036288, 4033040383, ++STORE, 4033040384, 4033105919, ++STORE, 4033105920, 4033396735, ++STORE, 4033396736, 4033822719, ++STORE, 4033822720, 4033839103, ++STORE, 4033839104, 4033843199, ++STORE, 4033843200, 4033851391, ++STORE, 4033851392, 4033863679, ++STORE, 4033863680, 4033880063, ++STORE, 4033880064, 4033933311, ++STORE, 4033933312, 4034023423, ++STORE, 4034023424, 4034031615, ++STORE, 4034031616, 4034035711, ++STORE, 4034035712, 4034043903, ++STORE, 4034043904, 4034142207, ++STORE, 4034142208, 4034191359, ++STORE, 4034191360, 4034260991, ++STORE, 4034260992, 4034269183, ++STORE, 4034269184, 4034273279, ++STORE, 4034273280, 4034281471, ++STORE, 4034281472, 4034412543, ++STORE, 4034412544, 4034445311, ++STORE, 4034445312, 4034490367, ++STORE, 4034490368, 4034494463, ++STORE, 4034494464, 4034498559, ++STORE, 4034498560, 4034662399, ++STORE, 4034662400, 4034666495, ++STORE, 4034666496, 4034670591, ++STORE, 4034670592, 4034674687, ++STORE, 4034674688, 4034678783, ++STORE, 4034678784, 4034682879, ++STORE, 4034682880, 4034781183, ++STORE, 4034781184, 4035043327, ++STORE, 4035043328, 4035047423, ++STORE, 4035047424, 4035055615, ++STORE, 4035055616, 4035059711, ++STORE, 4035059712, 4035063807, ++STORE, 4035063808, 4035067903, ++STORE, 4035067904, 4035100671, ++STORE, 4035100672, 4035375103, ++STORE, 4035375104, 4035383295, ++STORE, 4035383296, 4035395583, ++STORE, 4035395584, 4035399679, ++STORE, 4035399680, 4035403775, ++STORE, 4035403776, 4035407871, ++STORE, 4035407872, 4035411967, ++STORE, 4035411968, 4035477503, ++STORE, 4035477504, 4035608575, ++STORE, 4035608576, 4035641343, ++STORE, 4035641344, 4035682303, ++STORE, 4035682304, 4035686399, ++STORE, 4035686400, 4035690495, ++STORE, 4035690496, 4035694591, ++STORE, 4035694592, 4035743743, ++STORE, 4035743744, 4035784703, ++STORE, 4035784704, 4035829759, ++STORE, 4035829760, 4035837951, ++STORE, 4035837952, 4035842047, ++STORE, 4035842048, 4035846143, ++STORE, 4035846144, 4035850239, ++STORE, 4035850240, 4036001791, ++STORE, 4036001792, 4036005887, ++STORE, 4036005888, 4036214783, ++STORE, 4036214784, 4036218879, ++STORE, 4036218880, 4036603903, ++STORE, 4036603904, 4036648959, ++STORE, 4036648960, 4036653055, ++STORE, 4036653056, 4036657151, ++STORE, 4036657152, 4036665343, ++STORE, 4036665344, 4036780031, ++STORE, 4036780032, 4036829183, ++STORE, 4036829184, 4036984831, ++STORE, 4036984832, 4036993023, ++STORE, 4036993024, 4036997119, ++STORE, 4036997120, 4037001215, ++STORE, 4037001216, 4037009407, ++STORE, 4037009408, 4037025791, ++STORE, 4037025792, 4037095423, ++STORE, 4037095424, 4037181439, ++STORE, 4037181440, 4037193727, ++STORE, 4037193728, 4037197823, ++STORE, 4037197824, 4037206015, ++STORE, 4037206016, 4037320703, ++STORE, 4037320704, 4037337087, ++STORE, 4037337088, 4037349375, ++STORE, 4037349376, 4037357567, ++STORE, 4037357568, 4037361663, ++STORE, 4037369856, 4037386239, ++STORE, 4037386240, 4037672959, ++STORE, 4037672960, 4037689343, ++STORE, 4037689344, 4037730303, ++STORE, 4037730304, 4037734399, ++STORE, 4037734400, 4037738495, ++STORE, 4037738496, 4037742591, ++STORE, 4037742592, 4037758975, ++STORE, 4037758976, 4037890047, ++STORE, 4037890048, 4037931007, ++STORE, 4037931008, 4037976063, ++STORE, 4037976064, 4037984255, ++STORE, 4037984256, 4037988351, ++STORE, 4037988352, 4038053887, ++STORE, 4038053888, 4038184959, ++STORE, 4038184960, 4038189055, ++STORE, 4038189056, 4038197247, ++STORE, 4038197248, 4038201343, ++STORE, 4038201344, 4038205439, ++STORE, 4038205440, 4038209535, ++STORE, 4038217728, 4038250495, ++STORE, 4038250496, 4038512639, ++STORE, 4038512640, 4038516735, ++STORE, 4038516736, 4038520831, ++STORE, 4038520832, 4038524927, ++STORE, 4038524928, 4038529023, ++STORE, 4038529024, 4038533119, ++STORE, 4038541312, 4038623231, ++STORE, 4038623232, 4038754303, ++STORE, 4038754304, 4038885375, ++STORE, 4038885376, 4038889471, ++STORE, 4038897664, 4038963199, ++STORE, 4038963200, 4038967295, ++STORE, 4038967296, 4038983679, ++STORE, 4038983680, 4039114751, ++STORE, 4039114752, 4039245823, ++STORE, 4039245824, 4039376895, ++STORE, 4039376896, 4040687615, ++STORE, 4040687616, 4040691711, ++STORE, 4040691712, 4040806399, ++STORE, 4040806400, 4040937471, ++STORE, 4040937472, 4040941567, ++STORE, 4040945664, 4040949759, ++STORE, 4040949760, 4041080831, ++STORE, 4041080832, 4041211903, ++STORE, 4041211904, 4043046911, ++STORE, 4043046912, 4043051007, ++STORE, 4043051008, 4043055103, ++STORE, 4043055104, 4043137023, ++STORE, 4043137024, 4043141119, ++STORE, 4043141120, 4043145215, ++STORE, 4043145216, 4043153407, ++STORE, 4043153408, 4043186175, ++STORE, 4043186176, 4043317247, ++STORE, 4043317248, 4043448319, ++STORE, 4043448320, 4043579391, ++STORE, 4043579392, 4043583487, ++STORE, 4043583488, 4043599871, ++STORE, 4043599872, 4043661311, ++STORE, 4043661312, 4043792383, ++STORE, 4043792384, 4043796479, ++STORE, 4043796480, 4043800575, ++STORE, 4043800576, 4043816959, ++STORE, 4043816960, 4043821055, ++STORE, 4043821056, 4043825151, ++STORE, 4043825152, 4043829247, ++STORE, 4043829248, 4043833343, ++STORE, 4043833344, 4047241215, ++STORE, 4047241216, 4047249407, ++STORE, 4047249408, 4047253503, ++STORE, 4047253504, 4047323135, ++STORE, 4047323136, 4047327231, ++STORE, 4047327232, 4047458303, ++STORE, 4047458304, 4047589375, ++STORE, 4047589376, 4047720447, ++STORE, 4047720448, 4047773695, ++STORE, 4047773696, 4047790079, ++STORE, 4047790080, 4047921151, ++STORE, 4047921152, 4048052223, ++STORE, 4048052224, 4048183295, ++STORE, 4048183296, 4049002495, ++STORE, 4049002496, 4049133567, ++STORE, 4049133568, 4049154047, ++STORE, 4049154048, 4049158143, ++STORE, 4049158144, 4049162239, ++STORE, 4049162240, 4049166335, ++STORE, 4049166336, 4049174527, ++STORE, 4049174528, 4049182719, ++STORE, 4049182720, 4049186815, ++STORE, 4049186816, 4049190911, ++STORE, 4049190912, 4049195007, ++STORE, 4049195008, 4049203199, ++STORE, 4049203200, 4049207295, ++STORE, 4049207296, 4049211391, ++STORE, 4049211392, 4049215487, ++STORE, 4049215488, 4049219583, ++STORE, 4049219584, 4049227775, ++STORE, 4049227776, 4049231871, ++STORE, 4049231872, 4049235967, ++STORE, 4049235968, 4049244159, ++STORE, 4049244160, 4049248255, ++STORE, 4049248256, 4049252351, ++STORE, 4049252352, 4049256447, ++STORE, 4049256448, 4049268735, ++STORE, 4049268736, 4049272831, ++STORE, 4049272832, 4049313791, ++STORE, 4049313792, 4049723391, ++STORE, 4049723392, 4049727487, ++STORE, 4049727488, 4049858559, ++STORE, 4049858560, 4049989631, ++STORE, 4049989632, 4049993727, ++STORE, 4049993728, 4050026495, ++STORE, 4050026496, 4050030591, ++STORE, 4050030592, 4050161663, ++STORE, 4050161664, 4050169855, ++STORE, 4050169856, 4050223103, ++STORE, 4050223104, 4050632703, ++STORE, 4050632704, 4050636799, ++STORE, 4050636800, 4050640895, ++STORE, 4050640896, 4050644991, ++STORE, 4050644992, 4050661375, ++STORE, 4050661376, 4050665471, ++STORE, 4050665472, 4050673663, ++STORE, 4050673664, 4050677759, ++STORE, 4050677760, 4050694143, ++STORE, 4050694144, 4050702335, ++STORE, 4050702336, 4050956287, ++STORE, 4050956288, 4051963903, ++STORE, 4051963904, 4051980287, ++STORE, 4051980288, 4051988479, ++STORE, 4051988480, 4052000767, ++STORE, 4052000768, 4052004863, ++STORE, 4052004864, 4052029439, ++STORE, 4284014592, 4284018687, ++STORE, 4284018688, 4292403199, ++SNULL, 4041080832, 4041211903, ++SNULL, 3795763200, 3795894271, ++STORE, 3629522944, 3696631807, ++SNULL, 3663077375, 3696631807, ++STORE, 3629522944, 3663077375, ++STORE, 3663077376, 3696631807, ++SNULL, 3663077376, 3696631807, ++STORE, 3663077376, 3696631807, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3626471424, 3627524095, ++SNULL, 3626471424, 3626475519, ++STORE, 3626475520, 3627524095, ++STORE, 3626471424, 3626475519, ++SNULL, 3627519999, 3627524095, ++STORE, 3626475520, 3627519999, ++STORE, 3627520000, 3627524095, ++STORE, 3625418752, 3626475519, ++SNULL, 3625418752, 3625422847, ++STORE, 3625422848, 3626475519, ++STORE, 3625418752, 3625422847, ++SNULL, 3626467327, 3626475519, ++STORE, 3625422848, 3626467327, ++STORE, 3626467328, 3626475519, ++STORE, 3624366080, 3625422847, ++SNULL, 3624366080, 3624370175, ++STORE, 3624370176, 3625422847, ++STORE, 3624366080, 3624370175, ++SNULL, 3625414655, 3625422847, ++STORE, 3624370176, 3625414655, ++STORE, 3625414656, 3625422847, ++STORE, 4041191424, 4041211903, ++SNULL, 4041195519, 4041211903, ++STORE, 4041191424, 4041195519, ++STORE, 4041195520, 4041211903, ++STORE, 4041170944, 4041191423, ++SNULL, 4041175039, 4041191423, ++STORE, 4041170944, 4041175039, ++STORE, 4041175040, 4041191423, ++SNULL, 3625426943, 3626467327, ++STORE, 3625422848, 3625426943, ++STORE, 3625426944, 3626467327, ++STORE, 4041162752, 4041170943, ++SNULL, 3626479615, 3627519999, ++STORE, 3626475520, 3626479615, ++STORE, 3626479616, 3627519999, ++STORE, 4041154560, 4041162751, ++STORE, 4041154560, 4041170943, ++STORE, 4041134080, 4041154559, ++SNULL, 4041138175, 4041154559, ++STORE, 4041134080, 4041138175, ++STORE, 4041138176, 4041154559, ++SNULL, 3624374271, 3625414655, ++STORE, 3624370176, 3624374271, ++STORE, 3624374272, 3625414655, ++STORE, 4041125888, 4041134079, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3487174656, 3487584255, ++STORE, 4041121792, 4041125887, ++SNULL, 4041121792, 4041125887, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 3487174656, 3487584255, ++STORE, 3222274048, 3223326719, ++SNULL, 3222274048, 3222278143, ++STORE, 3222278144, 3223326719, ++STORE, 3222274048, 3222278143, ++SNULL, 3223322623, 3223326719, ++STORE, 3222278144, 3223322623, ++STORE, 3223322624, 3223326719, ++STORE, 3221221376, 3222278143, ++SNULL, 3221221376, 3221225471, ++STORE, 3221225472, 3222278143, ++STORE, 3221221376, 3221225471, ++SNULL, 3222269951, 3222278143, ++STORE, 3221225472, 3222269951, ++STORE, 3222269952, 3222278143, ++STORE, 3220168704, 3221225471, ++SNULL, 3220168704, 3220172799, ++STORE, 3220172800, 3221225471, ++STORE, 3220168704, 3220172799, ++SNULL, 3221217279, 3221225471, ++STORE, 3220172800, 3221217279, ++STORE, 3221217280, 3221225471, ++STORE, 4041117696, 4041125887, ++STORE, 4041117696, 4041134079, ++STORE, 3219083264, 3220172799, ++SNULL, 3219083264, 3219087359, ++STORE, 3219087360, 3220172799, ++STORE, 3219083264, 3219087359, ++SNULL, 3220164607, 3220172799, ++STORE, 3219087360, 3220164607, ++STORE, 3220164608, 3220172799, ++STORE, 4041109504, 4041117695, ++STORE, 4041109504, 4041134079, ++STORE, 3217997824, 3219087359, ++SNULL, 3217997824, 3218001919, ++STORE, 3218001920, 3219087359, ++STORE, 3217997824, 3218001919, ++SNULL, 3219079167, 3219087359, ++STORE, 3218001920, 3219079167, ++STORE, 3219079168, 3219087359, ++STORE, 4041101312, 4041109503, ++STORE, 4041101312, 4041134079, ++STORE, 3216912384, 3218001919, ++SNULL, 3216912384, 3216916479, ++STORE, 3216916480, 3218001919, ++STORE, 3216912384, 3216916479, ++SNULL, 3217993727, 3218001919, ++STORE, 3216916480, 3217993727, ++STORE, 3217993728, 3218001919, ++STORE, 4041093120, 4041101311, ++STORE, 4041093120, 4041134079, ++STORE, 3215826944, 3216916479, ++SNULL, 3215826944, 3215831039, ++STORE, 3215831040, 3216916479, ++STORE, 3215826944, 3215831039, ++SNULL, 3216908287, 3216916479, ++STORE, 3215831040, 3216908287, ++STORE, 3216908288, 3216916479, ++STORE, 4016779264, 4016799743, ++SNULL, 4016783359, 4016799743, ++STORE, 4016779264, 4016783359, ++STORE, 4016783360, 4016799743, ++STORE, 4016758784, 4016779263, ++SNULL, 4016762879, 4016779263, ++STORE, 4016758784, 4016762879, ++STORE, 4016762880, 4016779263, ++SNULL, 3222282239, 3223322623, ++STORE, 3222278144, 3222282239, ++STORE, 3222282240, 3223322623, ++STORE, 4041084928, 4041093119, ++STORE, 4041084928, 4041134079, ++SNULL, 3221229567, 3222269951, ++STORE, 3221225472, 3221229567, ++STORE, 3221229568, 3222269951, ++STORE, 4015644672, 4015665151, ++STORE, 4038889472, 4038897663, ++SNULL, 4015648767, 4015665151, ++STORE, 4015644672, 4015648767, ++STORE, 4015648768, 4015665151, ++STORE, 4015624192, 4015644671, ++SNULL, 4015628287, 4015644671, ++STORE, 4015624192, 4015628287, ++STORE, 4015628288, 4015644671, ++SNULL, 3219091455, 3220164607, ++STORE, 3219087360, 3219091455, ++STORE, 3219091456, 3220164607, ++STORE, 4015603712, 4015624191, ++SNULL, 4015607807, 4015624191, ++STORE, 4015603712, 4015607807, ++STORE, 4015607808, 4015624191, ++SNULL, 3218006015, 3219079167, ++STORE, 3218001920, 3218006015, ++STORE, 3218006016, 3219079167, ++STORE, 3949674496, 3949694975, ++SNULL, 3949678591, 3949694975, ++STORE, 3949674496, 3949678591, ++STORE, 3949678592, 3949694975, ++SNULL, 3216920575, 3217993727, ++STORE, 3216916480, 3216920575, ++STORE, 3216920576, 3217993727, ++STORE, 3948924928, 3948945407, ++SNULL, 3948929023, 3948945407, ++STORE, 3948924928, 3948929023, ++STORE, 3948929024, 3948945407, ++SNULL, 3215835135, 3216908287, ++STORE, 3215831040, 3215835135, ++STORE, 3215835136, 3216908287, ++SNULL, 3220176895, 3221217279, ++STORE, 3220172800, 3220176895, ++STORE, 3220176896, 3221217279, ++STORE, 3214786560, 3215826943, ++STORE, 3213733888, 3214786559, ++SNULL, 3213733888, 3213737983, ++STORE, 3213737984, 3214786559, ++STORE, 3213733888, 3213737983, ++SNULL, 3214782463, 3214786559, ++STORE, 3213737984, 3214782463, ++STORE, 3214782464, 3214786559, ++STORE, 4038533120, 4038541311, ++STORE, 3948421120, 3948441599, ++SNULL, 3948425215, 3948441599, ++STORE, 3948421120, 3948425215, ++STORE, 3948425216, 3948441599, ++SNULL, 3213742079, 3214782463, ++STORE, 3213737984, 3213742079, ++STORE, 3213742080, 3214782463, ++STORE, 4038209536, 4038217727, ++STORE, 3212681216, 3213737983, ++SNULL, 3212681216, 3212685311, ++STORE, 3212685312, 3213737983, ++STORE, 3212681216, 3212685311, ++SNULL, 3213729791, 3213737983, ++STORE, 3212685312, 3213729791, ++STORE, 3213729792, 3213737983, ++STORE, 3795763200, 3795894271, ++STORE, 3946872832, 3946893311, ++SNULL, 3946876927, 3946893311, ++STORE, 3946872832, 3946876927, ++STORE, 3946876928, 3946893311, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3487174656, 3487584255, ++SNULL, 3212689407, 3213729791, ++STORE, 3212685312, 3212689407, ++STORE, 3212689408, 3213729791, ++STORE, 4041080832, 4041084927, ++STORE, 4040941568, 4040945663, ++STORE, 4037361664, 4037369855, ++STORE, 4000817152, 4000821247, ++STORE, 3999440896, 3999444991, ++STORE, 3212161024, 3212681215, ++SNULL, 3212161024, 3212439551, ++STORE, 3212439552, 3212681215, ++STORE, 3212161024, 3212439551, ++SNULL, 3212161024, 3212439551, ++SNULL, 3212464127, 3212681215, ++STORE, 3212439552, 3212464127, ++STORE, 3212464128, 3212681215, ++SNULL, 3212464128, 3212681215, ++SNULL, 3212439552, 3212451839, ++STORE, 3212451840, 3212464127, ++STORE, 3212439552, 3212451839, ++SNULL, 3212439552, 3212451839, ++STORE, 3212439552, 3212451839, ++SNULL, 3212451840, 3212455935, ++STORE, 3212455936, 3212464127, ++STORE, 3212451840, 3212455935, ++SNULL, 3212451840, 3212455935, ++STORE, 3212451840, 3212455935, ++SNULL, 3212455936, 3212460031, ++STORE, 3212460032, 3212464127, ++STORE, 3212455936, 3212460031, ++SNULL, 3212455936, 3212460031, ++STORE, 3212455936, 3212460031, ++SNULL, 3212460032, 3212464127, ++STORE, 3212460032, 3212464127, ++STORE, 3997679616, 3997683711, ++SNULL, 4049235968, 4049240063, ++STORE, 4049240064, 4049244159, ++STORE, 4049235968, 4049240063, ++SNULL, 4049240064, 4049244159, ++STORE, 4049240064, 4049244159, ++SNULL, 3997679616, 3997683711, ++SNULL, 3999440896, 3999444991, ++SNULL, 4000817152, 4000821247, ++SNULL, 4040941568, 4040945663, ++SNULL, 4041080832, 4041084927, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 3487174656, 3487584255, ++SNULL, 3212451840, 3212455935, ++STORE, 3212451840, 3212455935, ++STORE, 4041080832, 4041084927, ++STORE, 3623890944, 3624169471, ++SNULL, 4041080832, 4041084927, ++STORE, 4041080832, 4041084927, ++SNULL, 4041080832, 4041084927, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 4041080832, 4041084927, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3211386880, 3212439551, ++SNULL, 3211386880, 3211390975, ++STORE, 3211390976, 3212439551, ++STORE, 3211386880, 3211390975, ++SNULL, 3212435455, 3212439551, ++STORE, 3211390976, 3212435455, ++STORE, 3212435456, 3212439551, ++STORE, 4040941568, 4040945663, ++STORE, 3937169408, 3937189887, ++STORE, 3623485440, 3623616511, ++SNULL, 717225983, 1388314623, ++STORE, 314572800, 717225983, ++STORE, 717225984, 1388314623, ++SNULL, 717225984, 1388314623, ++STORE, 3937112064, 3937132543, ++SNULL, 3937116159, 3937132543, ++STORE, 3937112064, 3937116159, ++STORE, 3937116160, 3937132543, ++SNULL, 3211395071, 3212435455, ++STORE, 3211390976, 3211395071, ++STORE, 3211395072, 3212435455, ++STORE, 4000817152, 4000821247, ++STORE, 3974823936, 3974832127, ++STORE, 3595284480, 3595431935, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3487174656, 3487584255, ++STORE, 3999440896, 3999444991, ++STORE, 3997679616, 3997683711, ++STORE, 3996295168, 3996299263, ++STORE, 3996090368, 3996094463, ++STORE, 3210866688, 3211386879, ++SNULL, 3210866688, 3211001855, ++STORE, 3211001856, 3211386879, ++STORE, 3210866688, 3211001855, ++SNULL, 3210866688, 3211001855, ++SNULL, 3211038719, 3211386879, ++STORE, 3211001856, 3211038719, ++STORE, 3211038720, 3211386879, ++SNULL, 3211038720, 3211386879, ++SNULL, 3211001856, 3211022335, ++STORE, 3211022336, 3211038719, ++STORE, 3211001856, 3211022335, ++SNULL, 3211001856, 3211022335, ++STORE, 3211001856, 3211022335, ++SNULL, 3211022336, 3211030527, ++STORE, 3211030528, 3211038719, ++STORE, 3211022336, 3211030527, ++SNULL, 3211022336, 3211030527, ++STORE, 3211022336, 3211030527, ++SNULL, 3211030528, 3211034623, ++STORE, 3211034624, 3211038719, ++STORE, 3211030528, 3211034623, ++SNULL, 3211030528, 3211034623, ++STORE, 3211030528, 3211034623, ++SNULL, 3211034624, 3211038719, ++STORE, 3211034624, 3211038719, ++STORE, 3994906624, 3994910719, ++SNULL, 4049240064, 4049244159, ++STORE, 4049240064, 4049244159, ++SNULL, 3994906624, 3994910719, ++SNULL, 3996090368, 3996094463, ++SNULL, 3996295168, 3996299263, ++SNULL, 3997679616, 3997683711, ++SNULL, 3999440896, 3999444991, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 3487174656, 3487584255, ++SNULL, 3211022336, 3211030527, ++STORE, 3211022336, 3211030527, ++STORE, 3999440896, 3999444991, ++STORE, 3210199040, 3211001855, ++SNULL, 3999440896, 3999444991, ++STORE, 3999440896, 3999444991, ++SNULL, 3999440896, 3999444991, ++STORE, 3594821632, 3594952703, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 1914101759, 1969434623, ++STORE, 1914097664, 1914101759, ++STORE, 1914101760, 1969434623, ++STORE, 3567108096, 3567239167, ++STORE, 3973832704, 3973840895, ++STORE, 3209113600, 3210199039, ++SNULL, 3209113600, 3209117695, ++STORE, 3209117696, 3210199039, ++STORE, 3209113600, 3209117695, ++SNULL, 3210194943, 3210199039, ++STORE, 3209117696, 3210194943, ++STORE, 3210194944, 3210199039, ++STORE, 3935858688, 3935879167, ++SNULL, 3935862783, 3935879167, ++STORE, 3935858688, 3935862783, ++STORE, 3935862784, 3935879167, ++SNULL, 3209121791, 3210194943, ++STORE, 3209117696, 3209121791, ++STORE, 3209121792, 3210194943, ++STORE, 3528749056, 3528880127, ++STORE, 3968200704, 3968208895, ++STORE, 3208028160, 3209117695, ++SNULL, 3208028160, 3208032255, ++STORE, 3208032256, 3209117695, ++STORE, 3208028160, 3208032255, ++SNULL, 3209109503, 3209117695, ++STORE, 3208032256, 3209109503, ++STORE, 3209109504, 3209117695, ++STORE, 3888123904, 3888144383, ++SNULL, 3888127999, 3888144383, ++STORE, 3888123904, 3888127999, ++STORE, 3888128000, 3888144383, ++SNULL, 3208036351, 3209109503, ++STORE, 3208032256, 3208036351, ++STORE, 3208036352, 3209109503, ++SNULL, 3968200704, 3968208895, ++SNULL, 3888123904, 3888144383, ++SNULL, 3209109504, 3209113599, ++STORE, 3209113600, 3209117695, ++STORE, 3209109504, 3209113599, ++SNULL, 3208028160, 3209113599, ++STORE, 3208060928, 3209117695, ++SNULL, 3208060928, 3208065023, ++STORE, 3208065024, 3209117695, ++STORE, 3208060928, 3208065023, ++SNULL, 3209109503, 3209117695, ++STORE, 3208065024, 3209109503, ++STORE, 3209109504, 3209117695, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3888123904, 3888144383, ++SNULL, 3888127999, 3888144383, ++STORE, 3888123904, 3888127999, ++STORE, 3888128000, 3888144383, ++SNULL, 3208069119, 3209109503, ++STORE, 3208065024, 3208069119, ++STORE, 3208069120, 3209109503, ++STORE, 3968200704, 3968208895, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3527778304, 3527909375, ++STORE, 3999440896, 3999444991, ++STORE, 3997679616, 3997683711, ++STORE, 1914097664, 1914105855, ++STORE, 1914105856, 1969434623, ++STORE, 3957583872, 3957592063, ++STORE, 3206975488, 3208065023, ++SNULL, 3206975488, 3206979583, ++STORE, 3206979584, 3208065023, ++STORE, 3206975488, 3206979583, ++SNULL, 3208056831, 3208065023, ++STORE, 3206979584, 3208056831, ++STORE, 3208056832, 3208065023, ++STORE, 3956736000, 3956744191, ++STORE, 3205890048, 3206979583, ++SNULL, 3205890048, 3205894143, ++STORE, 3205894144, 3206979583, ++STORE, 3205890048, 3205894143, ++SNULL, 3206971391, 3206979583, ++STORE, 3205894144, 3206971391, ++STORE, 3206971392, 3206979583, ++STORE, 3806101504, 3806121983, ++SNULL, 3806105599, 3806121983, ++STORE, 3806101504, 3806105599, ++STORE, 3806105600, 3806121983, ++SNULL, 3206983679, 3208056831, ++STORE, 3206979584, 3206983679, ++STORE, 3206983680, 3208056831, ++STORE, 3806081024, 3806101503, ++SNULL, 3806085119, 3806101503, ++STORE, 3806081024, 3806085119, ++STORE, 3806085120, 3806101503, ++SNULL, 3205898239, 3206971391, ++STORE, 3205894144, 3205898239, ++STORE, 3205898240, 3206971391, ++STORE, 3956015104, 3956023295, ++STORE, 3204804608, 3205894143, ++SNULL, 3204804608, 3204808703, ++STORE, 3204808704, 3205894143, ++STORE, 3204804608, 3204808703, ++SNULL, 3205885951, 3205894143, ++STORE, 3204808704, 3205885951, ++STORE, 3205885952, 3205894143, ++STORE, 3803471872, 3803492351, ++STORE, 3803451392, 3803471871, ++STORE, 3803451392, 3803492351, ++SNULL, 3957583872, 3957592063, ++SNULL, 3806101504, 3806121983, ++SNULL, 3206975487, 3206979583, ++STORE, 3206971392, 3206975487, ++STORE, 3206975488, 3206979583, ++SNULL, 3208056832, 3208060927, ++STORE, 3208060928, 3208065023, ++STORE, 3208056832, 3208060927, ++SNULL, 3206975488, 3208060927, ++STORE, 3801845760, 3801878527, ++STORE, 3806101504, 3806121983, ++SNULL, 3806105599, 3806121983, ++STORE, 3806101504, 3806105599, ++STORE, 3806105600, 3806121983, ++SNULL, 3204812799, 3205885951, ++STORE, 3204808704, 3204812799, ++STORE, 3204812800, 3205885951, ++STORE, 1914097664, 1914109951, ++STORE, 1914109952, 1969434623, ++STORE, 3957583872, 3957592063, ++STORE, 3206971392, 3208065023, ++SNULL, 3206971392, 3206979583, ++STORE, 3206979584, 3208065023, ++STORE, 3206971392, 3206979583, ++SNULL, 3208056831, 3208065023, ++STORE, 3206979584, 3208056831, ++STORE, 3208056832, 3208065023, ++STORE, 3801825280, 3801845759, ++SNULL, 3801829375, 3801845759, ++STORE, 3801825280, 3801829375, ++STORE, 3801829376, 3801845759, ++SNULL, 3206983679, 3208056831, ++STORE, 3206979584, 3206983679, ++STORE, 3206983680, 3208056831, ++STORE, 3202707456, 3204804607, ++SNULL, 3202707456, 3204804607, ++STORE, 3202707456, 3204804607, ++STORE, 3200610304, 3202707455, ++SNULL, 3202707456, 3204804607, ++SNULL, 3200610304, 3202707455, ++STORE, 3202707456, 3204804607, ++SNULL, 3202707456, 3204804607, ++STORE, 3202707456, 3204804607, ++SNULL, 3202707456, 3204804607, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3527647232, 3527778303, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++STORE, 3487059968, 3487584255, ++SNULL, 3487059968, 3487301631, ++STORE, 3487301632, 3487584255, ++STORE, 3487059968, 3487301631, ++SNULL, 3487059968, 3487301631, ++SNULL, 3487563775, 3487584255, ++STORE, 3487301632, 3487563775, ++STORE, 3487563776, 3487584255, ++SNULL, 3487563776, 3487584255, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3524046848, 3524177919, ++STORE, 3487170560, 3487301631, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3487039488, 3487170559, ++STORE, 3487039488, 3487301631, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3204280320, 3204804607, ++SNULL, 3204280320, 3204448255, ++STORE, 3204448256, 3204804607, ++STORE, 3204280320, 3204448255, ++SNULL, 3204280320, 3204448255, ++SNULL, 3204710399, 3204804607, ++STORE, 3204448256, 3204710399, ++STORE, 3204710400, 3204804607, ++SNULL, 3204710400, 3204804607, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3996295168, 3996299263, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++SNULL, 3996295168, 3996299263, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3486908416, 3487039487, ++STORE, 3486908416, 3487301631, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++STORE, 3201826816, 3202351103, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++SNULL, 3803471871, 3803492351, ++STORE, 3803451392, 3803471871, ++STORE, 3803471872, 3803492351, ++SNULL, 3803471872, 3803492351, ++SNULL, 3803451392, 3803471871, ++STORE, 3798999040, 3799101439, ++SNULL, 3798999040, 3799101439, ++STORE, 3952644096, 3952652287, ++STORE, 3203362816, 3204448255, ++SNULL, 3203362816, 3203366911, ++STORE, 3203366912, 3204448255, ++STORE, 3203362816, 3203366911, ++SNULL, 3204444159, 3204448255, ++STORE, 3203366912, 3204444159, ++STORE, 3204444160, 3204448255, ++STORE, 3803471872, 3803492351, ++SNULL, 3803475967, 3803492351, ++STORE, 3803471872, 3803475967, ++STORE, 3803475968, 3803492351, ++SNULL, 3203371007, 3204444159, ++STORE, 3203366912, 3203371007, ++STORE, 3203371008, 3204444159, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3200774144, 3201826815, ++SNULL, 3200774144, 3200778239, ++STORE, 3200778240, 3201826815, ++STORE, 3200774144, 3200778239, ++SNULL, 3201822719, 3201826815, ++STORE, 3200778240, 3201822719, ++STORE, 3201822720, 3201826815, ++STORE, 3803451392, 3803471871, ++SNULL, 3803455487, 3803471871, ++STORE, 3803451392, 3803455487, ++STORE, 3803455488, 3803471871, ++SNULL, 3200782335, 3201822719, ++STORE, 3200778240, 3200782335, ++STORE, 3200782336, 3201822719, ++STORE, 3949666304, 3949674495, ++STORE, 3949408256, 3949416447, ++STORE, 3199688704, 3200778239, ++SNULL, 3199688704, 3199692799, ++STORE, 3199692800, 3200778239, ++STORE, 3199688704, 3199692799, ++SNULL, 3200770047, 3200778239, ++STORE, 3199692800, 3200770047, ++STORE, 3200770048, 3200778239, ++STORE, 3799306240, 3799326719, ++SNULL, 3799310335, 3799326719, ++STORE, 3799306240, 3799310335, ++STORE, 3799310336, 3799326719, ++SNULL, 3199696895, 3200770047, ++STORE, 3199692800, 3199696895, ++STORE, 3199696896, 3200770047, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++STORE, 3799277568, 3799306239, ++SNULL, 3799277568, 3799306239, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++SNULL, 4041162751, 4041170943, ++STORE, 4041154560, 4041162751, ++STORE, 4041162752, 4041170943, ++SNULL, 4041162752, 4041170943, ++SNULL, 4041154560, 4041162751, ++SNULL, 4041191424, 4041211903, ++SNULL, 4041170944, 4041191423, ++SNULL, 3626471423, 3626475519, ++STORE, 3626467328, 3626471423, ++STORE, 3626471424, 3626475519, ++SNULL, 3626471424, 3627524095, ++SNULL, 3625418751, 3625422847, ++STORE, 3625414656, 3625418751, ++STORE, 3625418752, 3625422847, ++SNULL, 3625418752, 3626471423, ++STORE, 3627393024, 3627524095, ++STORE, 3627261952, 3627393023, ++STORE, 3627261952, 3627524095, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++STORE, 3195494400, 3197591551, ++SNULL, 3197591552, 3199688703, ++SNULL, 3195494400, 3197591551, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++STORE, 3195494400, 3197591551, ++SNULL, 3197591552, 3199688703, ++SNULL, 3195494400, 3197591551, ++STORE, 3798999040, 3799101439, ++SNULL, 3798999040, 3799101439, ++/* ++ * mmap: unmapped_area_topdown: ffff9a9f14ddaa80 ++ * Gap was found: mt 4041162752 gap_end 4041183232 ++ * mmap: window was 4052029440 - 4096 size 28672 ++ * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423 ++ * mmap: mas.index 4041162752 align mask 0 offset 0 ++ * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678) ++ */ ++ }; ++ ++ unsigned long set43[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734187720704, 140737488351231, ++SNULL, 140734187724800, 140737488351231, ++STORE, 140734187589632, 140734187724799, ++STORE, 4194304, 6443007, ++STORE, 4337664, 6443007, ++STORE, 4194304, 4337663, ++SNULL, 4337664, 6443007, ++STORE, 6430720, 6443007, ++STORE, 206158430208, 206160674815, ++STORE, 206158569472, 206160674815, ++STORE, 206158430208, 206158569471, ++SNULL, 206158569472, 206160674815, ++STORE, 206160662528, 206160670719, ++STORE, 206160670720, 206160674815, ++STORE, 140734188756992, 140734188765183, ++STORE, 140734188740608, 140734188756991, ++STORE, 140501948112896, 140501948116991, ++ }; ++ ++ int count = 0; ++ void *ptr = NULL; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ mt_set_non_kernel(3); ++ check_erase2_testset(mt, set, ARRAY_SIZE(set)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set2, ARRAY_SIZE(set2)); ++ start = 140735933894656; ++ MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL)); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(2); ++ mt_init_flags(mt, 0); ++ check_erase2_testset(mt, set3, ARRAY_SIZE(set3)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, 0); ++ check_erase2_testset(mt, set4, ARRAY_SIZE(set4)); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (xa_is_zero(entry)) ++ continue; ++ } ++ rcu_read_unlock(); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mt_set_non_kernel(100); ++ check_erase2_testset(mt, set5, ARRAY_SIZE(set5)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set6, ARRAY_SIZE(set6)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set7, ARRAY_SIZE(set7)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set8, ARRAY_SIZE(set8)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set9, ARRAY_SIZE(set9)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set10, ARRAY_SIZE(set10)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set11, ARRAY_SIZE(set11)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); ++ MT_BUG_ON(mt, mas.last != 140014592573439); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mas.tree = mt; ++ count = 0; ++ mas.index = 0; ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set12, ARRAY_SIZE(set12)); ++ rcu_barrier(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (xa_is_zero(entry)) ++ continue; ++ BUG_ON(count > 12); ++ count++; ++ } ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set13, ARRAY_SIZE(set13)); ++ mtree_erase(mt, 140373516443648); ++ rcu_read_lock(); ++ mas_empty_area_rev(&mas, 0, 140373518663680, 4096); ++ rcu_read_unlock(); ++ mtree_destroy(mt); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set14, ARRAY_SIZE(set14)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set15, ARRAY_SIZE(set15)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ /* set16 was to find a bug on limit updating at slot 0. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set16, ARRAY_SIZE(set16)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); ++ MT_BUG_ON(mt, mas.last != 139921865547775); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ /* ++ * set17 found a bug in walking backwards and not counting nulls at ++ * the end. This could cause a gap to be missed if the null had any ++ * size. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set17, ARRAY_SIZE(set17)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); ++ MT_BUG_ON(mt, mas.last != 139953197322239); ++/* MT_BUG_ON(mt, mas.index != 139953197318144); */ ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ /* ++ * set18 found a bug in walking backwards and not setting the max from ++ * the node, but using the parent node. This was only an issue if the ++ * next slot in the parent had what we needed. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set18, ARRAY_SIZE(set18)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); ++ MT_BUG_ON(mt, mas.last != 140222968475647); ++ /*MT_BUG_ON(mt, mas.index != 140222966259712); */ ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ /* ++ * set19 found 2 bugs in prev. ++ * 1. If we hit root without finding anything, then there was an ++ * infinite loop. ++ * 2. The first ascending wasn't using the correct slot which may have ++ * caused missed entries. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set19, ARRAY_SIZE(set19)); ++ rcu_barrier(); ++ mas.index = 140656779083776; ++ entry = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(140656779083776)); ++ entry = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, entry != xa_mk_value(140656766251008)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ /* ++ * set20 found a bug in mas_may_move_gap due to the slot being ++ * overwritten during the __mas_add operation and setting it to zero. ++ */ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set20, ARRAY_SIZE(set20)); ++ rcu_barrier(); ++ check_load(mt, 94849009414144, NULL); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set21, ARRAY_SIZE(set21)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(999); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set22, ARRAY_SIZE(set22)); ++ rcu_barrier(); ++ mt_validate(mt); ++ ptr = mtree_load(mt, 140551363362816); ++ MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set23, ARRAY_SIZE(set23)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set24, ARRAY_SIZE(set24)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set25, ARRAY_SIZE(set25)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Split on NULL followed by delete - causes gap issues. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set26, ARRAY_SIZE(set26)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); ++ MT_BUG_ON(mt, mas.last != 140109040959487); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Split on NULL followed by delete - causes gap issues. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set27, ARRAY_SIZE(set27)); ++ rcu_barrier(); ++ MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336)); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set28, ARRAY_SIZE(set28)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); ++ /* Search for the size of gap then align it (offset 0) */ ++ mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); ++ MT_BUG_ON(mt, mas.index != 139918401601536); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* This test found issues with retry moving rebalanced nodes so the ++ * incorrect parent pivot was updated. ++ */ ++ mt_set_non_kernel(999); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set29, ARRAY_SIZE(set29)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* This test found issues with deleting all entries in a node when ++ * surrounded by entries in the next nodes, then deleting the entries ++ * surrounding the node filled with deleted entries. ++ */ ++ mt_set_non_kernel(999); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set30, ARRAY_SIZE(set30)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* This test found an issue with deleting all entries in a node that was ++ * the end node and mas_gap incorrectly set next = curr, and curr = prev ++ * then moved next to the left, losing data. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set31, ARRAY_SIZE(set31)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set32, ARRAY_SIZE(set32)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++/* ++ * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found: ++ * mt 140582827569152 gap_end 140582869532672 ++ * mmap: window was 140583656296448 - 4096 size 134217728 ++ * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879 ++ * mmap: mas.index 140582827569152 align mask 0 offset 0 ++ * mmap: rb_find_vma find on ++ * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00) ++ */ ++ ++ /* move gap failed due to an entirely empty node */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set33, ARRAY_SIZE(set33)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); ++ MT_BUG_ON(mt, mas.last != 140583003750399); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* ++ * Incorrect gap in tree caused by mas_prev not setting the limits ++ * correctly while walking down. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set34, ARRAY_SIZE(set34)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Empty leaf at the end of a parent caused incorrect gap. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set35, ARRAY_SIZE(set35)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ /* Empty leaf at the end of a parent caused incorrect gap. */ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set36, ARRAY_SIZE(set36)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set37, ARRAY_SIZE(set37)); ++ rcu_barrier(); ++ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set38, ARRAY_SIZE(set38)); ++ rcu_barrier(); ++ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set41, ARRAY_SIZE(set41)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* move gap failed due to an entirely empty node. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set42, ARRAY_SIZE(set42)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 4052029440, 28672); ++ MT_BUG_ON(mt, mas.last != 4041211903); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* gap calc off by one */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set43, ARRAY_SIZE(set43)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++} ++ ++static noinline void check_alloc_rev_range(struct maple_tree *mt) ++{ ++ /* ++ * Generated by: ++ * cat /proc/self/maps | awk '{print $1}'| ++ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' ++ */ ++ ++ unsigned long range[] = { ++ /* Inclusive , Exclusive. */ ++ 0x565234af2000, 0x565234af4000, ++ 0x565234af4000, 0x565234af9000, ++ 0x565234af9000, 0x565234afb000, ++ 0x565234afc000, 0x565234afd000, ++ 0x565234afd000, 0x565234afe000, ++ 0x565235def000, 0x565235e10000, ++ 0x7f36d4bfd000, 0x7f36d4ee2000, ++ 0x7f36d4ee2000, 0x7f36d4f04000, ++ 0x7f36d4f04000, 0x7f36d504c000, ++ 0x7f36d504c000, 0x7f36d5098000, ++ 0x7f36d5098000, 0x7f36d5099000, ++ 0x7f36d5099000, 0x7f36d509d000, ++ 0x7f36d509d000, 0x7f36d509f000, ++ 0x7f36d509f000, 0x7f36d50a5000, ++ 0x7f36d50b9000, 0x7f36d50db000, ++ 0x7f36d50db000, 0x7f36d50dc000, ++ 0x7f36d50dc000, 0x7f36d50fa000, ++ 0x7f36d50fa000, 0x7f36d5102000, ++ 0x7f36d5102000, 0x7f36d5103000, ++ 0x7f36d5103000, 0x7f36d5104000, ++ 0x7f36d5104000, 0x7f36d5105000, ++ 0x7fff5876b000, 0x7fff5878d000, ++ 0x7fff5878e000, 0x7fff58791000, ++ 0x7fff58791000, 0x7fff58793000, ++ }; ++ ++ unsigned long holes[] = { ++ /* ++ * Note: start of hole is INCLUSIVE ++ * end of hole is EXCLUSIVE ++ * (opposite of the above table.) ++ * Start of hole, end of hole, size of hole (+1) ++ */ ++ 0x565234afb000, 0x565234afc000, 0x1000, ++ 0x565234afe000, 0x565235def000, 0x12F1000, ++ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, ++ }; ++ ++ /* ++ * req_range consists of 4 values. ++ * 1. min index ++ * 2. max index ++ * 3. size ++ * 4. number that should be returned. ++ * 5. return value ++ */ ++ unsigned long req_range[] = { ++ 0x565234af9000, /* Min */ ++ 0x7fff58791000, /* Max */ ++ 0x1000, /* Size */ ++ 0x7fff5878d << 12, /* First rev hole of size 0x1000 */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ 0x565234AF1 << 12, /* Max */ ++ 0x3000, /* Size */ ++ 0x565234AEE << 12, /* max - 3. */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ -1, /* Max */ ++ 0x1000, /* Size */ ++ 562949953421311 << 12,/* First rev hole of size 0x1000 */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ 0x7F36D510A << 12, /* Max */ ++ 0x4000, /* Size */ ++ 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */ ++ 0, /* Return value success. */ ++ ++ /* Ascend test. */ ++ 0x0, ++ 34148798629 << 12, ++ 19 << 12, ++ 34148797418 << 12, ++ 0x0, ++ ++ /* Too big test. */ ++ 0x0, ++ 18446744073709551615UL, ++ 562915594369134UL << 12, ++ 0x0, ++ -EBUSY, ++ ++ }; ++ ++ int i, range_count = ARRAY_SIZE(range); ++ int req_range_count = ARRAY_SIZE(req_range); ++ unsigned long min = 0; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, ++ GFP_KERNEL); ++#define DEBUG_REV_RANGE 0 ++ for (i = 0; i < range_count; i += 2) { ++ /* Inclusive, Inclusive (with the -1) */ ++ ++#if DEBUG_REV_RANGE ++ pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12, ++ (range[i + 1] >> 12) - 1); ++#endif ++ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, ++ xa_mk_value(range[i] >> 12), 0); ++ mt_validate(mt); ++ } ++ ++ ++ for (i = 0; i < ARRAY_SIZE(holes); i += 3) { ++#if DEBUG_REV_RANGE ++ pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n", ++ min, holes[i+1]>>12, holes[i+2]>>12, ++ holes[i] >> 12); ++#endif ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, min, ++ holes[i+1] >> 12, ++ holes[i+2] >> 12)); ++#if DEBUG_REV_RANGE ++ pr_debug("Found %lu %lu\n", mas.index, mas.last); ++ pr_debug("gap %lu %lu\n", (holes[i] >> 12), ++ (holes[i+1] >> 12)); ++#endif ++ MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12)); ++ MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12)); ++ min = holes[i+1] >> 12; ++ mas_reset(&mas); ++ } ++ ++ for (i = 0; i < req_range_count; i += 5) { ++#if DEBUG_REV_RANGE ++ pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n", ++ req_range[i] >> 12, ++ (req_range[i + 1] >> 12) - 1, ++ req_range[i+2] >> 12, ++ req_range[i+3] >> 12); ++#endif ++ check_mtree_alloc_rrange(mt, ++ req_range[i] >> 12, /* start */ ++ req_range[i+1] >> 12, /* end */ ++ req_range[i+2] >> 12, /* size */ ++ req_range[i+3] >> 12, /* expected address */ ++ req_range[i+4], /* expected return */ ++ xa_mk_value(req_range[i] >> 12)); /* pointer */ ++ mt_validate(mt); ++ } ++ ++ mt_set_non_kernel(1); ++ mtree_erase(mt, 34148798727); /* create a deleted range. */ ++ check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414, ++ 34148798725, 0, mt); ++ ++ mtree_destroy(mt); ++} ++ ++static noinline void check_alloc_range(struct maple_tree *mt) ++{ ++ /* ++ * Generated by: ++ * cat /proc/self/maps|awk '{print $1}'| ++ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' ++ */ ++ ++ unsigned long range[] = { ++ /* Inclusive , Exclusive. */ ++ 0x565234af2000, 0x565234af4000, ++ 0x565234af4000, 0x565234af9000, ++ 0x565234af9000, 0x565234afb000, ++ 0x565234afc000, 0x565234afd000, ++ 0x565234afd000, 0x565234afe000, ++ 0x565235def000, 0x565235e10000, ++ 0x7f36d4bfd000, 0x7f36d4ee2000, ++ 0x7f36d4ee2000, 0x7f36d4f04000, ++ 0x7f36d4f04000, 0x7f36d504c000, ++ 0x7f36d504c000, 0x7f36d5098000, ++ 0x7f36d5098000, 0x7f36d5099000, ++ 0x7f36d5099000, 0x7f36d509d000, ++ 0x7f36d509d000, 0x7f36d509f000, ++ 0x7f36d509f000, 0x7f36d50a5000, ++ 0x7f36d50b9000, 0x7f36d50db000, ++ 0x7f36d50db000, 0x7f36d50dc000, ++ 0x7f36d50dc000, 0x7f36d50fa000, ++ 0x7f36d50fa000, 0x7f36d5102000, ++ 0x7f36d5102000, 0x7f36d5103000, ++ 0x7f36d5103000, 0x7f36d5104000, ++ 0x7f36d5104000, 0x7f36d5105000, ++ 0x7fff5876b000, 0x7fff5878d000, ++ 0x7fff5878e000, 0x7fff58791000, ++ 0x7fff58791000, 0x7fff58793000, ++ }; ++ unsigned long holes[] = { ++ /* Start of hole, end of hole, size of hole (+1) */ ++ 0x565234afb000, 0x565234afc000, 0x1000, ++ 0x565234afe000, 0x565235def000, 0x12F1000, ++ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, ++ }; ++ ++ /* ++ * req_range consists of 4 values. ++ * 1. min index ++ * 2. max index ++ * 3. size ++ * 4. number that should be returned. ++ * 5. return value ++ */ ++ unsigned long req_range[] = { ++ 0x565234af9000, /* Min */ ++ 0x7fff58791000, /* Max */ ++ 0x1000, /* Size */ ++ 0x565234afb000, /* First hole in our data of size 1000. */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ 0x7fff58791000, /* Max */ ++ 0x1F00, /* Size */ ++ 0x0, /* First hole in our data of size 2000. */ ++ 0, /* Return value success. */ ++ ++ /* Test ascend. */ ++ 34148797436 << 12, /* Min */ ++ 0x7fff587AF000, /* Max */ ++ 0x3000, /* Size */ ++ 34148798629 << 12, /* Expected location */ ++ 0, /* Return value success. */ ++ ++ /* Test failing. */ ++ 34148798623 << 12, /* Min */ ++ 34148798683 << 12, /* Max */ ++ 0x15000, /* Size */ ++ 0, /* Expected location */ ++ -EBUSY, /* Return value failed. */ ++ ++ /* Test filling entire gap. */ ++ 34148798623 << 12, /* Min */ ++ 0x7fff587AF000, /* Max */ ++ 0x10000, /* Size */ ++ 34148798632 << 12, /* Expected location */ ++ 0, /* Return value success. */ ++ ++ /* Test walking off the end of root. */ ++ 0, /* Min */ ++ -1, /* Max */ ++ -1, /* Size */ ++ 0, /* Expected location */ ++ -EBUSY, /* Return value failure. */ ++ ++ /* Test looking for too large a hole across entire range. */ ++ 0, /* Min */ ++ -1, /* Max */ ++ 4503599618982063UL << 12, /* Size */ ++ 34359052178 << 12, /* Expected location */ ++ -EBUSY, /* Return failure. */ ++ }; ++ int i, range_count = ARRAY_SIZE(range); ++ int req_range_count = ARRAY_SIZE(req_range); ++ unsigned long min = 0x565234af2000; ++ ++ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, ++ GFP_KERNEL); ++ for (i = 0; i < range_count; i += 2) { ++#define DEBUG_ALLOC_RANGE 0 ++#if DEBUG_ALLOC_RANGE ++ pr_debug("\tInsert %lu-%lu\n", range[i] >> 12, ++ (range[i + 1] >> 12) - 1); ++ mt_dump(mt); ++#endif ++ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, ++ xa_mk_value(range[i] >> 12), 0); ++ mt_validate(mt); ++ } ++ ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ for (i = 0; i < ARRAY_SIZE(holes); i += 3) { ++ ++#if DEBUG_ALLOC_RANGE ++ pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12, ++ holes[i+1] >> 12, holes[i+2] >> 12, ++ min, holes[i+1]); ++#endif ++ MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12, ++ holes[i+1] >> 12, ++ holes[i+2] >> 12)); ++ MT_BUG_ON(mt, mas.index != holes[i] >> 12); ++ min = holes[i+1]; ++ mas_reset(&mas); ++ } ++ for (i = 0; i < req_range_count; i += 5) { ++#if DEBUG_ALLOC_RANGE ++ pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n", ++ i/5, req_range[i] >> 12, req_range[i + 1] >> 12, ++ req_range[i + 2] >> 12, req_range[i + 3] >> 12, ++ req_range[i], req_range[i+1]); ++#endif ++ check_mtree_alloc_range(mt, ++ req_range[i] >> 12, /* start */ ++ req_range[i+1] >> 12, /* end */ ++ req_range[i+2] >> 12, /* size */ ++ req_range[i+3] >> 12, /* expected address */ ++ req_range[i+4], /* expected return */ ++ xa_mk_value(req_range[i] >> 12)); /* pointer */ ++ mt_validate(mt); ++#if DEBUG_ALLOC_RANGE ++ mt_dump(mt); ++#endif ++ } ++ ++ mtree_destroy(mt); ++} ++ ++static noinline void check_ranges(struct maple_tree *mt) ++{ ++ int i, val, val2; ++ unsigned long r[] = { ++ 10, 15, ++ 20, 25, ++ 17, 22, /* Overlaps previous range. */ ++ 9, 1000, /* Huge. */ ++ 100, 200, ++ 45, 168, ++ 118, 128, ++ }; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0); ++ check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0); ++ check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ /* Store */ ++ check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0); ++ check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0); ++ check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ MT_BUG_ON(mt, mt_height(mt)); ++ ++ check_seq(mt, 50, false); ++ mt_set_non_kernel(4); ++ check_store_range(mt, 5, 47, xa_mk_value(47), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Create tree of 1-100 */ ++ check_seq(mt, 100, false); ++ /* Store 45-168 */ ++ mt_set_non_kernel(10); ++ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Create tree of 1-200 */ ++ check_seq(mt, 200, false); ++ /* Store 45-168 */ ++ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ check_seq(mt, 30, false); ++ check_store_range(mt, 6, 18, xa_mk_value(6), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Overwrite across multiple levels. */ ++ /* Create tree of 1-400 */ ++ check_seq(mt, 400, false); ++ mt_set_non_kernel(50); ++ /* Store 118-128 */ ++ check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0); ++ mt_set_non_kernel(50); ++ mtree_test_erase(mt, 140); ++ mtree_test_erase(mt, 141); ++ mtree_test_erase(mt, 142); ++ mtree_test_erase(mt, 143); ++ mtree_test_erase(mt, 130); ++ mtree_test_erase(mt, 131); ++ mtree_test_erase(mt, 132); ++ mtree_test_erase(mt, 133); ++ mtree_test_erase(mt, 134); ++ mtree_test_erase(mt, 135); ++ check_load(mt, r[12], xa_mk_value(r[12])); ++ check_load(mt, r[13], xa_mk_value(r[12])); ++ check_load(mt, r[13] - 1, xa_mk_value(r[12])); ++ check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); ++ check_load(mt, 135, NULL); ++ check_load(mt, 140, NULL); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ ++ ++ /* Overwrite multiple levels at the end of the tree (slot 7) */ ++ mt_set_non_kernel(50); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 353, 361, xa_mk_value(353), 0); ++ check_store_range(mt, 347, 352, xa_mk_value(347), 0); ++ ++ check_load(mt, 346, xa_mk_value(346)); ++ for (i = 347; i <= 352; i++) ++ check_load(mt, i, xa_mk_value(347)); ++ for (i = 353; i <= 361; i++) ++ check_load(mt, i, xa_mk_value(353)); ++ check_load(mt, 362, xa_mk_value(362)); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(50); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 352, 364, NULL, 0); ++ check_store_range(mt, 351, 363, xa_mk_value(352), 0); ++ check_load(mt, 350, xa_mk_value(350)); ++ check_load(mt, 351, xa_mk_value(352)); ++ for (i = 352; i <= 363; i++) ++ check_load(mt, i, xa_mk_value(352)); ++ check_load(mt, 364, NULL); ++ check_load(mt, 365, xa_mk_value(365)); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(5); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 352, 364, NULL, 0); ++ check_store_range(mt, 351, 364, xa_mk_value(352), 0); ++ check_load(mt, 350, xa_mk_value(350)); ++ check_load(mt, 351, xa_mk_value(352)); ++ for (i = 352; i <= 364; i++) ++ check_load(mt, i, xa_mk_value(352)); ++ check_load(mt, 365, xa_mk_value(365)); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ ++ mt_set_non_kernel(50); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 362, 367, xa_mk_value(362), 0); ++ check_store_range(mt, 353, 361, xa_mk_value(353), 0); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ /* ++ * Interesting cases: ++ * 1. Overwrite the end of a node and end in the first entry of the next ++ * node. ++ * 2. Split a single range ++ * 3. Overwrite the start of a range ++ * 4. Overwrite the end of a range ++ * 5. Overwrite the entire range ++ * 6. Overwrite a range that causes multiple parent nodes to be ++ * combined ++ * 7. Overwrite a range that causes multiple parent nodes and part of ++ * root to be combined ++ * 8. Overwrite the whole tree ++ * 9. Try to overwrite the zero entry of an alloc tree. ++ * 10. Write a range larger than a nodes current pivot ++ */ ++ ++ mt_set_non_kernel(50); ++ for (i = 0; i <= 500; i++) { ++ val = i*5; ++ val2 = (i+1)*5; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0); ++ check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0); ++ check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0); ++ check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0); ++ check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0); ++ mtree_destroy(mt); ++ mt_set_non_kernel(0); ++ ++ mt_set_non_kernel(50); ++ for (i = 0; i <= 500; i++) { ++ val = i*5; ++ val2 = (i+1)*5; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0); ++ check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0); ++ check_store_range(mt, 2425, 2425, xa_mk_value(2), 0); ++ check_store_range(mt, 2460, 2470, NULL, 0); ++ check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0); ++ check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Test rebalance gaps */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mt_set_non_kernel(50); ++ for (i = 0; i <= 50; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 161, 161, xa_mk_value(161), 0); ++ check_store_range(mt, 162, 162, xa_mk_value(162), 0); ++ check_store_range(mt, 163, 163, xa_mk_value(163), 0); ++ check_store_range(mt, 240, 249, NULL, 0); ++ mtree_erase(mt, 200); ++ mtree_erase(mt, 210); ++ mtree_erase(mt, 220); ++ mtree_erase(mt, 230); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 500; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 4600, 4959, xa_mk_value(1), 0); ++ mt_validate(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 500; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0); ++ check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0); ++ check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0); ++ check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0); ++ check_store_range(mt, 4842, 4849, NULL, 0); ++ mt_validate(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 1300; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ } ++ /* Cause a 3 child split all the way up the tree. */ ++ for (i = 5; i < 215; i += 10) ++ check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); ++ for (i = 5; i < 65; i += 10) ++ check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); ++ ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ for (i = 5; i < 45; i += 10) ++ check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); ++ MT_BUG_ON(mt, mt_height(mt) < 4); ++ mtree_destroy(mt); ++ ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 1200; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ } ++ /* Fill parents and leaves before split. */ ++ for (i = 5; i < 455; i += 10) ++ check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); ++ ++ for (i = 1; i < 16; i++) ++ check_store_range(mt, 8185 + i, 8185 + i + 1, ++ xa_mk_value(8185+i), 0); ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ /* triple split across multiple levels. */ ++ check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); ++ MT_BUG_ON(mt, mt_height(mt) != 4); ++} ++ ++static noinline void check_next_entry(struct maple_tree *mt) ++{ ++ void *entry = NULL; ++ unsigned long limit = 30, i = 0; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ MA_STATE(mas, mt, i, i); ++ ++ check_seq(mt, limit, false); ++ rcu_read_lock(); ++ ++ /* Check the first one and get ma_state in the correct state. */ ++ MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++)); ++ for ( ; i <= limit + 1; i++) { ++ entry = mas_next(&mas, limit); ++ if (i > limit) ++ MT_BUG_ON(mt, entry != NULL); ++ else ++ MT_BUG_ON(mt, xa_mk_value(i) != entry); ++ } ++ rcu_read_unlock(); ++ mtree_destroy(mt); ++} ++ ++static noinline void check_prev_entry(struct maple_tree *mt) ++{ ++ unsigned long index = 16; ++ void *value; ++ int i; ++ ++ MA_STATE(mas, mt, index, index); ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_seq(mt, 30, false); ++ ++ rcu_read_lock(); ++ value = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, value != xa_mk_value(index)); ++ value = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, value != xa_mk_value(index - 1)); ++ rcu_read_unlock(); ++ mtree_destroy(mt); ++ ++ /* Check limits on prev */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_lock(&mas); ++ for (i = 0; i <= index; i++) { ++ mas_set_range(&mas, i*10, i*10+5); ++ mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL); ++ } ++ ++ mas_set(&mas, 20); ++ value = mas_walk(&mas); ++ MT_BUG_ON(mt, value != xa_mk_value(2)); ++ ++ value = mas_prev(&mas, 19); ++ MT_BUG_ON(mt, value != NULL); ++ ++ mas_set(&mas, 80); ++ value = mas_walk(&mas); ++ MT_BUG_ON(mt, value != xa_mk_value(8)); ++ ++ value = mas_prev(&mas, 76); ++ MT_BUG_ON(mt, value != NULL); ++ ++ mas_unlock(&mas); ++} ++ ++static noinline void check_root_expand(struct maple_tree *mt) ++{ ++ MA_STATE(mas, mt, 0, 0); ++ void *ptr; ++ ++ ++ mas_lock(&mas); ++ mas_set(&mas, 3); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != ULONG_MAX); ++ ++ ptr = &check_prev_entry; ++ mas_set(&mas, 1); ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ++ mas_set(&mas, 0); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ ++ mas_set(&mas, 1); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != &check_prev_entry); ++ ++ mas_set(&mas, 2); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ mas_unlock(&mas); ++ mtree_destroy(mt); ++ ++ ++ mt_init_flags(mt, 0); ++ mas_lock(&mas); ++ ++ mas_set(&mas, 0); ++ ptr = &check_prev_entry; ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ++ mas_set(&mas, 5); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, mas.index != 1); ++ MT_BUG_ON(mt, mas.last != ULONG_MAX); ++ ++ mas_set_range(&mas, 0, 100); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != &check_prev_entry); ++ MT_BUG_ON(mt, mas.last != 0); ++ mas_unlock(&mas); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, 0); ++ mas_lock(&mas); ++ ++ mas_set(&mas, 0); ++ ptr = (void *)((unsigned long) check_prev_entry | 1UL); ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ptr = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); ++ ++ mas_set(&mas, 1); ++ ptr = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); ++ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL)); ++ ++ mas_unlock(&mas); ++ ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, 0); ++ mas_lock(&mas); ++ mas_set(&mas, 0); ++ ptr = (void *)((unsigned long) check_prev_entry | 2UL); ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ptr = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); ++ ++ mas_set(&mas, 1); ++ ptr = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); ++ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL)); ++ ++ ++ mas_unlock(&mas); ++} ++ ++static noinline void check_prealloc(struct maple_tree *mt) ++{ ++ unsigned long i, max = 100; ++ unsigned long allocated; ++ unsigned char height; ++ struct maple_node *mn; ++ void *ptr = check_prealloc; ++ MA_STATE(mas, mt, 10, 20); ++ ++ mt_set_non_kernel(1000); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); ++ ma_free_rcu(mn); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ma_free_rcu(mn); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mt_set_non_kernel(1); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ mas_destroy(&mas); ++ ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mt_set_non_kernel(1); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++} ++ ++static noinline void check_spanning_write(struct maple_tree *mt) ++{ ++ unsigned long i, max = 5000; ++ MA_STATE(mas, mt, 1200, 2380); ++ ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 1205); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ for (i = 1; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mtree_lock(mt); ++ mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 1205); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a right cousin rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 1205); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test non-alloc tree spanning store */ ++ mt_init_flags(mt, 0); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 0, 300); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 15); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a right sibling rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 0, 12865); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 15); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a left sibling rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 90, 13665); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 95); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a left cousin rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 46805, 49995); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 46815); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* ++ * Test spanning store that requires a left cousin rebalance all the way ++ * to root ++ */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 32395, 49995); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 46815); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* ++ * Test spanning store that requires a right cousin rebalance all the ++ * way to root ++ */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ mas_set_range(&mas, 38875, 43190); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 38900); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store ending at full node (depth 2)*/ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ mtree_lock(mt); ++ mas_set(&mas, 47606); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ mas_set(&mas, 47607); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ mas_set(&mas, 47608); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ mas_set(&mas, 47609); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ /* Ensure the parent node is full */ ++ mas_ascend(&mas); ++ MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); ++ mas_set_range(&mas, 11516, 48940); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning write with many levels of no siblings */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ mas_set_range(&mas, 43200, 49999); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 43200); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 100; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mtree_lock(mt); ++ mas_set_range(&mas, 76, 875); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mtree_unlock(mt); ++} ++ ++static noinline void check_null_expand(struct maple_tree *mt) ++{ ++ unsigned long i, max = 100; ++ unsigned char data_end; ++ MA_STATE(mas, mt, 959, 959); ++ ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ /* Test expanding null at start. */ ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 959, 963); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 963) != NULL); ++ MT_BUG_ON(mt, data_end != mas_data_end(&mas)); ++ ++ /* Test expanding null at end. */ ++ mas_set(&mas, 880); ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 884, 887); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 884) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 889) != NULL); ++ MT_BUG_ON(mt, data_end != mas_data_end(&mas)); ++ ++ /* Test expanding null at start and end. */ ++ mas_set(&mas, 890); ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 900, 905); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 899) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 900) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 905) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 906) != NULL); ++ MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); ++ ++ /* Test expanding null across multiple slots. */ ++ mas_set(&mas, 800); ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 810, 825); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 809) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 810) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 825) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 826) != NULL); ++ MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); ++} ++ ++static noinline void check_gap_combining(struct maple_tree *mt) ++{ ++ struct maple_enode *mn1, *mn2; ++ void *entry; ++ ++ unsigned long seq100[] = { ++ /* 0-5 */ ++ 74, 75, 76, ++ 50, 100, 2, ++ ++ /* 6-12 */ ++ 44, 45, 46, 43, ++ 20, 50, 3, ++ ++ /* 13-20*/ ++ 80, 81, 82, ++ 76, 2, 79, 85, 4, ++ }; ++ unsigned long seq2000[] = { ++ 1152, 1151, ++ 1100, 1200, 2, ++ }; ++ unsigned long seq400[] = { ++ 286, 318, ++ 256, 260, 266, 270, 275, 280, 290, 398, ++ 286, 310, ++ }; ++ ++ unsigned long index = seq100[0]; ++ ++ MA_STATE(mas, mt, index, index); ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_seq(mt, 100, false); /* create 100 singletons. */ ++ ++ mt_set_non_kernel(1); ++ mtree_test_erase(mt, seq100[2]); ++ check_load(mt, seq100[2], NULL); ++ mtree_test_erase(mt, seq100[1]); ++ check_load(mt, seq100[1], NULL); ++ ++ rcu_read_lock(); ++ entry = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index)); ++ mn1 = mas.node; ++ mas_next(&mas, ULONG_MAX); ++ entry = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); ++ mn2 = mas.node; ++ MT_BUG_ON(mt, mn1 == mn2); /* test the test. */ ++ ++ /* ++ * At this point, there is a gap of 2 at index + 1 between seq100[3] and ++ * seq100[4]. Search for the gap. ++ */ ++ mt_set_non_kernel(1); ++ mas_reset(&mas); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4], ++ seq100[5])); ++ MT_BUG_ON(mt, mas.index != index + 1); ++ rcu_read_unlock(); ++ ++ mtree_test_erase(mt, seq100[6]); ++ check_load(mt, seq100[6], NULL); ++ mtree_test_erase(mt, seq100[7]); ++ check_load(mt, seq100[7], NULL); ++ mtree_test_erase(mt, seq100[8]); ++ index = seq100[9]; ++ ++ rcu_read_lock(); ++ mas.index = index; ++ mas.last = index; ++ mas_reset(&mas); ++ entry = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index)); ++ mn1 = mas.node; ++ entry = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); ++ mas_next(&mas, ULONG_MAX); /* go to the next entry. */ ++ mn2 = mas.node; ++ MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */ ++ ++ /* ++ * At this point, there is a gap of 3 at seq100[6]. Find it by ++ * searching 20 - 50 for size 3. ++ */ ++ mas_reset(&mas); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11], ++ seq100[12])); ++ MT_BUG_ON(mt, mas.index != seq100[6]); ++ rcu_read_unlock(); ++ ++ mt_set_non_kernel(1); ++ mtree_store(mt, seq100[13], NULL, GFP_KERNEL); ++ check_load(mt, seq100[13], NULL); ++ check_load(mt, seq100[14], xa_mk_value(seq100[14])); ++ mtree_store(mt, seq100[14], NULL, GFP_KERNEL); ++ check_load(mt, seq100[13], NULL); ++ check_load(mt, seq100[14], NULL); ++ ++ mas_reset(&mas); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15], ++ seq100[17])); ++ MT_BUG_ON(mt, mas.index != seq100[13]); ++ mt_validate(mt); ++ rcu_read_unlock(); ++ ++ /* ++ * *DEPRECATED: no retries anymore* Test retry entry in the start of a ++ * gap. ++ */ ++ mt_set_non_kernel(2); ++ mtree_test_store_range(mt, seq100[18], seq100[14], NULL); ++ mtree_test_erase(mt, seq100[15]); ++ mas_reset(&mas); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19], ++ seq100[20])); ++ rcu_read_unlock(); ++ MT_BUG_ON(mt, mas.index != seq100[18]); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* seq 2000 tests are for multi-level tree gaps */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_seq(mt, 2000, false); ++ mt_set_non_kernel(1); ++ mtree_test_erase(mt, seq2000[0]); ++ mtree_test_erase(mt, seq2000[1]); ++ ++ mt_set_non_kernel(2); ++ mas_reset(&mas); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3], ++ seq2000[4])); ++ MT_BUG_ON(mt, mas.index != seq2000[1]); ++ rcu_read_unlock(); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* seq 400 tests rebalancing over two levels. */ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_seq(mt, 400, false); ++ mtree_test_store_range(mt, seq400[0], seq400[1], NULL); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_seq(mt, 400, false); ++ mt_set_non_kernel(50); ++ mtree_test_store_range(mt, seq400[2], seq400[9], ++ xa_mk_value(seq400[2])); ++ mtree_test_store_range(mt, seq400[3], seq400[9], ++ xa_mk_value(seq400[3])); ++ mtree_test_store_range(mt, seq400[4], seq400[9], ++ xa_mk_value(seq400[4])); ++ mtree_test_store_range(mt, seq400[5], seq400[9], ++ xa_mk_value(seq400[5])); ++ mtree_test_store_range(mt, seq400[0], seq400[9], ++ xa_mk_value(seq400[0])); ++ mtree_test_store_range(mt, seq400[6], seq400[9], ++ xa_mk_value(seq400[6])); ++ mtree_test_store_range(mt, seq400[7], seq400[9], ++ xa_mk_value(seq400[7])); ++ mtree_test_store_range(mt, seq400[8], seq400[9], ++ xa_mk_value(seq400[8])); ++ mtree_test_store_range(mt, seq400[10], seq400[11], ++ xa_mk_value(seq400[10])); ++ mt_validate(mt); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++} ++static noinline void check_node_overwrite(struct maple_tree *mt) ++{ ++ int i, max = 4000; ++ ++ for (i = 0; i < max; i++) ++ mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100)); ++ ++ mtree_test_store_range(mt, 319951, 367950, NULL); ++ /*mt_dump(mt); */ ++ mt_validate(mt); ++} ++ ++static void mas_dfs_preorder(struct ma_state *mas) ++{ ++ ++ struct maple_enode *prev; ++ unsigned char end, slot = 0; ++ ++ if (mas_is_start(mas)) { ++ mas_start(mas); ++ return; ++ } ++ ++ if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) ++ goto done; ++ ++walk_up: ++ end = mas_data_end(mas); ++ if (mte_is_leaf(mas->node) || ++ (slot > end)) { ++ if (mte_is_root(mas->node)) ++ goto done; ++ ++ slot = mte_parent_slot(mas->node) + 1; ++ mas_ascend(mas); ++ goto walk_up; ++ } ++ ++ prev = mas->node; ++ mas->node = mas_get_slot(mas, slot); ++ if (!mas->node || slot > end) { ++ if (mte_is_root(prev)) ++ goto done; ++ ++ mas->node = prev; ++ slot = mte_parent_slot(mas->node) + 1; ++ mas_ascend(mas); ++ goto walk_up; ++ } ++ ++ return; ++done: ++ mas->node = MAS_NONE; ++} ++ ++ ++static void check_dfs_preorder(struct maple_tree *mt) ++{ ++ unsigned long count = 0, max = 1000; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ check_seq(mt, max, false); ++ do { ++ count++; ++ mas_dfs_preorder(&mas); ++ } while (!mas_is_none(&mas)); ++ MT_BUG_ON(mt, count != 74); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_reset(&mas); ++ count = 0; ++ check_seq(mt, max, false); ++ do { ++ count++; ++ mas_dfs_preorder(&mas); ++ } while (!mas_is_none(&mas)); ++ /*printk("count %lu\n", count); */ ++ MT_BUG_ON(mt, count != 77); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_reset(&mas); ++ count = 0; ++ check_rev_seq(mt, max, false); ++ do { ++ count++; ++ mas_dfs_preorder(&mas); ++ } while (!mas_is_none(&mas)); ++ /*printk("count %lu\n", count); */ ++ MT_BUG_ON(mt, count != 77); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_reset(&mas); ++ mt_zero_nr_tallocated(); ++ mt_set_non_kernel(200); ++ mas_expected_entries(&mas, max); ++ for (count = 0; count <= max; count++) { ++ mas.index = mas.last = count; ++ mas_store(&mas, xa_mk_value(count)); ++ MT_BUG_ON(mt, mas_is_err(&mas)); ++ } ++ mas_destroy(&mas); ++ rcu_barrier(); ++ /* ++ * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n", ++ * max, mt_get_alloc_size()/1024, mt_nr_allocated(), ++ * mt_nr_tallocated()); ++ */ ++ ++} ++ ++#if defined(BENCH_SLOT_STORE) ++static noinline void bench_slot_store(struct maple_tree *mt) ++{ ++ int i, brk = 105, max = 1040, brk_start = 100, count = 20000000; ++ ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < count; i++) { ++ mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL); ++ mtree_store_range(mt, brk_start, brk, xa_mk_value(brk), ++ GFP_KERNEL); ++ } ++} ++#endif ++ ++#if defined(BENCH_NODE_STORE) ++static noinline void bench_node_store(struct maple_tree *mt) ++{ ++ int i, overwrite = 76, max = 240, count = 20000000; ++ ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < count; i++) { ++ mtree_store_range(mt, overwrite, overwrite + 15, ++ xa_mk_value(overwrite), GFP_KERNEL); ++ ++ overwrite += 5; ++ if (overwrite >= 135) ++ overwrite = 76; ++ } ++} ++#endif ++ ++#if defined(BENCH_AWALK) ++static noinline void bench_awalk(struct maple_tree *mt) ++{ ++ int i, max = 2500, count = 50000000; ++ MA_STATE(mas, mt, 1470, 1470); ++ ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); ++ ++ mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL); ++ ++ for (i = 0; i < count; i++) { ++ mas_empty_area_rev(&mas, 0, 2000, 10); ++ mas_reset(&mas); ++ } ++} ++#endif ++#if defined(BENCH_WALK) ++static noinline void bench_walk(struct maple_tree *mt) ++{ ++ int i, max = 2500, count = 550000000; ++ MA_STATE(mas, mt, 1470, 1470); ++ ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < count; i++) { ++ mas_walk(&mas); ++ mas_reset(&mas); ++ } ++ ++} ++#endif ++ ++#if defined(BENCH_MT_FOR_EACH) ++static noinline void bench_mt_for_each(struct maple_tree *mt) ++{ ++ int i, count = 1000000; ++ unsigned long max = 2500, index = 0; ++ void *entry; ++ ++ for (i = 0; i < max; i += 5) ++ mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < count; i++) { ++ unsigned long j = 0; ++ ++ mt_for_each(mt, entry, index, max) { ++ MT_BUG_ON(mt, entry != xa_mk_value(j)); ++ j += 5; ++ } ++ ++ index = 0; ++ } ++ ++} ++#endif ++ ++static noinline void check_forking(struct maple_tree *mt) ++{ ++ ++ struct maple_tree newmt; ++ int i, nr_entries = 134; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mt_set_non_kernel(99999); ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ newmas.tree = &newmt; ++ mas_reset(&newmas); ++ mas_reset(&mas); ++ mas.index = 0; ++ mas.last = 0; ++ if (mas_expected_entries(&newmas, nr_entries)) { ++ pr_err("OOM!"); ++ BUG_ON(1); ++ } ++ mas_for_each(&mas, val, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store(&newmas, val); ++ } ++ mas_destroy(&newmas); ++ mt_validate(&newmt); ++ mt_set_non_kernel(0); ++ mtree_destroy(&newmt); ++} ++ ++static noinline void check_mas_store_gfp(struct maple_tree *mt) ++{ ++ ++ struct maple_tree newmt; ++ int i, nr_entries = 135; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mt_set_non_kernel(99999); ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ newmas.tree = &newmt; ++ mas_reset(&newmas); ++ mas_set(&mas, 0); ++ mas_for_each(&mas, val, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store_gfp(&newmas, val, GFP_KERNEL); ++ } ++ ++ mt_validate(&newmt); ++ mt_set_non_kernel(0); ++ mtree_destroy(&newmt); ++} ++ ++#if defined(BENCH_FORK) ++static noinline void bench_forking(struct maple_tree *mt) ++{ ++ ++ struct maple_tree newmt; ++ int i, nr_entries = 134, nr_fork = 80000; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < nr_fork; i++) { ++ mt_set_non_kernel(99999); ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ newmas.tree = &newmt; ++ mas_reset(&newmas); ++ mas_reset(&mas); ++ mas.index = 0; ++ mas.last = 0; ++ if (mas_expected_entries(&newmas, nr_entries)) { ++ printk("OOM!"); ++ BUG_ON(1); ++ } ++ mas_for_each(&mas, val, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store(&newmas, val); ++ } ++ mas_destroy(&newmas); ++ mt_validate(&newmt); ++ mt_set_non_kernel(0); ++ mtree_destroy(&newmt); ++ } ++} ++#endif ++ ++static noinline void next_prev_test(struct maple_tree *mt) ++{ ++ int i, nr_entries = 200; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ struct maple_enode *mn; ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i <= nr_entries / 2; i++) { ++ mas_next(&mas, 1000); ++ if (mas_is_none(&mas)) ++ break; ++ ++ } ++ mas_reset(&mas); ++ mas_set(&mas, 0); ++ i = 0; ++ mas_for_each(&mas, val, 1000) { ++ i++; ++ } ++ ++ mas_reset(&mas); ++ mas_set(&mas, 0); ++ i = 0; ++ mas_for_each(&mas, val, 1000) { ++ mas_pause(&mas); ++ i++; ++ } ++ ++ /* ++ * 680 - 685 = 0x61a00001930c ++ * 686 - 689 = NULL; ++ * 690 - 695 = 0x61a00001930c ++ * Check simple next/prev ++ */ ++ mas_set(&mas, 686); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != NULL); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); ++ MT_BUG_ON(mt, mas.index != 690); ++ MT_BUG_ON(mt, mas.last != 695); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(680 / 10)); ++ MT_BUG_ON(mt, mas.index != 680); ++ MT_BUG_ON(mt, mas.last != 685); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); ++ MT_BUG_ON(mt, mas.index != 690); ++ MT_BUG_ON(mt, mas.last != 695); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(700 / 10)); ++ MT_BUG_ON(mt, mas.index != 700); ++ MT_BUG_ON(mt, mas.last != 705); ++ ++ /* Check across node boundaries of the tree */ ++ mas_set(&mas, 70); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); ++ MT_BUG_ON(mt, mas.index != 70); ++ MT_BUG_ON(mt, mas.last != 75); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(80 / 10)); ++ MT_BUG_ON(mt, mas.index != 80); ++ MT_BUG_ON(mt, mas.last != 85); ++ ++ val = mas_prev(&mas, 70); ++ MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); ++ MT_BUG_ON(mt, mas.index != 70); ++ MT_BUG_ON(mt, mas.last != 75); ++ ++ /* Check across two levels of the tree */ ++ mas_reset(&mas); ++ mas_set(&mas, 707); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != NULL); ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); ++ MT_BUG_ON(mt, mas.index != 710); ++ MT_BUG_ON(mt, mas.last != 715); ++ mn = mas.node; ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(720 / 10)); ++ MT_BUG_ON(mt, mas.index != 720); ++ MT_BUG_ON(mt, mas.last != 725); ++ MT_BUG_ON(mt, mn == mas.node); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); ++ MT_BUG_ON(mt, mas.index != 710); ++ MT_BUG_ON(mt, mas.last != 715); ++ ++ /* Check running off the end and back on */ ++ mas_reset(&mas); ++ mas_set(&mas, 2000); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); ++ MT_BUG_ON(mt, mas.index != 2000); ++ MT_BUG_ON(mt, mas.last != 2005); ++ ++ val = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, val != NULL); ++ MT_BUG_ON(mt, mas.index != ULONG_MAX); ++ MT_BUG_ON(mt, mas.last != ULONG_MAX); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); ++ MT_BUG_ON(mt, mas.index != 2000); ++ MT_BUG_ON(mt, mas.last != 2005); ++ ++ /* Check running off the start and back on */ ++ mas_reset(&mas); ++ mas_set(&mas, 10); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != xa_mk_value(1)); ++ MT_BUG_ON(mt, mas.index != 10); ++ MT_BUG_ON(mt, mas.last != 15); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(0)); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != 5); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != NULL); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != 0); ++ ++ mas.index = 0; ++ mas.last = 5; ++ mas_store(&mas, NULL); ++ mas_reset(&mas); ++ mas_set(&mas, 10); ++ mas_walk(&mas); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != NULL); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != 0); ++ ++ mtree_destroy(mt); ++ ++ mt_init(mt); ++ mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); ++ mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL); ++ mas_set(&mas, 5); ++ val = mas_prev(&mas, 4); ++ MT_BUG_ON(mt, val != NULL); ++} ++ ++#define RCU_RANGE_COUNT 1000 ++#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true;} MT_BUG_ON(test->mt, y);} ++struct rcu_test_struct2 { ++ struct maple_tree *mt; ++ ++ bool start; ++ bool stop; ++ unsigned int thread_count; ++ ++ unsigned int seen_toggle; ++ unsigned int seen_added; ++ unsigned int seen_modified; ++ unsigned int seen_deleted; ++ int pause; ++ ++ unsigned long index[RCU_RANGE_COUNT]; ++ unsigned long last[RCU_RANGE_COUNT]; ++}; ++ ++struct rcu_reader_struct { ++ unsigned int id; ++ int mod; ++ int del; ++ int flip; ++ int add; ++ int next; ++ struct rcu_test_struct2 *test; ++}; ++ ++/* RCU reader helper function */ ++static void rcu_reader_register(struct rcu_test_struct2 *test) ++{ ++ rcu_register_thread(); ++ uatomic_inc(&test->thread_count); ++ ++ while (!test->start) ++ usleep(test->pause * 100); ++} ++ ++static void rcu_reader_setup(struct rcu_reader_struct *reader, ++ unsigned int id, struct rcu_test_struct2 *test) ++{ ++ reader->id = id; ++ reader->test = test; ++ reader->mod = reader->id % 10; ++ reader->del = (reader->mod + 1) % 10; ++ reader->flip = (reader->mod + 2) % 10; ++ reader->add = (reader->mod + 3) % 10; ++ reader->next = (reader->mod + 4) % 10; ++} ++ ++/* RCU reader in increasing index */ ++static void *rcu_reader_fwd(void *ptr) ++{ ++ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; ++ struct rcu_test_struct2 *test = reader->test; ++ unsigned long index = reader->id; ++ bool toggled, modified, deleted, added; ++ int i; ++ void *entry, *prev = NULL; ++ MA_STATE(mas, test->mt, 0, 0); ++ ++ rcu_reader_register(test); ++ toggled = modified = deleted = added = false; ++ ++ while (!test->stop) { ++ i = 0; ++ /* mas_for_each ?*/ ++ rcu_read_lock(); ++ mas_set(&mas, test->index[index]); ++ mas_for_each(&mas, entry, test->last[index + 9]) { ++ unsigned long r_start, r_end, alt_start; ++ void *expected, *alt; ++ ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ ++ if (i == reader->del) { ++ if (!deleted) { ++ alt_start = test->index[index + reader->flip]; ++ /* delete occurred. */ ++ if (mas.index == alt_start) { ++ uatomic_inc(&test->seen_deleted); ++ deleted = true; ++ } ++ } ++ if (deleted) { ++ i = reader->flip; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } ++ ++ if (!added && (i == reader->add)) { ++ alt_start = test->index[index + reader->next]; ++ if (mas.index == r_start) { ++ uatomic_inc(&test->seen_added); ++ added = true; ++ } else if (mas.index == alt_start) { ++ i = reader->next; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } ++ ++ RCU_MT_BUG_ON(test, mas.index != r_start); ++ RCU_MT_BUG_ON(test, mas.last != r_end); ++ ++ if (i == reader->flip) { ++ alt = xa_mk_value(index + i + RCU_RANGE_COUNT); ++ if (prev) { ++ if (toggled && entry == expected) ++ uatomic_inc(&test->seen_toggle); ++ else if (!toggled && entry == alt) ++ uatomic_inc(&test->seen_toggle); ++ } ++ ++ if (entry == expected) ++ toggled = false; ++ else if (entry == alt) ++ toggled = true; ++ else { ++ printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); ++ RCU_MT_BUG_ON(test, 1); ++ } ++ ++ prev = entry; ++ } else if (i == reader->mod) { ++ alt = xa_mk_value(index + i * 2 + 1 + ++ RCU_RANGE_COUNT); ++ if (entry != expected) { ++ if (!modified) ++ uatomic_inc(&test->seen_modified); ++ modified = true; ++ } else { ++ if (modified) ++ uatomic_inc(&test->seen_modified); ++ modified = false; ++ } ++ ++ if (modified) ++ RCU_MT_BUG_ON(test, entry != alt); ++ ++ } else { ++ if (entry != expected) ++ printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); ++ RCU_MT_BUG_ON(test, entry != expected); ++ } ++ ++ i++; ++ } ++ rcu_read_unlock(); ++ usleep(test->pause); ++ } ++ ++ rcu_unregister_thread(); ++ return NULL; ++} ++ ++/* RCU reader in decreasing index */ ++static void *rcu_reader_rev(void *ptr) ++{ ++ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; ++ struct rcu_test_struct2 *test = reader->test; ++ unsigned long index = reader->id; ++ bool toggled, modified, deleted, added; ++ int i; ++ void *prev = NULL; ++ MA_STATE(mas, test->mt, 0, 0); ++ ++ rcu_reader_register(test); ++ toggled = modified = deleted = added = false; ++ ++ ++ while (!test->stop) { ++ void *entry; ++ ++ i = 9; ++ mas_set(&mas, test->index[index + i]); ++ ++ rcu_read_lock(); ++ while (i--) { ++ unsigned long r_start, r_end, alt_start; ++ void *expected, *alt; ++ int line = __LINE__; ++ ++ entry = mas_prev(&mas, test->index[index]); ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ ++ if (i == reader->del) { ++ alt_start = test->index[index + reader->mod]; ++ if (mas.index == alt_start) { ++ line = __LINE__; ++ if (!deleted) ++ uatomic_inc(&test->seen_deleted); ++ deleted = true; ++ } ++ if (deleted) { ++ line = __LINE__; ++ i = reader->mod; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } ++ if (!added && (i == reader->add)) { ++ alt_start = test->index[index + reader->flip]; ++ if (mas.index == r_start) { ++ line = __LINE__; ++ uatomic_inc(&test->seen_added); ++ added = true; ++ } else if (mas.index == alt_start) { ++ line = __LINE__; ++ i = reader->flip; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } ++ ++ if (i == reader->mod) ++ line = __LINE__; ++ else if (i == reader->flip) ++ line = __LINE__; ++ ++ if (mas.index != r_start) { ++ alt = xa_mk_value(index + i * 2 + 1 + ++ RCU_RANGE_COUNT); ++ mt_dump(test->mt); ++ printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n", ++ mas.index, mas.last, entry, ++ r_start, r_end, expected, alt, ++ line, i); ++ } ++ RCU_MT_BUG_ON(test, mas.index != r_start); ++ RCU_MT_BUG_ON(test, mas.last != r_end); ++ ++ if (i == reader->mod) { ++ alt = xa_mk_value(index + i * 2 + 1 + ++ RCU_RANGE_COUNT); ++ ++ if (entry != expected) { ++ if (!modified) ++ uatomic_inc(&test->seen_modified); ++ modified = true; ++ } else { ++ if (modified) ++ uatomic_inc(&test->seen_modified); ++ modified = false; ++ } ++ if (modified) ++ RCU_MT_BUG_ON(test, entry != alt); ++ ++ ++ } else if (i == reader->flip) { ++ alt = xa_mk_value(index + i + ++ RCU_RANGE_COUNT); ++ if (prev) { ++ if (toggled && entry == expected) ++ uatomic_inc(&test->seen_toggle); ++ else if (!toggled && entry == alt) ++ uatomic_inc(&test->seen_toggle); ++ } ++ ++ if (entry == expected) ++ toggled = false; ++ else if (entry == alt) ++ toggled = true; ++ else { ++ printk("%lu-%lu %p != %p or %p\n", ++ mas.index, mas.last, entry, ++ expected, alt); ++ RCU_MT_BUG_ON(test, 1); ++ } ++ ++ prev = entry; ++ } else { ++ if (entry != expected) ++ printk("%lu-%lu %p != %p\n", mas.index, ++ mas.last, entry, expected); ++ RCU_MT_BUG_ON(test, entry != expected); ++ } ++ } ++ rcu_read_unlock(); ++ usleep(test->pause); ++ } ++ ++ rcu_unregister_thread(); ++ return NULL; ++} ++ ++static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test, ++ int count, struct rcu_reader_struct *test_reader) ++{ ++ int i, j = 10000; ++ bool toggle = true; ++ ++ test->start = true; /* Release the hounds! */ ++ usleep(5); ++ ++ while (j--) { ++ toggle = !toggle; ++ i = count; ++ while (i--) { ++ unsigned long start, end; ++ struct rcu_reader_struct *this = &test_reader[i]; ++ ++ /* Mod offset */ ++ if (j == 600) { ++ start = test->index[this->id + this->mod]; ++ end = test->last[this->id + this->mod]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(this->id + this->mod * 2 + ++ 1 + RCU_RANGE_COUNT), ++ GFP_KERNEL); ++ } ++ ++ /* Toggle */ ++ if (!(j % 5)) { ++ start = test->index[this->id + this->flip]; ++ end = test->last[this->id + this->flip]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value((toggle ? start : ++ this->id + this->flip + ++ RCU_RANGE_COUNT)), ++ GFP_KERNEL); ++ } ++ ++ /* delete */ ++ if (j == 400) { ++ start = test->index[this->id + this->del]; ++ end = test->last[this->id + this->del]; ++ mtree_store_range(mt, start, end, NULL, GFP_KERNEL); ++ } ++ ++ /* add */ ++ if (j == 500) { ++ start = test->index[this->id + this->add]; ++ end = test->last[this->id + this->add]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(start), GFP_KERNEL); ++ } ++ } ++ usleep(test->pause); ++ /* If a test fails, don't flood the console */ ++ if (test->stop) ++ break; ++ } ++} ++ ++static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test, ++ int count, struct rcu_reader_struct *test_reader) ++{ ++ int j, i; ++ bool toggle = true; ++ ++ test->start = true; /* Release the hounds! */ ++ usleep(5); ++ for (j = 0; j < 10000; j++) { ++ toggle = !toggle; ++ for (i = 0; i < count; i++) { ++ unsigned long start, end; ++ struct rcu_reader_struct *this = &test_reader[i]; ++ ++ /* Mod offset */ ++ if (j == 600) { ++ start = test->index[this->id + this->mod]; ++ end = test->last[this->id + this->mod]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(this->id + this->mod * 2 + ++ 1 + RCU_RANGE_COUNT), ++ GFP_KERNEL); ++ } ++ ++ /* Toggle */ ++ if (!(j % 5)) { ++ start = test->index[this->id + this->flip]; ++ end = test->last[this->id + this->flip]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value((toggle ? start : ++ this->id + this->flip + ++ RCU_RANGE_COUNT)), ++ GFP_KERNEL); ++ } ++ ++ /* delete */ ++ if (j == 400) { ++ start = test->index[this->id + this->del]; ++ end = test->last[this->id + this->del]; ++ mtree_store_range(mt, start, end, NULL, GFP_KERNEL); ++ } ++ ++ /* add */ ++ if (j == 500) { ++ start = test->index[this->id + this->add]; ++ end = test->last[this->id + this->add]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(start), GFP_KERNEL); ++ } ++ } ++ usleep(test->pause); ++ /* If a test fails, don't flood the console */ ++ if (test->stop) ++ break; ++ } ++} ++ ++/* ++ * This is to check: ++ * 1. Range that is not ever present ++ * 2. Range that is always present ++ * 3. Things being added but not removed. ++ * 4. Things being removed but not added. ++ * 5. Things are being added and removed, searches my succeed or fail ++ * ++ * This sets up two readers for every 10 entries; one forward and one reverse ++ * reading. ++ */ ++static void rcu_stress(struct maple_tree *mt, bool forward) ++{ ++ unsigned int count, i; ++ unsigned long r, seed; ++ pthread_t readers[RCU_RANGE_COUNT / 5]; ++ struct rcu_test_struct2 test; ++ struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5]; ++ void *(*function)(void *); ++ ++ /* Test setup */ ++ test.mt = mt; ++ test.pause = 5; ++ test.seen_toggle = 0; ++ test.seen_deleted = 0; ++ test.seen_added = 0; ++ test.seen_modified = 0; ++ test.thread_count = 0; ++ test.start = test.stop = false; ++ seed = time(NULL); ++ srand(seed); ++ for (i = 0; i < RCU_RANGE_COUNT; i++) { ++ r = seed + rand(); ++ mtree_store_range(mt, seed, r, ++ xa_mk_value(seed), GFP_KERNEL); ++ ++ /* Record start and end of entry */ ++ test.index[i] = seed; ++ test.last[i] = r; ++ seed = 1 + r + rand() % 10; ++ } ++ ++ i = count = ARRAY_SIZE(readers); ++ while (i--) { ++ unsigned long id; ++ ++ id = i / 2 * 10; ++ if (i % 2) ++ function = rcu_reader_fwd; ++ else ++ function = rcu_reader_rev; ++ ++ rcu_reader_setup(&test_reader[i], id, &test); ++ if (pthread_create(&readers[i], NULL, *function, ++ &test_reader[i])) { ++ perror("creating reader thread"); ++ exit(1); ++ } ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(readers); i++) { ++ struct rcu_reader_struct *this = &test_reader[i]; ++ int add = this->id + this->add; ++ ++ /* Remove add entries from the tree for later addition */ ++ mtree_store_range(mt, test.index[add], test.last[add], ++ NULL, GFP_KERNEL); ++ } ++ ++ mt_set_in_rcu(mt); ++ do { ++ usleep(5); ++ } while (test.thread_count > ARRAY_SIZE(readers)); ++ ++ if (forward) ++ rcu_stress_fwd(mt, &test, count, test_reader); ++ else ++ rcu_stress_rev(mt, &test, count, test_reader); ++ ++ test.stop = true; ++ while (count--) ++ pthread_join(readers[count], NULL); ++ ++ mt_validate(mt); ++} ++ ++ ++struct rcu_test_struct { ++ struct maple_tree *mt; /* the maple tree */ ++ int count; /* Number of times to check value(s) */ ++ unsigned long index; /* The first index to check */ ++ void *entry1; /* The first entry value */ ++ void *entry2; /* The second entry value */ ++ void *entry3; /* The third entry value */ ++ ++ bool update_2; ++ bool update_3; ++ unsigned long range_start; ++ unsigned long range_end; ++ unsigned int loop_sleep; ++ unsigned int val_sleep; ++ ++ unsigned int failed; /* failed detection for other threads */ ++ unsigned int seen_entry2; /* Number of threads that have seen the new value */ ++ unsigned int seen_entry3; /* Number of threads that have seen the new value */ ++ unsigned int seen_both; /* Number of threads that have seen both new values */ ++ unsigned int seen_toggle; ++ unsigned int seen_added; ++ unsigned int seen_removed; ++ unsigned long last; /* The end of the range to write. */ ++ ++ unsigned long removed; /* The index of the removed entry */ ++ unsigned long added; /* The index of the removed entry */ ++ unsigned long toggle; /* The index of the removed entry */ ++}; ++ ++static inline ++int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2, ++ bool *update_3) ++{ ++ if (entry == test->entry1) ++ return 0; ++ ++ if (entry == test->entry2) { ++ if (!(*update_2)) { ++ uatomic_inc(&test->seen_entry2); ++ *update_2 = true; ++ if (update_3) ++ uatomic_inc(&test->seen_both); ++ } ++ return 0; ++ } ++ ++ if (entry == test->entry3) { ++ if (!(*update_3)) { ++ uatomic_inc(&test->seen_entry3); ++ *update_3 = true; ++ if (update_2) ++ uatomic_inc(&test->seen_both); ++ } ++ return 0; ++ } ++ ++ return 1; ++} ++ ++/* ++ * rcu_val() - Read a given value in the tree test->count times using the ++ * regular API ++ * ++ * @ptr: The pointer to the rcu_test_struct ++ */ ++static void *rcu_val(void *ptr) ++{ ++ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; ++ unsigned long count = test->count; ++ bool update_2 = false; ++ bool update_3 = false; ++ void *entry; ++ ++ rcu_register_thread(); ++ while (count--) { ++ usleep(test->val_sleep); ++ /* ++ * No locking required, regular API locking is handled in the ++ * maple tree code ++ */ ++ entry = mtree_load(test->mt, test->index); ++ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2, ++ &update_3)); ++ } ++ rcu_unregister_thread(); ++ return NULL; ++} ++ ++/* ++ * rcu_loop() - Loop over a section of the maple tree, checking for an expected ++ * value using the advanced API ++ * ++ * @ptr - The pointer to the rcu_test_struct ++ */ ++static void *rcu_loop(void *ptr) ++{ ++ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; ++ unsigned long count = test->count; ++ void *entry, *expected; ++ bool update_2 = false; ++ bool update_3 = false; ++ MA_STATE(mas, test->mt, test->range_start, test->range_start); ++ ++ rcu_register_thread(); ++ ++ /* ++ * Loop through the test->range_start - test->range_end test->count ++ * times ++ */ ++ while (count--) { ++ usleep(test->loop_sleep); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, test->range_end) { ++ /* The expected value is based on the start range. */ ++ expected = xa_mk_value(mas.index ? mas.index / 10 : 0); ++ ++ /* Out of the interesting range */ ++ if (mas.index < test->index || mas.index > test->last) { ++ if (entry != expected) { ++ printk("%lx - %lx = %p not %p\n", ++ mas.index, mas.last, entry, expected); ++ } ++ MT_BUG_ON(test->mt, entry != expected); ++ continue; ++ } ++ ++ if (entry == expected) ++ continue; /* Not seen. */ ++ ++ /* In the interesting range */ ++ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, ++ &update_2, ++ &update_3)); ++ } ++ rcu_read_unlock(); ++ mas_set(&mas, test->range_start); ++ } ++ ++ rcu_unregister_thread(); ++ return NULL; ++} ++ ++static noinline ++void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals) ++{ ++ ++ int i; ++ void *(*function)(void *); ++ pthread_t readers[20]; ++ ++ mt_set_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_in_rcu(mt)); ++ ++ for (i = 0; i < ARRAY_SIZE(readers); i++) { ++ if (i % 2) ++ function = rcu_loop; ++ else ++ function = rcu_val; ++ ++ if (pthread_create(&readers[i], NULL, *function, vals)) { ++ perror("creating reader thread"); ++ exit(1); ++ } ++ } ++ ++ usleep(5); /* small yield to ensure all threads are at least started. */ ++ mtree_store_range(mt, vals->index, vals->last, vals->entry2, ++ GFP_KERNEL); ++ while (i--) ++ pthread_join(readers[i], NULL); ++ ++ /* Make sure the test caught at least one update. */ ++ MT_BUG_ON(mt, !vals->seen_entry2); ++} ++ ++static noinline ++void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals) ++{ ++ ++ int i; ++ void *(*function)(void *); ++ pthread_t readers[20]; ++ unsigned int index = vals->index; ++ ++ mt_set_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_in_rcu(mt)); ++ ++ for (i = 0; i < ARRAY_SIZE(readers); i++) { ++ if (i % 2) ++ function = rcu_loop; ++ else ++ function = rcu_val; ++ ++ if (pthread_create(&readers[i], NULL, *function, vals)) { ++ perror("creating reader thread"); ++ exit(1); ++ } ++ } ++ ++ usleep(5); /* small yield to ensure all threads are at least started. */ ++ ++ while (index <= vals->last) { ++ mtree_store(mt, index, ++ (index % 2 ? vals->entry2 : vals->entry3), ++ GFP_KERNEL); ++ index++; ++ usleep(5); ++ } ++ ++ while (i--) ++ pthread_join(readers[i], NULL); ++ ++ /* Make sure the test caught at least one update. */ ++ MT_BUG_ON(mt, !vals->seen_entry2); ++ MT_BUG_ON(mt, !vals->seen_entry3); ++ MT_BUG_ON(mt, !vals->seen_both); ++} ++static noinline void check_rcu_simulated(struct maple_tree *mt) ++{ ++ unsigned long i, nr_entries = 1000; ++ unsigned long target = 4320; ++ unsigned long val = 0xDEAD; ++ ++ MA_STATE(mas_writer, mt, 0, 0); ++ MA_STATE(mas_reader, mt, target, target); ++ ++ rcu_register_thread(); ++ ++ mt_set_in_rcu(mt); ++ mas_lock(&mas_writer); ++ for (i = 0; i <= nr_entries; i++) { ++ mas_writer.index = i * 10; ++ mas_writer.last = i * 10 + 5; ++ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); ++ } ++ mas_unlock(&mas_writer); ++ ++ /* Overwrite one entry with a new value. */ ++ mas_set_range(&mas_writer, target, target + 5); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ ++ /* Overwrite 1/2 the entry */ ++ mas_set_range(&mas_writer, target, target + 2); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite last 1/2 the entry */ ++ mas_set_range(&mas_writer, target + 2, target + 5); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ rcu_read_unlock(); ++ ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite more than the entry */ ++ mas_set_range(&mas_writer, target - 5, target + 15); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite more than the node. */ ++ mas_set_range(&mas_writer, target - 400, target + 400); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite the tree */ ++ mas_set_range(&mas_writer, 0, ULONG_MAX); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Clear out tree & recreate it */ ++ mas_lock(&mas_writer); ++ mas_set_range(&mas_writer, 0, ULONG_MAX); ++ mas_store_gfp(&mas_writer, NULL, GFP_KERNEL); ++ mas_set_range(&mas_writer, 0, 0); ++ for (i = 0; i <= nr_entries; i++) { ++ mas_writer.index = i * 10; ++ mas_writer.last = i * 10 + 5; ++ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); ++ } ++ mas_unlock(&mas_writer); ++ ++ /* next check */ ++ /* Overwrite one entry with a new value. */ ++ mas_reset(&mas_reader); ++ mas_set_range(&mas_writer, target, target + 5); ++ mas_set_range(&mas_reader, target, target); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_prev(&mas_reader, 0); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ ++ /* prev check */ ++ /* Overwrite one entry with a new value. */ ++ mas_reset(&mas_reader); ++ mas_set_range(&mas_writer, target, target + 5); ++ mas_set_range(&mas_reader, target, target); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_next(&mas_reader, ULONG_MAX); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ rcu_unregister_thread(); ++} ++ ++static noinline void check_rcu_threaded(struct maple_tree *mt) ++{ ++ unsigned long i, nr_entries = 1000; ++ struct rcu_test_struct vals; ++ ++ vals.val_sleep = 200; ++ vals.loop_sleep = 110; ++ ++ rcu_register_thread(); ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ /* Store across several slots. */ ++ vals.count = 1000; ++ vals.mt = mt; ++ vals.index = 8650; ++ vals.last = 8666; ++ vals.entry1 = xa_mk_value(865); ++ vals.entry2 = xa_mk_value(8650); ++ vals.entry3 = xa_mk_value(8650); ++ vals.range_start = 0; ++ vals.range_end = ULONG_MAX; ++ vals.seen_entry2 = 0; ++ vals.seen_entry3 = 0; ++ ++ run_check_rcu(mt, &vals); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ /* 4390-4395: value 439 (0x1b7) [0x36f] */ ++ /* Store across several slots. */ ++ /* Spanning store. */ ++ vals.count = 10000; ++ vals.mt = mt; ++ vals.index = 4390; ++ vals.last = 4398; ++ vals.entry1 = xa_mk_value(4390); ++ vals.entry2 = xa_mk_value(439); ++ vals.entry3 = xa_mk_value(439); ++ vals.seen_entry2 = 0; ++ vals.range_start = 4316; ++ vals.range_end = 5035; ++ run_check_rcu(mt, &vals); ++ mtree_destroy(mt); ++ ++ ++ /* Forward writer for rcu stress */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ rcu_stress(mt, true); ++ mtree_destroy(mt); ++ ++ /* Reverse writer for rcu stress */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ rcu_stress(mt, false); ++ mtree_destroy(mt); ++ ++ /* Slow reader test with spanning store. */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ /* 4390-4395: value 439 (0x1b7) [0x36f] */ ++ /* Store across several slots. */ ++ /* Spanning store. */ ++ vals.count = 15000; ++ vals.mt = mt; ++ vals.index = 4390; ++ vals.last = 4398; ++ vals.entry1 = xa_mk_value(4390); ++ vals.entry2 = xa_mk_value(439); ++ vals.entry3 = xa_mk_value(4391); ++ vals.seen_toggle = 0; ++ vals.seen_added = 0; ++ vals.seen_removed = 0; ++ vals.range_start = 4316; ++ vals.range_end = 5035; ++ vals.removed = 4360; ++ vals.added = 4396; ++ vals.toggle = 4347; ++ vals.val_sleep = 400; ++ vals.loop_sleep = 200; ++ vals.seen_entry2 = 0; ++ vals.seen_entry3 = 0; ++ vals.seen_both = 0; ++ vals.entry3 = xa_mk_value(438); ++ ++ run_check_rcu_slowread(mt, &vals); ++ rcu_unregister_thread(); ++} ++ ++extern void test_kmem_cache_bulk(void); ++ ++/* Test spanning writes that require balancing right sibling or right cousin */ ++static noinline void check_spanning_relatives(struct maple_tree *mt) ++{ ++ ++ unsigned long i, nr_entries = 1000; ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ ++ mtree_store_range(mt, 9365, 9955, NULL, GFP_KERNEL); ++} ++ ++static noinline void check_fuzzer(struct maple_tree *mt) ++{ ++ /* ++ * 1. Causes a spanning rebalance of a single root node. ++ * Fixed by setting the correct limit in mast_cp_to_nodes() when the ++ * entire right side is consumed. ++ */ ++ mtree_test_insert(mt, 88, (void *)0xb1); ++ mtree_test_insert(mt, 84, (void *)0xa9); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 14, (void *)0x1d); ++ mtree_test_insert(mt, 7, (void *)0xf); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_insert(mt, 18, (void *)0x25); ++ mtree_test_store_range(mt, 8, 18, (void *)0x11); ++ mtree_destroy(mt); ++ ++ ++ /* ++ * 2. Cause a spanning rebalance of two nodes in root. ++ * Fixed by setting mast->r->max correctly. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store(mt, 87, (void *)0xaf); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 4); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_store(mt, 8, (void *)0x11); ++ mtree_test_store(mt, 44, (void *)0x59); ++ mtree_test_store(mt, 68, (void *)0x89); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 43, (void *)0x57); ++ mtree_test_insert(mt, 24, (void *)0x31); ++ mtree_test_insert(mt, 844, (void *)0x699); ++ mtree_test_store(mt, 84, (void *)0xa9); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_erase(mt, 4); ++ mtree_test_load(mt, 5); ++ mtree_test_erase(mt, 0); ++ mtree_destroy(mt); ++ ++ /* ++ * 3. Cause a node overflow on copy ++ * Fixed by using the correct check for node size in mas_wr_modify() ++ * Also discovered issue with metadata setting. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store_range(mt, 0, 18446744073709551615UL, (void *)0x1); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_erase(mt, 5); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 4); ++ mtree_test_store(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 5); ++ mtree_test_store(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 5); ++ mtree_test_erase(mt, 4); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_store(mt, 444, (void *)0x379); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 0); ++ mtree_test_store(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 0); ++ mtree_destroy(mt); ++ ++ /* ++ * 4. spanning store failure due to writing incorrect pivot value at ++ * last slot. ++ * Fixed by setting mast->r->max correctly in mast_cp_to_nodes() ++ * ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_insert(mt, 261, (void *)0x20b); ++ mtree_test_store(mt, 516, (void *)0x409); ++ mtree_test_store(mt, 6, (void *)0xd); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_insert(mt, 1256, (void *)0x9d1); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_erase(mt, 1); ++ mtree_test_store(mt, 56, (void *)0x71); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 24, (void *)0x31); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 2263, (void *)0x11af); ++ mtree_test_insert(mt, 446, (void *)0x37d); ++ mtree_test_store_range(mt, 6, 45, (void *)0xd); ++ mtree_test_store_range(mt, 3, 446, (void *)0x7); ++ mtree_destroy(mt); ++ ++ /* ++ * 5. mas_wr_extend_null() may overflow slots. ++ * Fix by checking against wr_mas->node_end. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store(mt, 48, (void *)0x61); ++ mtree_test_store(mt, 3, (void *)0x7); ++ mtree_test_load(mt, 0); ++ mtree_test_store(mt, 88, (void *)0xb1); ++ mtree_test_store(mt, 81, (void *)0xa3); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 2480, (void *)0x1361); ++ mtree_test_insert(mt, 18446744073709551615UL, ++ (void *)0xffffffffffffffff); ++ mtree_test_erase(mt, 18446744073709551615UL); ++ mtree_destroy(mt); ++ ++ /* ++ * 6. When reusing a node with an implied pivot and the node is ++ * shrinking, old data would be left in the implied slot ++ * Fixed by checking the last pivot for the mas->max and clear ++ * accordingly. This only affected the left-most node as that node is ++ * the only one allowed to end in NULL. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_erase(mt, 3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_load(mt, 2); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_load(mt, 2); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_erase(mt, 1); ++ mtree_test_store_range(mt, 4, 62, (void *)0x9); ++ mtree_test_erase(mt, 62); ++ mtree_test_store_range(mt, 1, 0, (void *)0x3); ++ mtree_test_insert(mt, 11, (void *)0x17); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_store(mt, 62, (void *)0x7d); ++ mtree_test_erase(mt, 62); ++ mtree_test_store_range(mt, 1, 15, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_store(mt, 62, (void *)0x7d); ++ mtree_test_erase(mt, 62); ++ mtree_test_insert(mt, 122, (void *)0xf5); ++ mtree_test_store(mt, 3, (void *)0x7); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_store_range(mt, 0, 1, (void *)0x1); ++ mtree_test_insert(mt, 85, (void *)0xab); ++ mtree_test_insert(mt, 72, (void *)0x91); ++ mtree_test_insert(mt, 81, (void *)0xa3); ++ mtree_test_insert(mt, 726, (void *)0x5ad); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 51, (void *)0x67); ++ mtree_test_insert(mt, 611, (void *)0x4c7); ++ mtree_test_insert(mt, 485, (void *)0x3cb); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 26, 1, (void *)0x35); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 22, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 53); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 1, (void *)0x3); ++ mtree_test_insert(mt, 222, (void *)0x1bd); ++ mtree_test_insert(mt, 485, (void *)0x3cb); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 0); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_store(mt, 621, (void *)0x4db); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_erase(mt, 5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 62, (void *)0x7d); ++ mtree_test_erase(mt, 62); ++ mtree_test_store_range(mt, 1, 0, (void *)0x3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store_range(mt, 4, 62, (void *)0x9); ++ mtree_test_erase(mt, 62); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 22, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 53); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 1, (void *)0x3); ++ mtree_test_insert(mt, 222, (void *)0x1bd); ++ mtree_test_insert(mt, 485, (void *)0x3cb); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_load(mt, 0); ++ mtree_test_load(mt, 0); ++ mtree_destroy(mt); ++ ++ /* ++ * 7. Previous fix was incomplete, fix mas_resuse_node() clearing of old ++ * data by overwriting it first - that way metadata is of no concern. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_load(mt, 1); ++ mtree_test_insert(mt, 102, (void *)0xcd); ++ mtree_test_erase(mt, 2); ++ mtree_test_erase(mt, 0); ++ mtree_test_load(mt, 0); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 110, (void *)0xdd); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 5, 0, (void *)0xb); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_store(mt, 112, (void *)0xe1); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 110, 2, (void *)0xdd); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_load(mt, 22); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 210, (void *)0x1a5); ++ mtree_test_store_range(mt, 0, 2, (void *)0x1); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 2); ++ mtree_test_erase(mt, 22); ++ mtree_test_erase(mt, 1); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 112); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 1, 2, (void *)0x3); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 2); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert_range(mt, 1, 2, (void *)0x3); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 112); ++ mtree_test_store_range(mt, 110, 12, (void *)0xdd); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_load(mt, 110); ++ mtree_test_insert_range(mt, 4, 71, (void *)0x9); ++ mtree_test_load(mt, 2); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_insert_range(mt, 11, 22, (void *)0x17); ++ mtree_test_erase(mt, 12); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_load(mt, 22); ++ mtree_destroy(mt); ++ ++ ++ /* ++ * 8. When rebalancing or spanning_rebalance(), the max of the new node ++ * may be set incorrectly to the final pivot and not the right max. ++ * Fix by setting the left max to orig right max if the entire node is ++ * consumed. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store(mt, 6, (void *)0xd); ++ mtree_test_store(mt, 67, (void *)0x87); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 6716, (void *)0x3479); ++ mtree_test_store(mt, 61, (void *)0x7b); ++ mtree_test_insert(mt, 13, (void *)0x1b); ++ mtree_test_store(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_load(mt, 0); ++ mtree_test_erase(mt, 67167); ++ mtree_test_insert_range(mt, 6, 7167, (void *)0xd); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_erase(mt, 67); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 667167); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_store(mt, 67, (void *)0x87); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_erase(mt, 67); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 67167, (void *)0x20cbf); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_load(mt, 7); ++ mtree_test_insert(mt, 16, (void *)0x21); ++ mtree_test_insert(mt, 36, (void *)0x49); ++ mtree_test_store(mt, 67, (void *)0x87); ++ mtree_test_store(mt, 6, (void *)0xd); ++ mtree_test_insert(mt, 367, (void *)0x2df); ++ mtree_test_insert(mt, 115, (void *)0xe7); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_store_range(mt, 1, 3, (void *)0x3); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 67167); ++ mtree_test_insert_range(mt, 6, 47, (void *)0xd); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 1, 67, (void *)0x3); ++ mtree_test_load(mt, 67); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 67167); ++ mtree_destroy(mt); ++ ++ /* ++ * 9. spanning store to the end of data caused an invalid metadata ++ * length which resulted in a crash eventually. ++ * Fix by checking if there is a value in pivot before incrementing the ++ * metadata end in mab_mas_cp(). To ensure this doesn't happen again, ++ * abstract the two locations this happens into a function called ++ * mas_leaf_set_meta(). ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 91, (void *)0xb7); ++ mtree_test_insert(mt, 18, (void *)0x25); ++ mtree_test_insert(mt, 81, (void *)0xa3); ++ mtree_test_store_range(mt, 0, 128, (void *)0x1); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 8); ++ mtree_test_insert(mt, 11, (void *)0x17); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); ++ mtree_test_erase(mt, 18446744073709551605UL); ++ mtree_test_store_range(mt, 0, 281, (void *)0x1); ++ mtree_test_erase(mt, 2); ++ mtree_test_insert(mt, 1211, (void *)0x977); ++ mtree_test_insert(mt, 111, (void *)0xdf); ++ mtree_test_insert(mt, 13, (void *)0x1b); ++ mtree_test_insert(mt, 211, (void *)0x1a7); ++ mtree_test_insert(mt, 11, (void *)0x17); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_insert(mt, 1218, (void *)0x985); ++ mtree_test_insert(mt, 61, (void *)0x7b); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 121, (void *)0xf3); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); ++ mtree_test_erase(mt, 18446744073709551605UL); ++} ++static noinline void check_dup_gaps(struct maple_tree *mt, ++ unsigned long nr_entries, bool zero_start, ++ unsigned long gap) ++{ ++ unsigned long i = 0; ++ struct maple_tree newmt; ++ int ret; ++ void *tmp; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, &newmt, 0, 0); ++ ++ ++ if (!zero_start) ++ i = 1; ++ ++ mt_zero_nr_tallocated(); ++ for (; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, (i+1)*10 - gap, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ mt_set_non_kernel(99999); ++ ret = mas_expected_entries(&newmas, nr_entries); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, ret != 0); ++ ++ mas_for_each(&mas, tmp, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store(&newmas, tmp); ++ } ++ ++ mas_destroy(&mas); ++ mas_destroy(&newmas); ++ mtree_destroy(&newmt); ++} ++ ++static noinline void check_dup(struct maple_tree *mt) ++{ ++ int i; ++ ++ /* Check with a value at zero */ ++ for (i = 10; i < 1000; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Check with a value at zero, no gap */ ++ for (i = 1000; i < 2000; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, true, 0); ++ mtree_destroy(mt); ++ } ++ ++ /* Check with a value at zero and unreasonably large */ ++ for (i = 100010; i < 100020; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Small to medium size not starting at zero*/ ++ for (i = 200; i < 1000; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, false, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Unreasonably large not starting at zero*/ ++ for (i = 100010; i < 100020; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, false, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Check non-allocation tree not starting at zero */ ++ for (i = 1500; i < 3000; i++) { ++ mt_init_flags(mt, 0); ++ check_dup_gaps(mt, i, false, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Check non-allocation tree starting at zero */ ++ for (i = 200; i < 1000; i++) { ++ mt_init_flags(mt, 0); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Unreasonably large */ ++ for (i = 100015; i < 100020; i++) { ++ mt_init_flags(mt, 0); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } ++ ++} ++ ++static DEFINE_MTREE(tree); ++static int maple_tree_seed(void) ++{ ++ unsigned long set[] = {5015, 5014, 5017, 25, 1000, ++ 1001, 1002, 1003, 1005, 0, ++ 5003, 5002}; ++ void *ptr = &set; ++ ++ pr_info("\nTEST STARTING\n\n"); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_root_expand(&tree); ++ mtree_destroy(&tree); ++ ++#if defined(BENCH_SLOT_STORE) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_slot_store(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_NODE_STORE) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_node_store(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_AWALK) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_awalk(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_WALK) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_walk(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_FORK) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_forking(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_MT_FOR_EACH) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_mt_for_each(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++ ++ test_kmem_cache_bulk(); ++ ++ mt_init_flags(&tree, 0); ++ check_new_node(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_prealloc(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_spanning_write(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_null_expand(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_dfs_preorder(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_forking(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_mas_store_gfp(&tree); ++ mtree_destroy(&tree); ++ ++ /* Test ranges (store and insert) */ ++ mt_init_flags(&tree, 0); ++ check_ranges(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_alloc_range(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_alloc_rev_range(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ ++ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ ++ ++ check_insert(&tree, set[9], &tree); /* Insert 0 */ ++ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ ++ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ ++ ++ check_insert(&tree, set[10], ptr); /* Insert 5003 */ ++ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ ++ check_load(&tree, set[11], NULL); /* See if 5002 -> NULL */ ++ check_load(&tree, set[10], ptr); /* See if 5003 -> ptr */ ++ ++ /* Clear out the tree */ ++ mtree_destroy(&tree); ++ ++ /* Try to insert, insert a dup, and load back what was inserted. */ ++ mt_init_flags(&tree, 0); ++ check_insert(&tree, set[0], &tree); /* Insert 5015 */ ++ check_dup_insert(&tree, set[0], &tree); /* Insert 5015 again */ ++ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ ++ ++ /* ++ * Second set of tests try to load a value that doesn't exist, inserts ++ * a second value, then loads the value again ++ */ ++ check_load(&tree, set[1], NULL); /* See if 5014 -> NULL */ ++ check_insert(&tree, set[1], ptr); /* insert 5014 -> ptr */ ++ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ ++ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ ++ /* ++ * Tree currently contains: ++ * p[0]: 14 -> (nil) p[1]: 15 -> ptr p[2]: 16 -> &tree p[3]: 0 -> (nil) ++ */ ++ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ ++ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ ++ ++ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ ++ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ ++ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ ++ check_load(&tree, set[7], &tree); /* 1003 = &tree ? */ ++ ++ /* Clear out tree */ ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ /* Test inserting into a NULL hole. */ ++ check_insert(&tree, set[5], ptr); /* insert 1001 -> ptr */ ++ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ ++ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ ++ check_load(&tree, set[5], ptr); /* See if 1001 -> ptr */ ++ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ ++ check_load(&tree, set[7], &tree); /* See if 1003 -> &tree */ ++ ++ /* Clear out the tree */ ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_erase_testset(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ /* ++ * set[] = {5015, 5014, 5017, 25, 1000, ++ * 1001, 1002, 1003, 1005, 0, ++ * 5003, 5002}; ++ */ ++ ++ check_insert(&tree, set[0], ptr); /* 5015 */ ++ check_insert(&tree, set[1], &tree); /* 5014 */ ++ check_insert(&tree, set[2], ptr); /* 5017 */ ++ check_insert(&tree, set[3], &tree); /* 25 */ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_insert(&tree, set[4], ptr); /* 1000 < Should split. */ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); /*25 */ ++ check_load(&tree, set[4], ptr); ++ check_insert(&tree, set[5], &tree); /* 1001 */ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_load(&tree, set[4], ptr); ++ check_load(&tree, set[5], &tree); ++ check_insert(&tree, set[6], ptr); ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_load(&tree, set[4], ptr); ++ check_load(&tree, set[5], &tree); ++ check_load(&tree, set[6], ptr); ++ check_insert(&tree, set[7], &tree); ++ check_load(&tree, set[0], ptr); ++ check_insert(&tree, set[8], ptr); ++ ++ check_insert(&tree, set[9], &tree); ++ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_load(&tree, set[4], ptr); ++ check_load(&tree, set[5], &tree); ++ check_load(&tree, set[6], ptr); ++ check_load(&tree, set[9], &tree); ++ mtree_destroy(&tree); ++ ++ check_nomem(&tree); ++ mt_init_flags(&tree, 0); ++ check_seq(&tree, 16, false); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_seq(&tree, 1000, true); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rev_seq(&tree, 1000, true); ++ mtree_destroy(&tree); ++ ++ check_lower_bound_split(&tree); ++ check_upper_bound_split(&tree); ++ check_mid_split(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_next_entry(&tree); ++ check_find(&tree); ++ check_find_2(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_prev_entry(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_erase2_sets(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_gap_combining(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_node_overwrite(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ next_prev_test(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rcu_simulated(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rcu_threaded(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_spanning_relatives(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rev_find(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_fuzzer(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_dup(&tree); ++ mtree_destroy(&tree); ++ ++#if defined(BENCH) ++skip: ++#endif ++ rcu_barrier(); ++ pr_info("maple_tree: %u of %u tests passed\n", ++ atomic_read(&maple_tree_tests_passed), ++ atomic_read(&maple_tree_tests_run)); ++ if (atomic_read(&maple_tree_tests_run) == ++ atomic_read(&maple_tree_tests_passed)) ++ return 0; ++ ++ return -EINVAL; ++} ++ ++static void maple_tree_harvest(void) ++{ ++ ++} ++ ++module_init(maple_tree_seed); ++module_exit(maple_tree_harvest); ++MODULE_AUTHOR("Liam R. Howlett "); ++MODULE_LICENSE("GPL"); +diff --git a/mm/Makefile b/mm/Makefile +index 9a564f836403..8083fa85a348 100644 +--- a/mm/Makefile ++++ b/mm/Makefile +@@ -52,7 +52,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ + readahead.o swap.o truncate.o vmscan.o shmem.o \ + util.o mmzone.o vmstat.o backing-dev.o \ + mm_init.o percpu.o slab_common.o \ +- compaction.o vmacache.o \ ++ compaction.o \ + interval_tree.o list_lru.o workingset.o \ + debug.o gup.o mmap_lock.o $(mmu-y) + +diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h +index d4f55f349100..bce37c487540 100644 +--- a/mm/damon/vaddr-test.h ++++ b/mm/damon/vaddr-test.h +@@ -14,33 +14,19 @@ + + #include + +-static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) ++static void __link_vmas(struct maple_tree *mt, struct vm_area_struct *vmas, ++ ssize_t nr_vmas) + { +- int i, j; +- unsigned long largest_gap, gap; ++ int i; ++ MA_STATE(mas, mt, 0, 0); + + if (!nr_vmas) + return; + +- for (i = 0; i < nr_vmas - 1; i++) { +- vmas[i].vm_next = &vmas[i + 1]; +- +- vmas[i].vm_rb.rb_left = NULL; +- vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb; +- +- largest_gap = 0; +- for (j = i; j < nr_vmas; j++) { +- if (j == 0) +- continue; +- gap = vmas[j].vm_start - vmas[j - 1].vm_end; +- if (gap > largest_gap) +- largest_gap = gap; +- } +- vmas[i].rb_subtree_gap = largest_gap; +- } +- vmas[i].vm_next = NULL; +- vmas[i].vm_rb.rb_right = NULL; +- vmas[i].rb_subtree_gap = 0; ++ mas_lock(&mas); ++ for (i = 0; i < nr_vmas; i++) ++ vma_mas_store(&vmas[i], &mas); ++ mas_unlock(&mas); + } + + /* +@@ -72,6 +58,7 @@ static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) + */ + static void damon_test_three_regions_in_vmas(struct kunit *test) + { ++ static struct mm_struct mm; + struct damon_addr_range regions[3] = {0,}; + /* 10-20-25, 200-210-220, 300-305, 307-330 */ + struct vm_area_struct vmas[] = { +@@ -83,9 +70,10 @@ static void damon_test_three_regions_in_vmas(struct kunit *test) + (struct vm_area_struct) {.vm_start = 307, .vm_end = 330}, + }; + +- __link_vmas(vmas, 6); ++ mt_init_flags(&mm.mm_mt, MM_MT_FLAGS); ++ __link_vmas(&mm.mm_mt, vmas, ARRAY_SIZE(vmas)); + +- __damon_va_three_regions(&vmas[0], regions); ++ __damon_va_three_regions(&mm, regions); + + KUNIT_EXPECT_EQ(test, 10ul, regions[0].start); + KUNIT_EXPECT_EQ(test, 25ul, regions[0].end); +diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c +index 3c7b9d6dca95..d24148a8149f 100644 +--- a/mm/damon/vaddr.c ++++ b/mm/damon/vaddr.c +@@ -113,37 +113,38 @@ static unsigned long sz_range(struct damon_addr_range *r) + * + * Returns 0 if success, or negative error code otherwise. + */ +-static int __damon_va_three_regions(struct vm_area_struct *vma, ++static int __damon_va_three_regions(struct mm_struct *mm, + struct damon_addr_range regions[3]) + { +- struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0}; +- struct vm_area_struct *last_vma = NULL; +- unsigned long start = 0; +- struct rb_root rbroot; +- +- /* Find two biggest gaps so that first_gap > second_gap > others */ +- for (; vma; vma = vma->vm_next) { +- if (!last_vma) { +- start = vma->vm_start; +- goto next; +- } ++ struct damon_addr_range first_gap = {0}, second_gap = {0}; ++ VMA_ITERATOR(vmi, mm, 0); ++ struct vm_area_struct *vma, *prev = NULL; ++ unsigned long start; + +- if (vma->rb_subtree_gap <= sz_range(&second_gap)) { +- rbroot.rb_node = &vma->vm_rb; +- vma = rb_entry(rb_last(&rbroot), +- struct vm_area_struct, vm_rb); ++ /* ++ * Find the two biggest gaps so that first_gap > second_gap > others. ++ * If this is too slow, it can be optimised to examine the maple ++ * tree gaps. ++ */ ++ for_each_vma(vmi, vma) { ++ unsigned long gap; ++ ++ if (!prev) { ++ start = vma->vm_start; + goto next; + } +- +- gap.start = last_vma->vm_end; +- gap.end = vma->vm_start; +- if (sz_range(&gap) > sz_range(&second_gap)) { +- swap(gap, second_gap); +- if (sz_range(&second_gap) > sz_range(&first_gap)) +- swap(second_gap, first_gap); ++ gap = vma->vm_start - prev->vm_end; ++ ++ if (gap > sz_range(&first_gap)) { ++ second_gap = first_gap; ++ first_gap.start = prev->vm_end; ++ first_gap.end = vma->vm_start; ++ } else if (gap > sz_range(&second_gap)) { ++ second_gap.start = prev->vm_end; ++ second_gap.end = vma->vm_start; + } + next: +- last_vma = vma; ++ prev = vma; + } + + if (!sz_range(&second_gap) || !sz_range(&first_gap)) +@@ -159,7 +160,7 @@ static int __damon_va_three_regions(struct vm_area_struct *vma, + regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); + regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); + regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); +- regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION); ++ regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION); + + return 0; + } +@@ -180,7 +181,7 @@ static int damon_va_three_regions(struct damon_target *t, + return -EINVAL; + + mmap_read_lock(mm); +- rc = __damon_va_three_regions(mm->mmap, regions); ++ rc = __damon_va_three_regions(mm, regions); + mmap_read_unlock(mm); + + mmput(mm); +diff --git a/mm/debug.c b/mm/debug.c +index bef329bf28f0..0fd15ba70d16 100644 +--- a/mm/debug.c ++++ b/mm/debug.c +@@ -139,13 +139,11 @@ EXPORT_SYMBOL(dump_page); + + void dump_vma(const struct vm_area_struct *vma) + { +- pr_emerg("vma %px start %px end %px\n" +- "next %px prev %px mm %px\n" ++ pr_emerg("vma %px start %px end %px mm %px\n" + "prot %lx anon_vma %px vm_ops %px\n" + "pgoff %lx file %px private_data %px\n" + "flags: %#lx(%pGv)\n", +- vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_next, +- vma->vm_prev, vma->vm_mm, ++ vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, + (unsigned long)pgprot_val(vma->vm_page_prot), + vma->anon_vma, vma->vm_ops, vma->vm_pgoff, + vma->vm_file, vma->vm_private_data, +@@ -155,11 +153,11 @@ EXPORT_SYMBOL(dump_vma); + + void dump_mm(const struct mm_struct *mm) + { +- pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n" ++ pr_emerg("mm %px task_size %lu\n" + #ifdef CONFIG_MMU + "get_unmapped_area %px\n" + #endif +- "mmap_base %lu mmap_legacy_base %lu highest_vm_end %lu\n" ++ "mmap_base %lu mmap_legacy_base %lu\n" + "pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n" + "hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n" + "pinned_vm %llx data_vm %lx exec_vm %lx stack_vm %lx\n" +@@ -183,11 +181,11 @@ void dump_mm(const struct mm_struct *mm) + "tlb_flush_pending %d\n" + "def_flags: %#lx(%pGv)\n", + +- mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size, ++ mm, mm->task_size, + #ifdef CONFIG_MMU + mm->get_unmapped_area, + #endif +- mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end, ++ mm->mmap_base, mm->mmap_legacy_base, + mm->pgd, atomic_read(&mm->mm_users), + atomic_read(&mm->mm_count), + mm_pgtables_bytes(mm), +diff --git a/mm/gup.c b/mm/gup.c +index 00926abb4426..4da7f1e3bba2 100644 +--- a/mm/gup.c ++++ b/mm/gup.c +@@ -1667,10 +1667,11 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) + if (!locked) { + locked = 1; + mmap_read_lock(mm); +- vma = find_vma(mm, nstart); ++ vma = find_vma_intersection(mm, nstart, end); + } else if (nstart >= vma->vm_end) +- vma = vma->vm_next; +- if (!vma || vma->vm_start >= end) ++ vma = find_vma_intersection(mm, vma->vm_end, end); ++ ++ if (!vma) + break; + /* + * Set [nstart; nend) to intersection of desired address +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 79e0b08b4cf9..a50ab1bf4038 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2334,11 +2334,11 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, + split_huge_pmd_if_needed(vma, end); + + /* +- * If we're also updating the vma->vm_next->vm_start, ++ * If we're also updating the next vma vm_start, + * check if we need to split it. + */ + if (adjust_next > 0) { +- struct vm_area_struct *next = vma->vm_next; ++ struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); + unsigned long nstart = next->vm_start; + nstart += adjust_next; + split_huge_pmd_if_needed(next, nstart); +diff --git a/mm/init-mm.c b/mm/init-mm.c +index fbe7844d0912..c9327abb771c 100644 +--- a/mm/init-mm.c ++++ b/mm/init-mm.c +@@ -1,6 +1,6 @@ + // SPDX-License-Identifier: GPL-2.0 + #include +-#include ++#include + #include + #include + #include +@@ -28,7 +28,7 @@ + * and size this cpu_bitmask to NR_CPUS. + */ + struct mm_struct init_mm = { +- .mm_rb = RB_ROOT, ++ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, init_mm.mmap_lock), + .pgd = swapper_pg_dir, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), +diff --git a/mm/internal.h b/mm/internal.h +index a1fddea6b34f..1adf57d9713e 100644 +--- a/mm/internal.h ++++ b/mm/internal.h +@@ -85,8 +85,9 @@ bool __folio_end_writeback(struct folio *folio); + void deactivate_file_folio(struct folio *folio); + void folio_activate(struct folio *folio); + +-void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, +- unsigned long floor, unsigned long ceiling); ++void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, ++ struct vm_area_struct *start_vma, unsigned long floor, ++ unsigned long ceiling); + void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); + + struct zap_details; +@@ -480,9 +481,6 @@ static inline bool is_data_mapping(vm_flags_t flags) + } + + /* mm/util.c */ +-void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev); +-void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma); + struct anon_vma *folio_anon_vma(struct folio *folio); + + #ifdef CONFIG_MMU +diff --git a/mm/khugepaged.c b/mm/khugepaged.c +index 70b7ac66411c..ef1f78d45e64 100644 +--- a/mm/khugepaged.c ++++ b/mm/khugepaged.c +@@ -1389,7 +1389,7 @@ static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *v + void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) + { + unsigned long haddr = addr & HPAGE_PMD_MASK; +- struct vm_area_struct *vma = find_vma(mm, haddr); ++ struct vm_area_struct *vma = vma_lookup(mm, haddr); + struct page *hpage; + pte_t *start_pte, *pte; + pmd_t *pmd; +@@ -2058,6 +2058,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + __releases(&khugepaged_mm_lock) + __acquires(&khugepaged_mm_lock) + { ++ struct vma_iterator vmi; + struct mm_slot *mm_slot; + struct mm_struct *mm; + struct vm_area_struct *vma; +@@ -2085,11 +2086,13 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + vma = NULL; + if (unlikely(!mmap_read_trylock(mm))) + goto breakouterloop_mmap_lock; +- if (likely(!khugepaged_test_exit(mm))) +- vma = find_vma(mm, khugepaged_scan.address); + + progress++; +- for (; vma; vma = vma->vm_next) { ++ if (unlikely(khugepaged_test_exit(mm))) ++ goto breakouterloop; ++ ++ vma_iter_init(&vmi, mm, khugepaged_scan.address); ++ for_each_vma(vmi, vma) { + unsigned long hstart, hend; + + cond_resched(); +diff --git a/mm/ksm.c b/mm/ksm.c +index 42ab153335a2..63b4b9d71597 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -981,11 +981,13 @@ static int unmerge_and_remove_all_rmap_items(void) + struct mm_slot, mm_list); + spin_unlock(&ksm_mmlist_lock); + +- for (mm_slot = ksm_scan.mm_slot; +- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { ++ for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; ++ mm_slot = ksm_scan.mm_slot) { ++ VMA_ITERATOR(vmi, mm_slot->mm, 0); ++ + mm = mm_slot->mm; + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (ksm_test_exit(mm)) + break; + if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) +@@ -2232,6 +2234,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + struct mm_slot *slot; + struct vm_area_struct *vma; + struct rmap_item *rmap_item; ++ struct vma_iterator vmi; + int nid; + + if (list_empty(&ksm_mm_head.mm_list)) +@@ -2290,13 +2293,13 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + } + + mm = slot->mm; ++ vma_iter_init(&vmi, mm, ksm_scan.address); ++ + mmap_read_lock(mm); + if (ksm_test_exit(mm)) +- vma = NULL; +- else +- vma = find_vma(mm, ksm_scan.address); ++ goto no_vmas; + +- for (; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!(vma->vm_flags & VM_MERGEABLE)) + continue; + if (ksm_scan.address < vma->vm_start) +@@ -2334,6 +2337,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + } + + if (ksm_test_exit(mm)) { ++no_vmas: + ksm_scan.address = 0; + ksm_scan.rmap_list = &slot->rmap_list; + } +diff --git a/mm/madvise.c b/mm/madvise.c +index 9ff51650f4f0..0c872ac11920 100644 +--- a/mm/madvise.c ++++ b/mm/madvise.c +@@ -1241,7 +1241,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, + if (start >= end) + break; + if (prev) +- vma = prev->vm_next; ++ vma = find_vma(mm, prev->vm_end); + else /* madvise_remove dropped mmap_lock */ + vma = find_vma(mm, start); + } +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index 1c18d7c1ce71..fa9714d9afa9 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -5874,7 +5874,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) + unsigned long precharge; + + mmap_read_lock(mm); +- walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL); ++ walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL); + mmap_read_unlock(mm); + + precharge = mc.precharge; +@@ -6172,9 +6172,7 @@ static void mem_cgroup_move_charge(void) + * When we have consumed all precharges and failed in doing + * additional charge, the page walk just aborts. + */ +- walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops, +- NULL); +- ++ walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL); + mmap_read_unlock(mc.mm); + atomic_dec(&mc.from->moving_account); + } +diff --git a/mm/memory.c b/mm/memory.c +index cd1b5bfd9f3e..6a416cac46d3 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -390,12 +390,21 @@ void free_pgd_range(struct mmu_gather *tlb, + } while (pgd++, addr = next, addr != end); + } + +-void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, +- unsigned long floor, unsigned long ceiling) ++void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, ++ struct vm_area_struct *vma, unsigned long floor, ++ unsigned long ceiling) + { +- while (vma) { +- struct vm_area_struct *next = vma->vm_next; ++ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); ++ ++ do { + unsigned long addr = vma->vm_start; ++ struct vm_area_struct *next; ++ ++ /* ++ * Note: USER_PGTABLES_CEILING may be passed as ceiling and may ++ * be 0. This will underflow and is okay. ++ */ ++ next = mas_find(&mas, ceiling - 1); + + /* + * Hide vma from rmap and truncate_pagecache before freeing +@@ -414,7 +423,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, + while (next && next->vm_start <= vma->vm_end + PMD_SIZE + && !is_vm_hugetlb_page(next)) { + vma = next; +- next = vma->vm_next; ++ next = mas_find(&mas, ceiling - 1); + unlink_anon_vmas(vma); + unlink_file_vma(vma); + } +@@ -422,7 +431,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, + floor, next ? next->vm_start : ceiling); + } + vma = next; +- } ++ } while (vma); + } + + void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) +@@ -1701,7 +1710,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, + * ensure that any thus-far unmapped pages are flushed before unmap_vmas() + * drops the lock and schedules. + */ +-void unmap_vmas(struct mmu_gather *tlb, ++void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, + struct vm_area_struct *vma, unsigned long start_addr, + unsigned long end_addr) + { +@@ -1711,12 +1720,14 @@ void unmap_vmas(struct mmu_gather *tlb, + /* Careful - we need to zap private pages too! */ + .even_cows = true, + }; ++ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); + + mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, + start_addr, end_addr); + mmu_notifier_invalidate_range_start(&range); +- for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) ++ do { + unmap_single_vma(tlb, vma, start_addr, end_addr, &details); ++ } while ((vma = mas_find(&mas, end_addr - 1)) != NULL); + mmu_notifier_invalidate_range_end(&range); + } + +@@ -1731,8 +1742,11 @@ void unmap_vmas(struct mmu_gather *tlb, + void zap_page_range(struct vm_area_struct *vma, unsigned long start, + unsigned long size) + { ++ struct maple_tree *mt = &vma->vm_mm->mm_mt; ++ unsigned long end = start + size; + struct mmu_notifier_range range; + struct mmu_gather tlb; ++ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); + + lru_add_drain(); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, +@@ -1740,8 +1754,9 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, + tlb_gather_mmu(&tlb, vma->vm_mm); + update_hiwater_rss(vma->vm_mm); + mmu_notifier_invalidate_range_start(&range); +- for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) ++ do { + unmap_single_vma(&tlb, vma, start, range.end, NULL); ++ } while ((vma = mas_find(&mas, end - 1)) != NULL); + mmu_notifier_invalidate_range_end(&range); + tlb_finish_mmu(&tlb); + } +diff --git a/mm/mempolicy.c b/mm/mempolicy.c +index b73d3248d976..6c27acb6cd63 100644 +--- a/mm/mempolicy.c ++++ b/mm/mempolicy.c +@@ -381,9 +381,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) + void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_write_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) ++ for_each_vma(vmi, vma) + mpol_rebind_policy(vma->vm_policy, new); + mmap_write_unlock(mm); + } +@@ -654,7 +655,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma, + static int queue_pages_test_walk(unsigned long start, unsigned long end, + struct mm_walk *walk) + { +- struct vm_area_struct *vma = walk->vma; ++ struct vm_area_struct *next, *vma = walk->vma; + struct queue_pages *qp = walk->private; + unsigned long endvma = vma->vm_end; + unsigned long flags = qp->flags; +@@ -669,9 +670,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, + /* hole at head side of range */ + return -EFAULT; + } ++ next = find_vma(vma->vm_mm, vma->vm_end); + if (!(flags & MPOL_MF_DISCONTIG_OK) && + ((vma->vm_end < qp->end) && +- (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) ++ (!next || vma->vm_end < next->vm_start))) + /* hole at middle or tail of range */ + return -EFAULT; + +@@ -785,26 +787,24 @@ static int vma_replace_policy(struct vm_area_struct *vma, + static int mbind_range(struct mm_struct *mm, unsigned long start, + unsigned long end, struct mempolicy *new_pol) + { ++ MA_STATE(mas, &mm->mm_mt, start - 1, start - 1); + struct vm_area_struct *prev; + struct vm_area_struct *vma; + int err = 0; + pgoff_t pgoff; +- unsigned long vmstart; +- unsigned long vmend; +- +- vma = find_vma(mm, start); +- VM_BUG_ON(!vma); + +- prev = vma->vm_prev; +- if (start > vma->vm_start) +- prev = vma; ++ prev = mas_find_rev(&mas, 0); ++ if (prev && (start < prev->vm_end)) ++ vma = prev; ++ else ++ vma = mas_next(&mas, end - 1); + +- for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) { +- vmstart = max(start, vma->vm_start); +- vmend = min(end, vma->vm_end); ++ for (; vma; vma = mas_next(&mas, end - 1)) { ++ unsigned long vmstart = max(start, vma->vm_start); ++ unsigned long vmend = min(end, vma->vm_end); + + if (mpol_equal(vma_policy(vma), new_pol)) +- continue; ++ goto next; + + pgoff = vma->vm_pgoff + + ((vmstart - vma->vm_start) >> PAGE_SHIFT); +@@ -813,6 +813,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, + new_pol, vma->vm_userfaultfd_ctx, + anon_vma_name(vma)); + if (prev) { ++ /* vma_merge() invalidated the mas */ ++ mas_pause(&mas); + vma = prev; + goto replace; + } +@@ -820,19 +822,25 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, + err = split_vma(vma->vm_mm, vma, vmstart, 1); + if (err) + goto out; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } + if (vma->vm_end != vmend) { + err = split_vma(vma->vm_mm, vma, vmend, 0); + if (err) + goto out; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } +- replace: ++replace: + err = vma_replace_policy(vma, new_pol); + if (err) + goto out; ++next: ++ prev = vma; + } + +- out: ++out: + return err; + } + +@@ -1047,6 +1055,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, + int flags) + { + nodemask_t nmask; ++ struct vm_area_struct *vma; + LIST_HEAD(pagelist); + int err = 0; + struct migration_target_control mtc = { +@@ -1062,8 +1071,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, + * need migration. Between passing in the full user address + * space range and MPOL_MF_DISCONTIG_OK, this call can not fail. + */ ++ vma = find_vma(mm, 0); + VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))); +- queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, ++ queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask, + flags | MPOL_MF_DISCONTIG_OK, &pagelist); + + if (!list_empty(&pagelist)) { +@@ -1193,14 +1203,13 @@ static struct page *new_page(struct page *page, unsigned long start) + struct folio *dst, *src = page_folio(page); + struct vm_area_struct *vma; + unsigned long address; ++ VMA_ITERATOR(vmi, current->mm, start); + gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL; + +- vma = find_vma(current->mm, start); +- while (vma) { ++ for_each_vma(vmi, vma) { + address = page_address_in_vma(page, vma); + if (address != -EFAULT) + break; +- vma = vma->vm_next; + } + + if (folio_test_hugetlb(src)) +@@ -1478,6 +1487,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le + unsigned long vmend; + unsigned long end; + int err = -ENOENT; ++ VMA_ITERATOR(vmi, mm, start); + + start = untagged_addr(start); + if (start & ~PAGE_MASK) +@@ -1503,9 +1513,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le + if (end == start) + return 0; + mmap_write_lock(mm); +- vma = find_vma(mm, start); +- for (; vma && vma->vm_start < end; vma = vma->vm_next) { +- ++ for_each_vma_range(vmi, vma, end) { + vmstart = max(start, vma->vm_start); + vmend = min(end, vma->vm_end); + new = mpol_dup(vma_policy(vma)); +diff --git a/mm/mlock.c b/mm/mlock.c +index b14e929084cc..43d19a1f28eb 100644 +--- a/mm/mlock.c ++++ b/mm/mlock.c +@@ -471,6 +471,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, + unsigned long nstart, end, tmp; + struct vm_area_struct *vma, *prev; + int error; ++ MA_STATE(mas, ¤t->mm->mm_mt, start, start); + + VM_BUG_ON(offset_in_page(start)); + VM_BUG_ON(len != PAGE_ALIGN(len)); +@@ -479,13 +480,14 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, + return -EINVAL; + if (end == start) + return 0; +- vma = find_vma(current->mm, start); +- if (!vma || vma->vm_start > start) ++ vma = mas_walk(&mas); ++ if (!vma) + return -ENOMEM; + +- prev = vma->vm_prev; + if (start > vma->vm_start) + prev = vma; ++ else ++ prev = mas_prev(&mas, 0); + + for (nstart = start ; ; ) { + vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; +@@ -505,7 +507,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, + if (nstart >= end) + break; + +- vma = prev->vm_next; ++ vma = find_vma(prev->vm_mm, prev->vm_end); + if (!vma || vma->vm_start != nstart) { + error = -ENOMEM; + break; +@@ -526,24 +528,23 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, + { + struct vm_area_struct *vma; + unsigned long count = 0; ++ unsigned long end; ++ VMA_ITERATOR(vmi, mm, start); + + if (mm == NULL) + mm = current->mm; + +- vma = find_vma(mm, start); +- if (vma == NULL) +- return 0; +- +- for (; vma ; vma = vma->vm_next) { +- if (start >= vma->vm_end) +- continue; +- if (start + len <= vma->vm_start) +- break; ++ /* Don't overflow past ULONG_MAX */ ++ if (unlikely(ULONG_MAX - len < start)) ++ end = ULONG_MAX; ++ else ++ end = start + len; ++ for_each_vma_range(vmi, vma, end) { + if (vma->vm_flags & VM_LOCKED) { + if (start > vma->vm_start) + count -= (start - vma->vm_start); +- if (start + len < vma->vm_end) { +- count += start + len - vma->vm_start; ++ if (end < vma->vm_end) { ++ count += end - vma->vm_start; + break; + } + count += vma->vm_end - vma->vm_start; +@@ -659,6 +660,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) + */ + static int apply_mlockall_flags(int flags) + { ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + struct vm_area_struct *vma, *prev = NULL; + vm_flags_t to_add = 0; + +@@ -679,7 +681,7 @@ static int apply_mlockall_flags(int flags) + to_add |= VM_LOCKONFAULT; + } + +- for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + vm_flags_t newflags; + + newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; +@@ -687,6 +689,7 @@ static int apply_mlockall_flags(int flags) + + /* Ignore errors */ + mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); ++ mas_pause(&mas); + cond_resched(); + } + out: +diff --git a/mm/mmap.c b/mm/mmap.c +index 9d780f415be3..6fc054c55cd8 100644 +--- a/mm/mmap.c ++++ b/mm/mmap.c +@@ -14,7 +14,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -39,7 +38,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -77,9 +75,10 @@ int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; + static bool ignore_rlimit_data; + core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); + +-static void unmap_region(struct mm_struct *mm, ++static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, + struct vm_area_struct *vma, struct vm_area_struct *prev, +- unsigned long start, unsigned long end); ++ struct vm_area_struct *next, unsigned long start, ++ unsigned long end); + + static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) + { +@@ -132,12 +131,10 @@ void unlink_file_vma(struct vm_area_struct *vma) + } + + /* +- * Close a vm structure and free it, returning the next. ++ * Close a vm structure and free it. + */ +-static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) ++static void remove_vma(struct vm_area_struct *vma) + { +- struct vm_area_struct *next = vma->vm_next; +- + might_sleep(); + if (vma->vm_ops && vma->vm_ops->close) + vma->vm_ops->close(vma); +@@ -145,20 +142,41 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) + fput(vma->vm_file); + mpol_put(vma_policy(vma)); + vm_area_free(vma); +- return next; + } + +-static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, +- struct list_head *uf); ++/* ++ * check_brk_limits() - Use platform specific check of range & verify mlock ++ * limits. ++ * @addr: The address to check ++ * @len: The size of increase. ++ * ++ * Return: 0 on success. ++ */ ++static int check_brk_limits(unsigned long addr, unsigned long len) ++{ ++ unsigned long mapped_addr; ++ ++ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); ++ if (IS_ERR_VALUE(mapped_addr)) ++ return mapped_addr; ++ ++ return mlock_future_check(current->mm, current->mm->def_flags, len); ++} ++static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long newbrk, unsigned long oldbrk, ++ struct list_head *uf); ++static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma, ++ unsigned long addr, unsigned long request, unsigned long flags); + SYSCALL_DEFINE1(brk, unsigned long, brk) + { + unsigned long newbrk, oldbrk, origbrk; + struct mm_struct *mm = current->mm; +- struct vm_area_struct *next; ++ struct vm_area_struct *brkvma, *next = NULL; + unsigned long min_brk; + bool populate; + bool downgraded = false; + LIST_HEAD(uf); ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + if (mmap_write_lock_killable(mm)) + return -EINTR; +@@ -200,35 +218,51 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) + + /* + * Always allow shrinking brk. +- * __do_munmap() may downgrade mmap_lock to read. ++ * do_brk_munmap() may downgrade mmap_lock to read. + */ + if (brk <= mm->brk) { + int ret; + ++ /* Search one past newbrk */ ++ mas_set(&mas, newbrk); ++ brkvma = mas_find(&mas, oldbrk); ++ BUG_ON(brkvma == NULL); ++ if (brkvma->vm_start >= oldbrk) ++ goto out; /* mapping intersects with an existing non-brk vma. */ + /* +- * mm->brk must to be protected by write mmap_lock so update it +- * before downgrading mmap_lock. When __do_munmap() fails, +- * mm->brk will be restored from origbrk. ++ * mm->brk must be protected by write mmap_lock. ++ * do_brk_munmap() may downgrade the lock, so update it ++ * before calling do_brk_munmap(). + */ + mm->brk = brk; +- ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true); +- if (ret < 0) { +- mm->brk = origbrk; +- goto out; +- } else if (ret == 1) { ++ ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf); ++ if (ret == 1) { + downgraded = true; +- } +- goto success; ++ goto success; ++ } else if (!ret) ++ goto success; ++ ++ mm->brk = origbrk; ++ goto out; + } + +- /* Check against existing mmap mappings. */ +- next = find_vma(mm, oldbrk); ++ if (check_brk_limits(oldbrk, newbrk - oldbrk)) ++ goto out; ++ ++ /* ++ * Only check if the next VMA is within the stack_guard_gap of the ++ * expansion area ++ */ ++ mas_set(&mas, oldbrk); ++ next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap); + if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + goto out; + ++ brkvma = mas_prev(&mas, mm->start_brk); + /* Ok, looks good - let it rip. */ +- if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) ++ if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) + goto out; ++ + mm->brk = brk; + + success: +@@ -247,104 +281,45 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) + return origbrk; + } + +-static inline unsigned long vma_compute_gap(struct vm_area_struct *vma) +-{ +- unsigned long gap, prev_end; +- +- /* +- * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we +- * allow two stack_guard_gaps between them here, and when choosing +- * an unmapped area; whereas when expanding we only require one. +- * That's a little inconsistent, but keeps the code here simpler. +- */ +- gap = vm_start_gap(vma); +- if (vma->vm_prev) { +- prev_end = vm_end_gap(vma->vm_prev); +- if (gap > prev_end) +- gap -= prev_end; +- else +- gap = 0; +- } +- return gap; +-} +- +-#ifdef CONFIG_DEBUG_VM_RB +-static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma) +-{ +- unsigned long max = vma_compute_gap(vma), subtree_gap; +- if (vma->vm_rb.rb_left) { +- subtree_gap = rb_entry(vma->vm_rb.rb_left, +- struct vm_area_struct, vm_rb)->rb_subtree_gap; +- if (subtree_gap > max) +- max = subtree_gap; +- } +- if (vma->vm_rb.rb_right) { +- subtree_gap = rb_entry(vma->vm_rb.rb_right, +- struct vm_area_struct, vm_rb)->rb_subtree_gap; +- if (subtree_gap > max) +- max = subtree_gap; +- } +- return max; +-} +- +-static int browse_rb(struct mm_struct *mm) +-{ +- struct rb_root *root = &mm->mm_rb; +- int i = 0, j, bug = 0; +- struct rb_node *nd, *pn = NULL; +- unsigned long prev = 0, pend = 0; +- +- for (nd = rb_first(root); nd; nd = rb_next(nd)) { +- struct vm_area_struct *vma; +- vma = rb_entry(nd, struct vm_area_struct, vm_rb); +- if (vma->vm_start < prev) { +- pr_emerg("vm_start %lx < prev %lx\n", +- vma->vm_start, prev); +- bug = 1; +- } +- if (vma->vm_start < pend) { +- pr_emerg("vm_start %lx < pend %lx\n", +- vma->vm_start, pend); +- bug = 1; +- } +- if (vma->vm_start > vma->vm_end) { +- pr_emerg("vm_start %lx > vm_end %lx\n", +- vma->vm_start, vma->vm_end); +- bug = 1; +- } +- spin_lock(&mm->page_table_lock); +- if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) { +- pr_emerg("free gap %lx, correct %lx\n", +- vma->rb_subtree_gap, +- vma_compute_subtree_gap(vma)); +- bug = 1; ++#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) ++extern void mt_validate(struct maple_tree *mt); ++extern void mt_dump(const struct maple_tree *mt); ++ ++/* Validate the maple tree */ ++static void validate_mm_mt(struct mm_struct *mm) ++{ ++ struct maple_tree *mt = &mm->mm_mt; ++ struct vm_area_struct *vma_mt; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ mt_validate(&mm->mm_mt); ++ mas_for_each(&mas, vma_mt, ULONG_MAX) { ++ if ((vma_mt->vm_start != mas.index) || ++ (vma_mt->vm_end - 1 != mas.last)) { ++ pr_emerg("issue in %s\n", current->comm); ++ dump_stack(); ++ dump_vma(vma_mt); ++ pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, ++ mas.index, mas.last); ++ pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, ++ vma_mt->vm_start, vma_mt->vm_end); ++ ++ mt_dump(mas.tree); ++ if (vma_mt->vm_end != mas.last + 1) { ++ pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", ++ mm, vma_mt->vm_start, vma_mt->vm_end, ++ mas.index, mas.last); ++ mt_dump(mas.tree); ++ } ++ VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); ++ if (vma_mt->vm_start != mas.index) { ++ pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", ++ mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); ++ mt_dump(mas.tree); ++ } ++ VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); + } +- spin_unlock(&mm->page_table_lock); +- i++; +- pn = nd; +- prev = vma->vm_start; +- pend = vma->vm_end; +- } +- j = 0; +- for (nd = pn; nd; nd = rb_prev(nd)) +- j++; +- if (i != j) { +- pr_emerg("backwards %d, forwards %d\n", j, i); +- bug = 1; +- } +- return bug ? -1 : i; +-} +- +-static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore) +-{ +- struct rb_node *nd; +- +- for (nd = rb_first(root); nd; nd = rb_next(nd)) { +- struct vm_area_struct *vma; +- vma = rb_entry(nd, struct vm_area_struct, vm_rb); +- VM_BUG_ON_VMA(vma != ignore && +- vma->rb_subtree_gap != vma_compute_subtree_gap(vma), +- vma); + } + } + +@@ -352,10 +327,13 @@ static void validate_mm(struct mm_struct *mm) + { + int bug = 0; + int i = 0; +- unsigned long highest_address = 0; +- struct vm_area_struct *vma = mm->mmap; ++ struct vm_area_struct *vma; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + +- while (vma) { ++ validate_mm_mt(mm); ++ ++ mas_for_each(&mas, vma, ULONG_MAX) { ++#ifdef CONFIG_DEBUG_VM_RB + struct anon_vma *anon_vma = vma->anon_vma; + struct anon_vma_chain *avc; + +@@ -365,93 +343,20 @@ static void validate_mm(struct mm_struct *mm) + anon_vma_interval_tree_verify(avc); + anon_vma_unlock_read(anon_vma); + } +- +- highest_address = vm_end_gap(vma); +- vma = vma->vm_next; ++#endif + i++; + } + if (i != mm->map_count) { +- pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); +- bug = 1; +- } +- if (highest_address != mm->highest_vm_end) { +- pr_emerg("mm->highest_vm_end %lx, found %lx\n", +- mm->highest_vm_end, highest_address); +- bug = 1; +- } +- i = browse_rb(mm); +- if (i != mm->map_count) { +- if (i != -1) +- pr_emerg("map_count %d rb %d\n", mm->map_count, i); ++ pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i); + bug = 1; + } + VM_BUG_ON_MM(bug, mm); + } +-#else +-#define validate_mm_rb(root, ignore) do { } while (0) +-#define validate_mm(mm) do { } while (0) +-#endif +- +-RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks, +- struct vm_area_struct, vm_rb, +- unsigned long, rb_subtree_gap, vma_compute_gap) +- +-/* +- * Update augmented rbtree rb_subtree_gap values after vma->vm_start or +- * vma->vm_prev->vm_end values changed, without modifying the vma's position +- * in the rbtree. +- */ +-static void vma_gap_update(struct vm_area_struct *vma) +-{ +- /* +- * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created +- * a callback function that does exactly what we want. +- */ +- vma_gap_callbacks_propagate(&vma->vm_rb, NULL); +-} +- +-static inline void vma_rb_insert(struct vm_area_struct *vma, +- struct rb_root *root) +-{ +- /* All rb_subtree_gap values must be consistent prior to insertion */ +- validate_mm_rb(root, NULL); +- +- rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks); +-} +- +-static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) +-{ +- /* +- * Note rb_erase_augmented is a fairly large inline function, +- * so make sure we instantiate it only once with our desired +- * augmented rbtree callbacks. +- */ +- rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); +-} +- +-static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, +- struct rb_root *root, +- struct vm_area_struct *ignore) +-{ +- /* +- * All rb_subtree_gap values must be consistent prior to erase, +- * with the possible exception of +- * +- * a. the "next" vma being erased if next->vm_start was reduced in +- * __vma_adjust() -> __vma_unlink() +- * b. the vma being erased in detach_vmas_to_be_unmapped() -> +- * vma_rb_erase() +- */ +- validate_mm_rb(root, ignore); +- +- __vma_rb_erase(vma, root); +-} + +-static __always_inline void vma_rb_erase(struct vm_area_struct *vma, +- struct rb_root *root) +-{ +- vma_rb_erase_ignore(vma, root, vma); +-} ++#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ ++#define validate_mm_mt(root) do { } while (0) ++#define validate_mm(mm) do { } while (0) ++#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ + + /* + * vma has some anon_vma assigned, and is already inserted on that +@@ -485,208 +390,220 @@ anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + } + +-static int find_vma_links(struct mm_struct *mm, unsigned long addr, +- unsigned long end, struct vm_area_struct **pprev, +- struct rb_node ***rb_link, struct rb_node **rb_parent) ++static unsigned long count_vma_pages_range(struct mm_struct *mm, ++ unsigned long addr, unsigned long end) + { +- struct rb_node **__rb_link, *__rb_parent, *rb_prev; ++ VMA_ITERATOR(vmi, mm, addr); ++ struct vm_area_struct *vma; ++ unsigned long nr_pages = 0; + +- mmap_assert_locked(mm); +- __rb_link = &mm->mm_rb.rb_node; +- rb_prev = __rb_parent = NULL; ++ for_each_vma_range(vmi, vma, end) { ++ unsigned long vm_start = max(addr, vma->vm_start); ++ unsigned long vm_end = min(end, vma->vm_end); + +- while (*__rb_link) { +- struct vm_area_struct *vma_tmp; ++ nr_pages += PHYS_PFN(vm_end - vm_start); ++ } + +- __rb_parent = *__rb_link; +- vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb); ++ return nr_pages; ++} + +- if (vma_tmp->vm_end > addr) { +- /* Fail if an existing vma overlaps the area */ +- if (vma_tmp->vm_start < end) +- return -ENOMEM; +- __rb_link = &__rb_parent->rb_left; +- } else { +- rb_prev = __rb_parent; +- __rb_link = &__rb_parent->rb_right; +- } +- } ++static void __vma_link_file(struct vm_area_struct *vma, ++ struct address_space *mapping) ++{ ++ if (vma->vm_flags & VM_SHARED) ++ mapping_allow_writable(mapping); + +- *pprev = NULL; +- if (rb_prev) +- *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); +- *rb_link = __rb_link; +- *rb_parent = __rb_parent; +- return 0; ++ flush_dcache_mmap_lock(mapping); ++ vma_interval_tree_insert(vma, &mapping->i_mmap); ++ flush_dcache_mmap_unlock(mapping); + } + + /* +- * vma_next() - Get the next VMA. +- * @mm: The mm_struct. +- * @vma: The current vma. ++ * vma_mas_store() - Store a VMA in the maple tree. ++ * @vma: The vm_area_struct ++ * @mas: The maple state + * +- * If @vma is NULL, return the first vma in the mm. ++ * Efficient way to store a VMA in the maple tree when the @mas has already ++ * walked to the correct location. + * +- * Returns: The next VMA after @vma. ++ * Note: the end address is inclusive in the maple tree. + */ +-static inline struct vm_area_struct *vma_next(struct mm_struct *mm, +- struct vm_area_struct *vma) ++void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) + { +- if (!vma) +- return mm->mmap; +- +- return vma->vm_next; ++ trace_vma_store(mas->tree, vma); ++ mas_set_range(mas, vma->vm_start, vma->vm_end - 1); ++ mas_store_prealloc(mas, vma); + } + + /* +- * munmap_vma_range() - munmap VMAs that overlap a range. +- * @mm: The mm struct +- * @start: The start of the range. +- * @len: The length of the range. +- * @pprev: pointer to the pointer that will be set to previous vm_area_struct +- * @rb_link: the rb_node +- * @rb_parent: the parent rb_node +- * +- * Find all the vm_area_struct that overlap from @start to +- * @end and munmap them. Set @pprev to the previous vm_area_struct. ++ * vma_mas_remove() - Remove a VMA from the maple tree. ++ * @vma: The vm_area_struct ++ * @mas: The maple state + * +- * Returns: -ENOMEM on munmap failure or 0 on success. ++ * Efficient way to remove a VMA from the maple tree when the @mas has already ++ * been established and points to the correct location. ++ * Note: the end address is inclusive in the maple tree. + */ +-static inline int +-munmap_vma_range(struct mm_struct *mm, unsigned long start, unsigned long len, +- struct vm_area_struct **pprev, struct rb_node ***link, +- struct rb_node **parent, struct list_head *uf) ++void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) + { +- +- while (find_vma_links(mm, start, start + len, pprev, link, parent)) +- if (do_munmap(mm, start, len, uf)) +- return -ENOMEM; +- +- return 0; ++ trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); ++ mas->index = vma->vm_start; ++ mas->last = vma->vm_end - 1; ++ mas_store_prealloc(mas, NULL); + } +-static unsigned long count_vma_pages_range(struct mm_struct *mm, +- unsigned long addr, unsigned long end) ++ ++/* ++ * vma_mas_szero() - Set a given range to zero. Used when modifying a ++ * vm_area_struct start or end. ++ * ++ * @mm: The struct_mm ++ * @start: The start address to zero ++ * @end: The end address to zero. ++ */ ++static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, ++ unsigned long end) + { +- unsigned long nr_pages = 0; +- struct vm_area_struct *vma; ++ trace_vma_mas_szero(mas->tree, start, end - 1); ++ mas_set_range(mas, start, end - 1); ++ mas_store_prealloc(mas, NULL); ++} + +- /* Find first overlapping mapping */ +- vma = find_vma_intersection(mm, addr, end); +- if (!vma) +- return 0; ++static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) ++{ ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ struct address_space *mapping = NULL; + +- nr_pages = (min(end, vma->vm_end) - +- max(addr, vma->vm_start)) >> PAGE_SHIFT; ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; + +- /* Iterate over the rest of the overlaps */ +- for (vma = vma->vm_next; vma; vma = vma->vm_next) { +- unsigned long overlap_len; ++ if (vma->vm_file) { ++ mapping = vma->vm_file->f_mapping; ++ i_mmap_lock_write(mapping); ++ } + +- if (vma->vm_start > end) +- break; ++ vma_mas_store(vma, &mas); + +- overlap_len = min(end, vma->vm_end) - vma->vm_start; +- nr_pages += overlap_len >> PAGE_SHIFT; ++ if (mapping) { ++ __vma_link_file(vma, mapping); ++ i_mmap_unlock_write(mapping); + } + +- return nr_pages; ++ mm->map_count++; ++ validate_mm(mm); ++ return 0; + } + +-void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, +- struct rb_node **rb_link, struct rb_node *rb_parent) ++/* ++ * vma_expand - Expand an existing VMA ++ * ++ * @mas: The maple state ++ * @vma: The vma to expand ++ * @start: The start of the vma ++ * @end: The exclusive end of the vma ++ * @pgoff: The page offset of vma ++ * @next: The current of next vma. ++ * ++ * Expand @vma to @start and @end. Can expand off the start and end. Will ++ * expand over @next if it's different from @vma and @end == @next->vm_end. ++ * Checking if the @vma can expand and merge with @next needs to be handled by ++ * the caller. ++ * ++ * Returns: 0 on success ++ */ ++inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long start, unsigned long end, pgoff_t pgoff, ++ struct vm_area_struct *next) + { +- /* Update tracking information for the gap following the new vma. */ +- if (vma->vm_next) +- vma_gap_update(vma->vm_next); +- else +- mm->highest_vm_end = vm_end_gap(vma); ++ struct mm_struct *mm = vma->vm_mm; ++ struct address_space *mapping = NULL; ++ struct rb_root_cached *root = NULL; ++ struct anon_vma *anon_vma = vma->anon_vma; ++ struct file *file = vma->vm_file; ++ bool remove_next = false; + +- /* +- * vma->vm_prev wasn't known when we followed the rbtree to find the +- * correct insertion point for that vma. As a result, we could not +- * update the vma vm_rb parents rb_subtree_gap values on the way down. +- * So, we first insert the vma with a zero rb_subtree_gap value +- * (to be consistent with what we did on the way down), and then +- * immediately update the gap to the correct value. Finally we +- * rebalance the rbtree after all augmented values have been set. +- */ +- rb_link_node(&vma->vm_rb, rb_parent, rb_link); +- vma->rb_subtree_gap = 0; +- vma_gap_update(vma); +- vma_rb_insert(vma, &mm->mm_rb); +-} ++ if (next && (vma != next) && (end == next->vm_end)) { ++ remove_next = true; ++ if (next->anon_vma && !vma->anon_vma) { ++ int error; + +-static void __vma_link_file(struct vm_area_struct *vma) +-{ +- struct file *file; ++ anon_vma = next->anon_vma; ++ vma->anon_vma = anon_vma; ++ error = anon_vma_clone(vma, next); ++ if (error) ++ return error; ++ } ++ } ++ ++ /* Not merging but overwriting any part of next is not handled. */ ++ VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start); ++ /* Only handles expanding */ ++ VM_BUG_ON(vma->vm_start < start || vma->vm_end > end); ++ ++ if (mas_preallocate(mas, vma, GFP_KERNEL)) ++ goto nomem; ++ ++ vma_adjust_trans_huge(vma, start, end, 0); + +- file = vma->vm_file; + if (file) { +- struct address_space *mapping = file->f_mapping; ++ mapping = file->f_mapping; ++ root = &mapping->i_mmap; ++ uprobe_munmap(vma, vma->vm_start, vma->vm_end); ++ i_mmap_lock_write(mapping); ++ } + +- if (vma->vm_flags & VM_SHARED) +- mapping_allow_writable(mapping); ++ if (anon_vma) { ++ anon_vma_lock_write(anon_vma); ++ anon_vma_interval_tree_pre_update_vma(vma); ++ } + ++ if (file) { + flush_dcache_mmap_lock(mapping); +- vma_interval_tree_insert(vma, &mapping->i_mmap); +- flush_dcache_mmap_unlock(mapping); ++ vma_interval_tree_remove(vma, root); + } +-} + +-static void +-__vma_link(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev, struct rb_node **rb_link, +- struct rb_node *rb_parent) +-{ +- __vma_link_list(mm, vma, prev); +- __vma_link_rb(mm, vma, rb_link, rb_parent); +-} ++ vma->vm_start = start; ++ vma->vm_end = end; ++ vma->vm_pgoff = pgoff; ++ /* Note: mas must be pointing to the expanding VMA */ ++ vma_mas_store(vma, mas); + +-static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev, struct rb_node **rb_link, +- struct rb_node *rb_parent) +-{ +- struct address_space *mapping = NULL; ++ if (file) { ++ vma_interval_tree_insert(vma, root); ++ flush_dcache_mmap_unlock(mapping); ++ } + +- if (vma->vm_file) { +- mapping = vma->vm_file->f_mapping; +- i_mmap_lock_write(mapping); ++ /* Expanding over the next vma */ ++ if (remove_next && file) { ++ __remove_shared_vm_struct(next, file, mapping); + } + +- __vma_link(mm, vma, prev, rb_link, rb_parent); +- __vma_link_file(vma); ++ if (anon_vma) { ++ anon_vma_interval_tree_post_update_vma(vma); ++ anon_vma_unlock_write(anon_vma); ++ } + +- if (mapping) ++ if (file) { + i_mmap_unlock_write(mapping); ++ uprobe_mmap(vma); ++ } + +- mm->map_count++; +- validate_mm(mm); +-} +- +-/* +- * Helper for vma_adjust() in the split_vma insert case: insert a vma into the +- * mm's list and rbtree. It has already been inserted into the interval tree. +- */ +-static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) +-{ +- struct vm_area_struct *prev; +- struct rb_node **rb_link, *rb_parent; ++ if (remove_next) { ++ if (file) { ++ uprobe_munmap(next, next->vm_start, next->vm_end); ++ fput(file); ++ } ++ if (next->anon_vma) ++ anon_vma_merge(vma, next); ++ mm->map_count--; ++ mpol_put(vma_policy(next)); ++ vm_area_free(next); ++ } + +- if (find_vma_links(mm, vma->vm_start, vma->vm_end, +- &prev, &rb_link, &rb_parent)) +- BUG(); +- __vma_link(mm, vma, prev, rb_link, rb_parent); +- mm->map_count++; +-} ++ validate_mm(mm); ++ return 0; + +-static __always_inline void __vma_unlink(struct mm_struct *mm, +- struct vm_area_struct *vma, +- struct vm_area_struct *ignore) +-{ +- vma_rb_erase_ignore(vma, &mm->mm_rb, ignore); +- __vma_unlink_list(mm, vma); +- /* Kill the cache */ +- vmacache_invalidate(mm); ++nomem: ++ return -ENOMEM; + } + + /* +@@ -701,18 +618,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + struct vm_area_struct *expand) + { + struct mm_struct *mm = vma->vm_mm; +- struct vm_area_struct *next = vma->vm_next, *orig_vma = vma; ++ struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); ++ struct vm_area_struct *orig_vma = vma; + struct address_space *mapping = NULL; + struct rb_root_cached *root = NULL; + struct anon_vma *anon_vma = NULL; + struct file *file = vma->vm_file; +- bool start_changed = false, end_changed = false; ++ bool vma_changed = false; + long adjust_next = 0; + int remove_next = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ struct vm_area_struct *exporter = NULL, *importer = NULL; + + if (next && !insert) { +- struct vm_area_struct *exporter = NULL, *importer = NULL; +- + if (end >= next->vm_end) { + /* + * vma expands, overlapping all the next, and +@@ -741,10 +659,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + * remove_next == 1 is case 1 or 7. + */ + remove_next = 1 + (end > next->vm_end); ++ if (remove_next == 2) ++ next_next = find_vma(mm, next->vm_end); ++ + VM_WARN_ON(remove_next == 2 && +- end != next->vm_next->vm_end); +- /* trim end to next, for case 6 first pass */ +- end = next->vm_end; ++ end != next_next->vm_end); + } + + exporter = next; +@@ -755,7 +674,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + * next, if the vma overlaps with it. + */ + if (remove_next == 2 && !next->anon_vma) +- exporter = next->vm_next; ++ exporter = next_next; + + } else if (end > next->vm_start) { + /* +@@ -792,9 +711,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + return error; + } + } +-again: +- vma_adjust_trans_huge(orig_vma, start, end, adjust_next); + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; ++ ++ vma_adjust_trans_huge(orig_vma, start, end, adjust_next); + if (file) { + mapping = file->f_mapping; + root = &mapping->i_mmap; +@@ -804,14 +725,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + uprobe_munmap(next, next->vm_start, next->vm_end); + + i_mmap_lock_write(mapping); +- if (insert) { ++ if (insert && insert->vm_file) { + /* + * Put into interval tree now, so instantiated pages + * are visible to arm/parisc __flush_dcache_page + * throughout; but we cannot insert into address + * space until vma start or end is updated. + */ +- __vma_link_file(insert); ++ __vma_link_file(insert, insert->vm_file->f_mapping); + } + } + +@@ -835,17 +756,37 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + } + + if (start != vma->vm_start) { ++ if ((vma->vm_start < start) && ++ (!insert || (insert->vm_end != start))) { ++ vma_mas_szero(&mas, vma->vm_start, start); ++ VM_WARN_ON(insert && insert->vm_start > vma->vm_start); ++ } else { ++ vma_changed = true; ++ } + vma->vm_start = start; +- start_changed = true; + } + if (end != vma->vm_end) { ++ if (vma->vm_end > end) { ++ if (!insert || (insert->vm_start != end)) { ++ vma_mas_szero(&mas, end, vma->vm_end); ++ mas_reset(&mas); ++ VM_WARN_ON(insert && ++ insert->vm_end < vma->vm_end); ++ } ++ } else { ++ vma_changed = true; ++ } + vma->vm_end = end; +- end_changed = true; + } ++ ++ if (vma_changed) ++ vma_mas_store(vma, &mas); ++ + vma->vm_pgoff = pgoff; + if (adjust_next) { + next->vm_start += adjust_next; + next->vm_pgoff += adjust_next >> PAGE_SHIFT; ++ vma_mas_store(next, &mas); + } + + if (file) { +@@ -855,42 +796,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + flush_dcache_mmap_unlock(mapping); + } + +- if (remove_next) { +- /* +- * vma_merge has merged next into vma, and needs +- * us to remove next before dropping the locks. +- */ +- if (remove_next != 3) +- __vma_unlink(mm, next, next); +- else +- /* +- * vma is not before next if they've been +- * swapped. +- * +- * pre-swap() next->vm_start was reduced so +- * tell validate_mm_rb to ignore pre-swap() +- * "next" (which is stored in post-swap() +- * "vma"). +- */ +- __vma_unlink(mm, next, vma); +- if (file) +- __remove_shared_vm_struct(next, file, mapping); ++ if (remove_next && file) { ++ __remove_shared_vm_struct(next, file, mapping); ++ if (remove_next == 2) ++ __remove_shared_vm_struct(next_next, file, mapping); + } else if (insert) { + /* + * split_vma has split insert from vma, and needs + * us to insert it before dropping the locks + * (it may either follow vma or precede it). + */ +- __insert_vm_struct(mm, insert); +- } else { +- if (start_changed) +- vma_gap_update(vma); +- if (end_changed) { +- if (!next) +- mm->highest_vm_end = vm_end_gap(vma); +- else if (!adjust_next) +- vma_gap_update(next); +- } ++ mas_reset(&mas); ++ vma_mas_store(insert, &mas); ++ mm->map_count++; + } + + if (anon_vma) { +@@ -909,6 +827,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + } + + if (remove_next) { ++again: + if (file) { + uprobe_munmap(next, next->vm_start, next->vm_end); + fput(file); +@@ -917,66 +836,24 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + anon_vma_merge(vma, next); + mm->map_count--; + mpol_put(vma_policy(next)); ++ if (remove_next != 2) ++ BUG_ON(vma->vm_end < next->vm_end); + vm_area_free(next); ++ + /* + * In mprotect's case 6 (see comments on vma_merge), +- * we must remove another next too. It would clutter +- * up the code too much to do both in one go. ++ * we must remove next_next too. + */ +- if (remove_next != 3) { +- /* +- * If "next" was removed and vma->vm_end was +- * expanded (up) over it, in turn +- * "next->vm_prev->vm_end" changed and the +- * "vma->vm_next" gap must be updated. +- */ +- next = vma->vm_next; +- } else { +- /* +- * For the scope of the comment "next" and +- * "vma" considered pre-swap(): if "vma" was +- * removed, next->vm_start was expanded (down) +- * over it and the "next" gap must be updated. +- * Because of the swap() the post-swap() "vma" +- * actually points to pre-swap() "next" +- * (post-swap() "next" as opposed is now a +- * dangling pointer). +- */ +- next = vma; +- } + if (remove_next == 2) { + remove_next = 1; +- end = next->vm_end; ++ next = next_next; + goto again; + } +- else if (next) +- vma_gap_update(next); +- else { +- /* +- * If remove_next == 2 we obviously can't +- * reach this path. +- * +- * If remove_next == 3 we can't reach this +- * path because pre-swap() next is always not +- * NULL. pre-swap() "next" is not being +- * removed and its next->vm_end is not altered +- * (and furthermore "end" already matches +- * next->vm_end in remove_next == 3). +- * +- * We reach this only in the remove_next == 1 +- * case if the "next" vma that was removed was +- * the highest vma of the mm. However in such +- * case next->vm_end == "end" and the extended +- * "vma" has vma->vm_end == next->vm_end so +- * mm->highest_vm_end doesn't need any update +- * in remove_next == 1 case. +- */ +- VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); +- } + } + if (insert && file) + uprobe_mmap(insert); + ++ mas_destroy(&mas); + validate_mm(mm); + + return 0; +@@ -1138,10 +1015,10 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, + if (vm_flags & VM_SPECIAL) + return NULL; + +- next = vma_next(mm, prev); ++ next = find_vma(mm, prev ? prev->vm_end : 0); + area = next; + if (area && area->vm_end == end) /* cases 6, 7, 8 */ +- next = next->vm_next; ++ next = find_vma(mm, next->vm_end); + + /* verify some invariant that must be enforced by the caller */ + VM_WARN_ON(prev && addr <= prev->vm_start); +@@ -1275,18 +1152,24 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_ + */ + struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) + { ++ MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); + struct anon_vma *anon_vma = NULL; ++ struct vm_area_struct *prev, *next; + + /* Try next first. */ +- if (vma->vm_next) { +- anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next); ++ next = mas_walk(&mas); ++ if (next) { ++ anon_vma = reusable_anon_vma(next, vma, next); + if (anon_vma) + return anon_vma; + } + ++ prev = mas_prev(&mas, 0); ++ VM_BUG_ON_VMA(prev != vma, vma); ++ prev = mas_prev(&mas, 0); + /* Try prev next. */ +- if (vma->vm_prev) +- anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma); ++ if (prev) ++ anon_vma = reusable_anon_vma(prev, prev, vma); + + /* + * We might reach here with anon_vma == NULL if we can't find +@@ -1375,6 +1258,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, + vm_flags_t vm_flags; + int pkey = 0; + ++ validate_mm(mm); + *populate = 0; + + if (!len) +@@ -1678,388 +1562,63 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) + return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; + } + +-unsigned long mmap_region(struct file *file, unsigned long addr, +- unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, +- struct list_head *uf) ++/** ++ * unmapped_area() - Find an area between the low_limit and the high_limit with ++ * the correct alignment and offset, all from @info. Note: current->mm is used ++ * for the search. ++ * ++ * @info: The unmapped area information including the range (low_limit - ++ * hight_limit), the alignment offset and mask. ++ * ++ * Return: A memory address or -ENOMEM. ++ */ ++static unsigned long unmapped_area(struct vm_unmapped_area_info *info) + { +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma, *prev, *merge; +- int error; +- struct rb_node **rb_link, *rb_parent; +- unsigned long charged = 0; +- +- /* Check against address space limit. */ +- if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { +- unsigned long nr_pages; +- +- /* +- * MAP_FIXED may remove pages of mappings that intersects with +- * requested mapping. Account for the pages it would unmap. +- */ +- nr_pages = count_vma_pages_range(mm, addr, addr + len); +- +- if (!may_expand_vm(mm, vm_flags, +- (len >> PAGE_SHIFT) - nr_pages)) +- return -ENOMEM; +- } +- +- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ +- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) +- return -ENOMEM; +- /* +- * Private writable mapping: check memory availability +- */ +- if (accountable_mapping(file, vm_flags)) { +- charged = len >> PAGE_SHIFT; +- if (security_vm_enough_memory_mm(mm, charged)) +- return -ENOMEM; +- vm_flags |= VM_ACCOUNT; +- } +- +- /* +- * Can we just expand an old mapping? +- */ +- vma = vma_merge(mm, prev, addr, addr + len, vm_flags, +- NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); +- if (vma) +- goto out; +- +- /* +- * Determine the object being mapped and call the appropriate +- * specific mapper. the address has already been validated, but +- * not unmapped, but the maps are removed from the list. +- */ +- vma = vm_area_alloc(mm); +- if (!vma) { +- error = -ENOMEM; +- goto unacct_error; +- } +- +- vma->vm_start = addr; +- vma->vm_end = addr + len; +- vma->vm_flags = vm_flags; +- vma->vm_page_prot = vm_get_page_prot(vm_flags); +- vma->vm_pgoff = pgoff; +- +- if (file) { +- if (vm_flags & VM_SHARED) { +- error = mapping_map_writable(file->f_mapping); +- if (error) +- goto free_vma; +- } +- +- vma->vm_file = get_file(file); +- error = call_mmap(file, vma); +- if (error) +- goto unmap_and_free_vma; +- +- /* Can addr have changed?? +- * +- * Answer: Yes, several device drivers can do it in their +- * f_op->mmap method. -DaveM +- * Bug: If addr is changed, prev, rb_link, rb_parent should +- * be updated for vma_link() +- */ +- WARN_ON_ONCE(addr != vma->vm_start); +- +- addr = vma->vm_start; +- +- /* If vm_flags changed after call_mmap(), we should try merge vma again +- * as we may succeed this time. +- */ +- if (unlikely(vm_flags != vma->vm_flags && prev)) { +- merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, +- NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); +- if (merge) { +- /* ->mmap() can change vma->vm_file and fput the original file. So +- * fput the vma->vm_file here or we would add an extra fput for file +- * and cause general protection fault ultimately. +- */ +- fput(vma->vm_file); +- vm_area_free(vma); +- vma = merge; +- /* Update vm_flags to pick up the change. */ +- vm_flags = vma->vm_flags; +- goto unmap_writable; +- } +- } +- +- vm_flags = vma->vm_flags; +- } else if (vm_flags & VM_SHARED) { +- error = shmem_zero_setup(vma); +- if (error) +- goto free_vma; +- } else { +- vma_set_anonymous(vma); +- } +- +- /* Allow architectures to sanity-check the vm_flags */ +- if (!arch_validate_flags(vma->vm_flags)) { +- error = -EINVAL; +- if (file) +- goto unmap_and_free_vma; +- else +- goto free_vma; +- } +- +- vma_link(mm, vma, prev, rb_link, rb_parent); +- +- /* +- * vma_merge() calls khugepaged_enter_vma() either, the below +- * call covers the non-merge case. +- */ +- khugepaged_enter_vma(vma, vma->vm_flags); +- +- /* Once vma denies write, undo our temporary denial count */ +-unmap_writable: +- if (file && vm_flags & VM_SHARED) +- mapping_unmap_writable(file->f_mapping); +- file = vma->vm_file; +-out: +- perf_event_mmap(vma); +- +- vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); +- if (vm_flags & VM_LOCKED) { +- if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || +- is_vm_hugetlb_page(vma) || +- vma == get_gate_vma(current->mm)) +- vma->vm_flags &= VM_LOCKED_CLEAR_MASK; +- else +- mm->locked_vm += (len >> PAGE_SHIFT); +- } ++ unsigned long length, gap; + +- if (file) +- uprobe_mmap(vma); +- +- /* +- * New (or expanded) vma always get soft dirty status. +- * Otherwise user-space soft-dirty page tracker won't +- * be able to distinguish situation when vma area unmapped, +- * then new mapped in-place (which must be aimed as +- * a completely new data area). +- */ +- vma->vm_flags |= VM_SOFTDIRTY; +- +- vma_set_page_prot(vma); +- +- return addr; +- +-unmap_and_free_vma: +- fput(vma->vm_file); +- vma->vm_file = NULL; +- +- /* Undo any partial mapping done by a device driver. */ +- unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); +- if (vm_flags & VM_SHARED) +- mapping_unmap_writable(file->f_mapping); +-free_vma: +- vm_area_free(vma); +-unacct_error: +- if (charged) +- vm_unacct_memory(charged); +- return error; +-} +- +-static unsigned long unmapped_area(struct vm_unmapped_area_info *info) +-{ +- /* +- * We implement the search by looking for an rbtree node that +- * immediately follows a suitable gap. That is, +- * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length; +- * - gap_end = vma->vm_start >= info->low_limit + length; +- * - gap_end - gap_start >= length +- */ +- +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma; +- unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + +- /* Adjust search limits by the desired length */ +- if (info->high_limit < length) +- return -ENOMEM; +- high_limit = info->high_limit - length; +- +- if (info->low_limit > high_limit) +- return -ENOMEM; +- low_limit = info->low_limit + length; +- +- /* Check if rbtree root looks promising */ +- if (RB_EMPTY_ROOT(&mm->mm_rb)) +- goto check_highest; +- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); +- if (vma->rb_subtree_gap < length) +- goto check_highest; +- +- while (true) { +- /* Visit left subtree if it looks promising */ +- gap_end = vm_start_gap(vma); +- if (gap_end >= low_limit && vma->vm_rb.rb_left) { +- struct vm_area_struct *left = +- rb_entry(vma->vm_rb.rb_left, +- struct vm_area_struct, vm_rb); +- if (left->rb_subtree_gap >= length) { +- vma = left; +- continue; +- } +- } +- +- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; +-check_current: +- /* Check if current node has a suitable gap */ +- if (gap_start > high_limit) +- return -ENOMEM; +- if (gap_end >= low_limit && +- gap_end > gap_start && gap_end - gap_start >= length) +- goto found; +- +- /* Visit right subtree if it looks promising */ +- if (vma->vm_rb.rb_right) { +- struct vm_area_struct *right = +- rb_entry(vma->vm_rb.rb_right, +- struct vm_area_struct, vm_rb); +- if (right->rb_subtree_gap >= length) { +- vma = right; +- continue; +- } +- } +- +- /* Go back up the rbtree to find next candidate node */ +- while (true) { +- struct rb_node *prev = &vma->vm_rb; +- if (!rb_parent(prev)) +- goto check_highest; +- vma = rb_entry(rb_parent(prev), +- struct vm_area_struct, vm_rb); +- if (prev == vma->vm_rb.rb_left) { +- gap_start = vm_end_gap(vma->vm_prev); +- gap_end = vm_start_gap(vma); +- goto check_current; +- } +- } +- } +- +-check_highest: +- /* Check highest gap, which does not precede any rbtree node */ +- gap_start = mm->highest_vm_end; +- gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */ +- if (gap_start > high_limit) ++ if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, ++ length)) + return -ENOMEM; + +-found: +- /* We found a suitable gap. Clip it with the original low_limit. */ +- if (gap_start < info->low_limit) +- gap_start = info->low_limit; +- +- /* Adjust gap address to the desired alignment */ +- gap_start += (info->align_offset - gap_start) & info->align_mask; +- +- VM_BUG_ON(gap_start + info->length > info->high_limit); +- VM_BUG_ON(gap_start + info->length > gap_end); +- return gap_start; ++ gap = mas.index; ++ gap += (info->align_offset - gap) & info->align_mask; ++ return gap; + } + ++/** ++ * unmapped_area_topdown() - Find an area between the low_limit and the ++ * high_limit with * the correct alignment and offset at the highest available ++ * address, all from @info. Note: current->mm is used for the search. ++ * ++ * @info: The unmapped area information including the range (low_limit - ++ * hight_limit), the alignment offset and mask. ++ * ++ * Return: A memory address or -ENOMEM. ++ */ + static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) + { +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma; +- unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ unsigned long length, gap; + ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + +- /* +- * Adjust search limits by the desired length. +- * See implementation comment at top of unmapped_area(). +- */ +- gap_end = info->high_limit; +- if (gap_end < length) +- return -ENOMEM; +- high_limit = gap_end - length; +- +- if (info->low_limit > high_limit) ++ if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, ++ length)) + return -ENOMEM; +- low_limit = info->low_limit + length; + +- /* Check highest gap, which does not precede any rbtree node */ +- gap_start = mm->highest_vm_end; +- if (gap_start <= high_limit) +- goto found_highest; +- +- /* Check if rbtree root looks promising */ +- if (RB_EMPTY_ROOT(&mm->mm_rb)) +- return -ENOMEM; +- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); +- if (vma->rb_subtree_gap < length) +- return -ENOMEM; +- +- while (true) { +- /* Visit right subtree if it looks promising */ +- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; +- if (gap_start <= high_limit && vma->vm_rb.rb_right) { +- struct vm_area_struct *right = +- rb_entry(vma->vm_rb.rb_right, +- struct vm_area_struct, vm_rb); +- if (right->rb_subtree_gap >= length) { +- vma = right; +- continue; +- } +- } +- +-check_current: +- /* Check if current node has a suitable gap */ +- gap_end = vm_start_gap(vma); +- if (gap_end < low_limit) +- return -ENOMEM; +- if (gap_start <= high_limit && +- gap_end > gap_start && gap_end - gap_start >= length) +- goto found; +- +- /* Visit left subtree if it looks promising */ +- if (vma->vm_rb.rb_left) { +- struct vm_area_struct *left = +- rb_entry(vma->vm_rb.rb_left, +- struct vm_area_struct, vm_rb); +- if (left->rb_subtree_gap >= length) { +- vma = left; +- continue; +- } +- } +- +- /* Go back up the rbtree to find next candidate node */ +- while (true) { +- struct rb_node *prev = &vma->vm_rb; +- if (!rb_parent(prev)) +- return -ENOMEM; +- vma = rb_entry(rb_parent(prev), +- struct vm_area_struct, vm_rb); +- if (prev == vma->vm_rb.rb_right) { +- gap_start = vma->vm_prev ? +- vm_end_gap(vma->vm_prev) : 0; +- goto check_current; +- } +- } +- } +- +-found: +- /* We found a suitable gap. Clip it with the original high_limit. */ +- if (gap_end > info->high_limit) +- gap_end = info->high_limit; +- +-found_highest: +- /* Compute highest gap address at the desired alignment */ +- gap_end -= info->length; +- gap_end -= (gap_end - info->align_offset) & info->align_mask; +- +- VM_BUG_ON(gap_end < info->low_limit); +- VM_BUG_ON(gap_end < gap_start); +- return gap_end; ++ gap = mas.last + 1 - info->length; ++ gap -= (gap - info->align_offset) & info->align_mask; ++ return gap; + } + + /* +@@ -2249,58 +1808,67 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, + + EXPORT_SYMBOL(get_unmapped_area); + +-/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ +-struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) ++/** ++ * find_vma_intersection() - Look up the first VMA which intersects the interval ++ * @mm: The process address space. ++ * @start_addr: The inclusive start user address. ++ * @end_addr: The exclusive end user address. ++ * ++ * Returns: The first VMA within the provided range, %NULL otherwise. Assumes ++ * start_addr < end_addr. ++ */ ++struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, ++ unsigned long start_addr, ++ unsigned long end_addr) + { +- struct rb_node *rb_node; +- struct vm_area_struct *vma; ++ unsigned long index = start_addr; + + mmap_assert_locked(mm); +- /* Check the cache first. */ +- vma = vmacache_find(mm, addr); +- if (likely(vma)) +- return vma; +- +- rb_node = mm->mm_rb.rb_node; +- +- while (rb_node) { +- struct vm_area_struct *tmp; +- +- tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb); ++ return mt_find(&mm->mm_mt, &index, end_addr - 1); ++} ++EXPORT_SYMBOL(find_vma_intersection); + +- if (tmp->vm_end > addr) { +- vma = tmp; +- if (tmp->vm_start <= addr) +- break; +- rb_node = rb_node->rb_left; +- } else +- rb_node = rb_node->rb_right; +- } ++/** ++ * find_vma() - Find the VMA for a given address, or the next VMA. ++ * @mm: The mm_struct to check ++ * @addr: The address ++ * ++ * Returns: The VMA associated with addr, or the next VMA. ++ * May return %NULL in the case of no VMA at addr or above. ++ */ ++struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) ++{ ++ unsigned long index = addr; + +- if (vma) +- vmacache_update(addr, vma); +- return vma; ++ mmap_assert_locked(mm); ++ return mt_find(&mm->mm_mt, &index, ULONG_MAX); + } +- + EXPORT_SYMBOL(find_vma); + +-/* +- * Same as find_vma, but also return a pointer to the previous VMA in *pprev. ++/** ++ * find_vma_prev() - Find the VMA for a given address, or the next vma and ++ * set %pprev to the previous VMA, if any. ++ * @mm: The mm_struct to check ++ * @addr: The address ++ * @pprev: The pointer to set to the previous VMA ++ * ++ * Note that RCU lock is missing here since the external mmap_lock() is used ++ * instead. ++ * ++ * Returns: The VMA associated with @addr, or the next vma. ++ * May return %NULL in the case of no vma at addr or above. + */ + struct vm_area_struct * + find_vma_prev(struct mm_struct *mm, unsigned long addr, + struct vm_area_struct **pprev) + { + struct vm_area_struct *vma; ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + +- vma = find_vma(mm, addr); +- if (vma) { +- *pprev = vma->vm_prev; +- } else { +- struct rb_node *rb_node = rb_last(&mm->mm_rb); +- +- *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL; +- } ++ vma = mas_walk(&mas); ++ *pprev = mas_prev(&mas, 0); ++ if (!vma) ++ vma = mas_next(&mas, ULONG_MAX); + return vma; + } + +@@ -2354,6 +1922,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + struct vm_area_struct *next; + unsigned long gap_addr; + int error = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + if (!(vma->vm_flags & VM_GROWSUP)) + return -EFAULT; +@@ -2371,16 +1940,21 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + if (gap_addr < address || gap_addr > TASK_SIZE) + gap_addr = TASK_SIZE; + +- next = vma->vm_next; +- if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { ++ next = find_vma_intersection(mm, vma->vm_end, gap_addr); ++ if (next && vma_is_accessible(next)) { + if (!(next->vm_flags & VM_GROWSUP)) + return -ENOMEM; + /* Check that both stack segments have the same anon_vma? */ + } + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; ++ + /* We must make sure the anon_vma is allocated. */ +- if (unlikely(anon_vma_prepare(vma))) ++ if (unlikely(anon_vma_prepare(vma))) { ++ mas_destroy(&mas); + return -ENOMEM; ++ } + + /* + * vma->vm_start/vm_end cannot change under us because the caller +@@ -2401,15 +1975,13 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + error = acct_stack_growth(vma, size, grow); + if (!error) { + /* +- * vma_gap_update() doesn't support concurrent +- * updates, but we only hold a shared mmap_lock +- * lock here, so we need to protect against +- * concurrent vma expansions. +- * anon_vma_lock_write() doesn't help here, as +- * we don't guarantee that all growable vmas +- * in a mm share the same root anon vma. +- * So, we reuse mm->page_table_lock to guard +- * against concurrent vma expansions. ++ * We only hold a shared mmap_lock lock here, so ++ * we need to protect against concurrent vma ++ * expansions. anon_vma_lock_write() doesn't ++ * help here, as we don't guarantee that all ++ * growable vmas in a mm share the same root ++ * anon vma. So, we reuse mm->page_table_lock ++ * to guard against concurrent vma expansions. + */ + spin_lock(&mm->page_table_lock); + if (vma->vm_flags & VM_LOCKED) +@@ -2417,11 +1989,9 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + vm_stat_account(mm, vma->vm_flags, grow); + anon_vma_interval_tree_pre_update_vma(vma); + vma->vm_end = address; ++ /* Overwrite old entry in mtree. */ ++ vma_mas_store(vma, &mas); + anon_vma_interval_tree_post_update_vma(vma); +- if (vma->vm_next) +- vma_gap_update(vma->vm_next); +- else +- mm->highest_vm_end = vm_end_gap(vma); + spin_unlock(&mm->page_table_lock); + + perf_event_mmap(vma); +@@ -2430,7 +2000,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + } + anon_vma_unlock_write(vma->anon_vma); + khugepaged_enter_vma(vma, vma->vm_flags); +- validate_mm(mm); ++ mas_destroy(&mas); + return error; + } + #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ +@@ -2438,10 +2008,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + /* + * vma is the first one with address < vma->vm_start. Have to extend vma. + */ +-int expand_downwards(struct vm_area_struct *vma, +- unsigned long address) ++int expand_downwards(struct vm_area_struct *vma, unsigned long address) + { + struct mm_struct *mm = vma->vm_mm; ++ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); + struct vm_area_struct *prev; + int error = 0; + +@@ -2450,7 +2020,7 @@ int expand_downwards(struct vm_area_struct *vma, + return -EPERM; + + /* Enforce stack_guard_gap */ +- prev = vma->vm_prev; ++ prev = mas_prev(&mas, 0); + /* Check that both stack segments have the same anon_vma? */ + if (prev && !(prev->vm_flags & VM_GROWSDOWN) && + vma_is_accessible(prev)) { +@@ -2458,9 +2028,14 @@ int expand_downwards(struct vm_area_struct *vma, + return -ENOMEM; + } + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; ++ + /* We must make sure the anon_vma is allocated. */ +- if (unlikely(anon_vma_prepare(vma))) ++ if (unlikely(anon_vma_prepare(vma))) { ++ mas_destroy(&mas); + return -ENOMEM; ++ } + + /* + * vma->vm_start/vm_end cannot change under us because the caller +@@ -2481,15 +2056,13 @@ int expand_downwards(struct vm_area_struct *vma, + error = acct_stack_growth(vma, size, grow); + if (!error) { + /* +- * vma_gap_update() doesn't support concurrent +- * updates, but we only hold a shared mmap_lock +- * lock here, so we need to protect against +- * concurrent vma expansions. +- * anon_vma_lock_write() doesn't help here, as +- * we don't guarantee that all growable vmas +- * in a mm share the same root anon vma. +- * So, we reuse mm->page_table_lock to guard +- * against concurrent vma expansions. ++ * We only hold a shared mmap_lock lock here, so ++ * we need to protect against concurrent vma ++ * expansions. anon_vma_lock_write() doesn't ++ * help here, as we don't guarantee that all ++ * growable vmas in a mm share the same root ++ * anon vma. So, we reuse mm->page_table_lock ++ * to guard against concurrent vma expansions. + */ + spin_lock(&mm->page_table_lock); + if (vma->vm_flags & VM_LOCKED) +@@ -2498,8 +2071,9 @@ int expand_downwards(struct vm_area_struct *vma, + anon_vma_interval_tree_pre_update_vma(vma); + vma->vm_start = address; + vma->vm_pgoff -= grow; ++ /* Overwrite old entry in mtree. */ ++ vma_mas_store(vma, &mas); + anon_vma_interval_tree_post_update_vma(vma); +- vma_gap_update(vma); + spin_unlock(&mm->page_table_lock); + + perf_event_mmap(vma); +@@ -2508,7 +2082,7 @@ int expand_downwards(struct vm_area_struct *vma, + } + anon_vma_unlock_write(vma->anon_vma); + khugepaged_enter_vma(vma, vma->vm_flags); +- validate_mm(mm); ++ mas_destroy(&mas); + return error; + } + +@@ -2581,25 +2155,26 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) + EXPORT_SYMBOL_GPL(find_extend_vma); + + /* +- * Ok - we have the memory areas we should free on the vma list, +- * so release them, and do the vma updates. ++ * Ok - we have the memory areas we should free on a maple tree so release them, ++ * and do the vma updates. + * + * Called with the mm semaphore held. + */ +-static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) ++static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) + { + unsigned long nr_accounted = 0; ++ struct vm_area_struct *vma; + + /* Update high watermark before we lower total_vm */ + update_hiwater_vm(mm); +- do { ++ mas_for_each(mas, vma, ULONG_MAX) { + long nrpages = vma_pages(vma); + + if (vma->vm_flags & VM_ACCOUNT) + nr_accounted += nrpages; + vm_stat_account(mm, vma->vm_flags, -nrpages); +- vma = remove_vma(vma); +- } while (vma); ++ remove_vma(vma); ++ } + vm_unacct_memory(nr_accounted); + validate_mm(mm); + } +@@ -2609,75 +2184,32 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) + * + * Called with the mm semaphore held. + */ +-static void unmap_region(struct mm_struct *mm, ++static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, + struct vm_area_struct *vma, struct vm_area_struct *prev, ++ struct vm_area_struct *next, + unsigned long start, unsigned long end) + { +- struct vm_area_struct *next = vma_next(mm, prev); + struct mmu_gather tlb; + + lru_add_drain(); + tlb_gather_mmu(&tlb, mm); + update_hiwater_rss(mm); +- unmap_vmas(&tlb, vma, start, end); +- free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, ++ unmap_vmas(&tlb, mt, vma, start, end); ++ free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, + next ? next->vm_start : USER_PGTABLES_CEILING); + tlb_finish_mmu(&tlb); + } + + /* +- * Create a list of vma's touched by the unmap, removing them from the mm's +- * vma list as we go.. ++ * __split_vma() bypasses sysctl_max_map_count checking. We use this where it ++ * has already been checked or doesn't make sense to fail. + */ +-static bool +-detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev, unsigned long end) ++int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, ++ unsigned long addr, int new_below) + { +- struct vm_area_struct **insertion_point; +- struct vm_area_struct *tail_vma = NULL; +- +- insertion_point = (prev ? &prev->vm_next : &mm->mmap); +- vma->vm_prev = NULL; +- do { +- vma_rb_erase(vma, &mm->mm_rb); +- if (vma->vm_flags & VM_LOCKED) +- mm->locked_vm -= vma_pages(vma); +- mm->map_count--; +- tail_vma = vma; +- vma = vma->vm_next; +- } while (vma && vma->vm_start < end); +- *insertion_point = vma; +- if (vma) { +- vma->vm_prev = prev; +- vma_gap_update(vma); +- } else +- mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; +- tail_vma->vm_next = NULL; +- +- /* Kill the cache */ +- vmacache_invalidate(mm); +- +- /* +- * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or +- * VM_GROWSUP VMA. Such VMAs can change their size under +- * down_read(mmap_lock) and collide with the VMA we are about to unmap. +- */ +- if (vma && (vma->vm_flags & VM_GROWSDOWN)) +- return false; +- if (prev && (prev->vm_flags & VM_GROWSUP)) +- return false; +- return true; +-} +- +-/* +- * __split_vma() bypasses sysctl_max_map_count checking. We use this where it +- * has already been checked or doesn't make sense to fail. +- */ +-int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, +- unsigned long addr, int new_below) +-{ +- struct vm_area_struct *new; +- int err; ++ struct vm_area_struct *new; ++ int err; ++ validate_mm_mt(mm); + + if (vma->vm_ops && vma->vm_ops->may_split) { + err = vma->vm_ops->may_split(vma, addr); +@@ -2720,6 +2252,9 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + if (!err) + return 0; + ++ /* Avoid vm accounting in close() operation */ ++ new->vm_start = new->vm_end; ++ new->vm_pgoff = 0; + /* Clean everything up if vma_adjust failed. */ + if (new->vm_ops && new->vm_ops->close) + new->vm_ops->close(new); +@@ -2730,6 +2265,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + mpol_put(vma_policy(new)); + out_free_vma: + vm_area_free(new); ++ validate_mm_mt(mm); + return err; + } + +@@ -2746,38 +2282,48 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + return __split_vma(mm, vma, addr, new_below); + } + +-/* Munmap is split into 2 main parts -- this part which finds +- * what needs doing, and the areas themselves, which do the +- * work. This now handles partial unmappings. +- * Jeremy Fitzhardinge +- */ +-int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, +- struct list_head *uf, bool downgrade) ++static inline int munmap_sidetree(struct vm_area_struct *vma, ++ struct ma_state *mas_detach) + { +- unsigned long end; +- struct vm_area_struct *vma, *prev, *last; +- +- if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) +- return -EINVAL; ++ mas_set_range(mas_detach, vma->vm_start, vma->vm_end - 1); ++ if (mas_store_gfp(mas_detach, vma, GFP_KERNEL)) ++ return -ENOMEM; + +- len = PAGE_ALIGN(len); +- end = start + len; +- if (len == 0) +- return -EINVAL; ++ if (vma->vm_flags & VM_LOCKED) ++ vma->vm_mm->locked_vm -= vma_pages(vma); + +- /* +- * arch_unmap() might do unmaps itself. It must be called +- * and finish any rbtree manipulation before this code +- * runs and also starts to manipulate the rbtree. +- */ +- arch_unmap(mm, start, end); ++ return 0; ++} + +- /* Find the first overlapping VMA where start < vma->vm_end */ +- vma = find_vma_intersection(mm, start, end); +- if (!vma) +- return 0; +- prev = vma->vm_prev; ++/* ++ * do_mas_align_munmap() - munmap the aligned region from @start to @end. ++ * @mas: The maple_state, ideally set up to alter the correct tree location. ++ * @vma: The starting vm_area_struct ++ * @mm: The mm_struct ++ * @start: The aligned start address to munmap. ++ * @end: The aligned end address to munmap. ++ * @uf: The userfaultfd list_head ++ * @downgrade: Set to true to attempt a write downgrade of the mmap_sem ++ * ++ * If @downgrade is true, check return code for potential release of the lock. ++ */ ++static int ++do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, ++ struct mm_struct *mm, unsigned long start, ++ unsigned long end, struct list_head *uf, bool downgrade) ++{ ++ struct vm_area_struct *prev, *next = NULL; ++ struct maple_tree mt_detach; ++ int count = 0; ++ int error = -ENOMEM; ++ MA_STATE(mas_detach, &mt_detach, 0, 0); ++ mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN); ++ mt_set_external_lock(&mt_detach, &mm->mmap_lock); ++ ++ if (mas_preallocate(mas, vma, GFP_KERNEL)) ++ return -ENOMEM; + ++ mas->last = end - 1; + /* + * If we need to split any vma, do it now to save pain later. + * +@@ -2785,8 +2331,9 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + * unmapped vm_area_struct will remain in use: so lower split_vma + * places tmp vma above, and higher split_vma places tmp vma below. + */ ++ ++ /* Does it split the first one? */ + if (start > vma->vm_start) { +- int error; + + /* + * Make sure that map_count on return from munmap() will +@@ -2794,22 +2341,61 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + * its limit temporarily, to help free resources as expected. + */ + if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) +- return -ENOMEM; ++ goto map_count_exceeded; + ++ /* ++ * mas_pause() is not needed since mas->index needs to be set ++ * differently than vma->vm_end anyways. ++ */ + error = __split_vma(mm, vma, start, 0); + if (error) +- return error; +- prev = vma; ++ goto start_split_failed; ++ ++ mas_set(mas, start); ++ vma = mas_walk(mas); + } + +- /* Does it split the last one? */ +- last = find_vma(mm, end); +- if (last && end > last->vm_start) { +- int error = __split_vma(mm, last, end, 1); ++ prev = mas_prev(mas, 0); ++ if (unlikely((!prev))) ++ mas_set(mas, start); ++ ++ /* ++ * Detach a range of VMAs from the mm. Using next as a temp variable as ++ * it is always overwritten. ++ */ ++ mas_for_each(mas, next, end - 1) { ++ /* Does it split the end? */ ++ if (next->vm_end > end) { ++ struct vm_area_struct *split; ++ ++ error = __split_vma(mm, next, end, 1); ++ if (error) ++ goto end_split_failed; ++ ++ mas_set(mas, end); ++ split = mas_prev(mas, 0); ++ error = munmap_sidetree(split, &mas_detach); ++ if (error) ++ goto munmap_sidetree_failed; ++ ++ count++; ++ if (vma == next) ++ vma = split; ++ break; ++ } ++ error = munmap_sidetree(next, &mas_detach); + if (error) +- return error; ++ goto munmap_sidetree_failed; ++ ++ count++; ++#ifdef CONFIG_DEBUG_VM_MAPLE_TREE ++ BUG_ON(next->vm_start < start); ++ BUG_ON(next->vm_start > end); ++#endif + } +- vma = vma_next(mm, prev); ++ ++ if (!next) ++ next = mas_next(mas, ULONG_MAX); + + if (unlikely(uf)) { + /* +@@ -2821,30 +2407,366 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + * split, despite we could. This is unlikely enough + * failure that it's not worth optimizing it for. + */ +- int error = userfaultfd_unmap_prep(vma, start, end, uf); ++ error = userfaultfd_unmap_prep(mm, start, end, uf); ++ + if (error) +- return error; ++ goto userfaultfd_error; ++ } ++ ++ /* Point of no return */ ++ mas_set_range(mas, start, end - 1); ++#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) ++ /* Make sure no VMAs are about to be lost. */ ++ { ++ MA_STATE(test, &mt_detach, start, end - 1); ++ struct vm_area_struct *vma_mas, *vma_test; ++ int test_count = 0; ++ ++ rcu_read_lock(); ++ vma_test = mas_find(&test, end - 1); ++ mas_for_each(mas, vma_mas, end - 1) { ++ BUG_ON(vma_mas != vma_test); ++ test_count++; ++ vma_test = mas_next(&test, end - 1); ++ } ++ rcu_read_unlock(); ++ BUG_ON(count != test_count); ++ mas_set_range(mas, start, end - 1); ++ } ++#endif ++ mas_store_prealloc(mas, NULL); ++ mm->map_count -= count; ++ /* ++ * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or ++ * VM_GROWSUP VMA. Such VMAs can change their size under ++ * down_read(mmap_lock) and collide with the VMA we are about to unmap. ++ */ ++ if (downgrade) { ++ if (next && (next->vm_flags & VM_GROWSDOWN)) ++ downgrade = false; ++ else if (prev && (prev->vm_flags & VM_GROWSUP)) ++ downgrade = false; ++ else ++ mmap_write_downgrade(mm); + } + +- /* Detach vmas from rbtree */ +- if (!detach_vmas_to_be_unmapped(mm, vma, prev, end)) +- downgrade = false; ++ unmap_region(mm, &mt_detach, vma, prev, next, start, end); ++ /* Statistics and freeing VMAs */ ++ mas_set(&mas_detach, start); ++ remove_mt(mm, &mas_detach); ++ __mt_destroy(&mt_detach); + +- if (downgrade) +- mmap_write_downgrade(mm); + +- unmap_region(mm, vma, prev, start, end); ++ validate_mm(mm); ++ return downgrade ? 1 : 0; + +- /* Fix up all other VM information */ +- remove_vma_list(mm, vma); ++userfaultfd_error: ++munmap_sidetree_failed: ++end_split_failed: ++ __mt_destroy(&mt_detach); ++start_split_failed: ++map_count_exceeded: ++ mas_destroy(mas); ++ return error; ++} + +- return downgrade ? 1 : 0; ++/* ++ * do_mas_munmap() - munmap a given range. ++ * @mas: The maple state ++ * @mm: The mm_struct ++ * @start: The start address to munmap ++ * @len: The length of the range to munmap ++ * @uf: The userfaultfd list_head ++ * @downgrade: set to true if the user wants to attempt to write_downgrade the ++ * mmap_sem ++ * ++ * This function takes a @mas that is either pointing to the previous VMA or set ++ * to MA_START and sets it up to remove the mapping(s). The @len will be ++ * aligned and any arch_unmap work will be preformed. ++ * ++ * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise. ++ */ ++int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, ++ unsigned long start, size_t len, struct list_head *uf, ++ bool downgrade) ++{ ++ unsigned long end; ++ struct vm_area_struct *vma; ++ ++ if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) ++ return -EINVAL; ++ ++ end = start + PAGE_ALIGN(len); ++ if (end == start) ++ return -EINVAL; ++ ++ /* arch_unmap() might do unmaps itself. */ ++ arch_unmap(mm, start, end); ++ ++ /* Find the first overlapping VMA */ ++ vma = mas_find(mas, end - 1); ++ if (!vma) ++ return 0; ++ ++ return do_mas_align_munmap(mas, vma, mm, start, end, uf, downgrade); + } + ++/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. ++ * @mm: The mm_struct ++ * @start: The start address to munmap ++ * @len: The length to be munmapped. ++ * @uf: The userfaultfd list_head ++ */ + int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + struct list_head *uf) + { +- return __do_munmap(mm, start, len, uf, false); ++ MA_STATE(mas, &mm->mm_mt, start, start); ++ ++ return do_mas_munmap(&mas, mm, start, len, uf, false); ++} ++ ++unsigned long mmap_region(struct file *file, unsigned long addr, ++ unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, ++ struct list_head *uf) ++{ ++ struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma = NULL; ++ struct vm_area_struct *next, *prev, *merge; ++ pgoff_t pglen = len >> PAGE_SHIFT; ++ unsigned long charged = 0; ++ unsigned long end = addr + len; ++ unsigned long merge_start = addr, merge_end = end; ++ pgoff_t vm_pgoff; ++ int error; ++ MA_STATE(mas, &mm->mm_mt, addr, end - 1); ++ ++ /* Check against address space limit. */ ++ if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { ++ unsigned long nr_pages; ++ ++ /* ++ * MAP_FIXED may remove pages of mappings that intersects with ++ * requested mapping. Account for the pages it would unmap. ++ */ ++ nr_pages = count_vma_pages_range(mm, addr, end); ++ ++ if (!may_expand_vm(mm, vm_flags, ++ (len >> PAGE_SHIFT) - nr_pages)) ++ return -ENOMEM; ++ } ++ ++ /* Unmap any existing mapping in the area */ ++ if (do_mas_munmap(&mas, mm, addr, len, uf, false)) ++ return -ENOMEM; ++ ++ /* ++ * Private writable mapping: check memory availability ++ */ ++ if (accountable_mapping(file, vm_flags)) { ++ charged = len >> PAGE_SHIFT; ++ if (security_vm_enough_memory_mm(mm, charged)) ++ return -ENOMEM; ++ vm_flags |= VM_ACCOUNT; ++ } ++ ++ next = mas_next(&mas, ULONG_MAX); ++ prev = mas_prev(&mas, 0); ++ if (vm_flags & VM_SPECIAL) ++ goto cannot_expand; ++ ++ /* Attempt to expand an old mapping */ ++ /* Check next */ ++ if (next && next->vm_start == end && !vma_policy(next) && ++ can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, ++ NULL_VM_UFFD_CTX, NULL)) { ++ merge_end = next->vm_end; ++ vma = next; ++ vm_pgoff = next->vm_pgoff - pglen; ++ } ++ ++ /* Check prev */ ++ if (prev && prev->vm_end == addr && !vma_policy(prev) && ++ (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, ++ pgoff, vma->vm_userfaultfd_ctx, NULL) : ++ can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, ++ NULL_VM_UFFD_CTX, NULL))) { ++ merge_start = prev->vm_start; ++ vma = prev; ++ vm_pgoff = prev->vm_pgoff; ++ } ++ ++ ++ /* Actually expand, if possible */ ++ if (vma && ++ !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) { ++ khugepaged_enter_vma(vma, vm_flags); ++ goto expanded; ++ } ++ ++ mas.index = addr; ++ mas.last = end - 1; ++cannot_expand: ++ /* ++ * Determine the object being mapped and call the appropriate ++ * specific mapper. the address has already been validated, but ++ * not unmapped, but the maps are removed from the list. ++ */ ++ vma = vm_area_alloc(mm); ++ if (!vma) { ++ error = -ENOMEM; ++ goto unacct_error; ++ } ++ ++ vma->vm_start = addr; ++ vma->vm_end = end; ++ vma->vm_flags = vm_flags; ++ vma->vm_page_prot = vm_get_page_prot(vm_flags); ++ vma->vm_pgoff = pgoff; ++ ++ if (file) { ++ if (vm_flags & VM_SHARED) { ++ error = mapping_map_writable(file->f_mapping); ++ if (error) ++ goto free_vma; ++ } ++ ++ vma->vm_file = get_file(file); ++ error = call_mmap(file, vma); ++ if (error) ++ goto unmap_and_free_vma; ++ ++ /* Can addr have changed?? ++ * ++ * Answer: Yes, several device drivers can do it in their ++ * f_op->mmap method. -DaveM ++ */ ++ WARN_ON_ONCE(addr != vma->vm_start); ++ ++ addr = vma->vm_start; ++ mas_reset(&mas); ++ ++ /* ++ * If vm_flags changed after call_mmap(), we should try merge ++ * vma again as we may succeed this time. ++ */ ++ if (unlikely(vm_flags != vma->vm_flags && prev)) { ++ merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, ++ NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); ++ if (merge) { ++ /* ++ * ->mmap() can change vma->vm_file and fput ++ * the original file. So fput the vma->vm_file ++ * here or we would add an extra fput for file ++ * and cause general protection fault ++ * ultimately. ++ */ ++ fput(vma->vm_file); ++ vm_area_free(vma); ++ vma = merge; ++ /* Update vm_flags to pick up the change. */ ++ addr = vma->vm_start; ++ vm_flags = vma->vm_flags; ++ goto unmap_writable; ++ } ++ } ++ ++ vm_flags = vma->vm_flags; ++ } else if (vm_flags & VM_SHARED) { ++ error = shmem_zero_setup(vma); ++ if (error) ++ goto free_vma; ++ } else { ++ vma_set_anonymous(vma); ++ } ++ ++ /* Allow architectures to sanity-check the vm_flags */ ++ if (!arch_validate_flags(vma->vm_flags)) { ++ error = -EINVAL; ++ if (file) ++ goto unmap_and_free_vma; ++ else ++ goto free_vma; ++ } ++ ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ error = -ENOMEM; ++ if (file) ++ goto unmap_and_free_vma; ++ else ++ goto free_vma; ++ } ++ ++ if (vma->vm_file) ++ i_mmap_lock_write(vma->vm_file->f_mapping); ++ ++ vma_mas_store(vma, &mas); ++ mm->map_count++; ++ if (vma->vm_file) { ++ if (vma->vm_flags & VM_SHARED) ++ mapping_allow_writable(vma->vm_file->f_mapping); ++ ++ flush_dcache_mmap_lock(vma->vm_file->f_mapping); ++ vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap); ++ flush_dcache_mmap_unlock(vma->vm_file->f_mapping); ++ i_mmap_unlock_write(vma->vm_file->f_mapping); ++ } ++ ++ /* ++ * vma_merge() calls khugepaged_enter_vma() either, the below ++ * call covers the non-merge case. ++ */ ++ khugepaged_enter_vma(vma, vma->vm_flags); ++ ++ /* Once vma denies write, undo our temporary denial count */ ++unmap_writable: ++ if (file && vm_flags & VM_SHARED) ++ mapping_unmap_writable(file->f_mapping); ++ file = vma->vm_file; ++expanded: ++ perf_event_mmap(vma); ++ ++ vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); ++ if (vm_flags & VM_LOCKED) { ++ if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || ++ is_vm_hugetlb_page(vma) || ++ vma == get_gate_vma(current->mm)) ++ vma->vm_flags &= VM_LOCKED_CLEAR_MASK; ++ else ++ mm->locked_vm += (len >> PAGE_SHIFT); ++ } ++ ++ if (file) ++ uprobe_mmap(vma); ++ ++ /* ++ * New (or expanded) vma always get soft dirty status. ++ * Otherwise user-space soft-dirty page tracker won't ++ * be able to distinguish situation when vma area unmapped, ++ * then new mapped in-place (which must be aimed as ++ * a completely new data area). ++ */ ++ vma->vm_flags |= VM_SOFTDIRTY; ++ ++ vma_set_page_prot(vma); ++ ++ validate_mm(mm); ++ return addr; ++ ++unmap_and_free_vma: ++ fput(vma->vm_file); ++ vma->vm_file = NULL; ++ ++ /* Undo any partial mapping done by a device driver. */ ++ unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end); ++ if (vm_flags & VM_SHARED) ++ mapping_unmap_writable(file->f_mapping); ++free_vma: ++ vm_area_free(vma); ++unacct_error: ++ if (charged) ++ vm_unacct_memory(charged); ++ validate_mm(mm); ++ return error; + } + + static int __vm_munmap(unsigned long start, size_t len, bool downgrade) +@@ -2852,11 +2774,12 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) + int ret; + struct mm_struct *mm = current->mm; + LIST_HEAD(uf); ++ MA_STATE(mas, &mm->mm_mt, start, start); + + if (mmap_write_lock_killable(mm)) + return -EINTR; + +- ret = __do_munmap(mm, start, len, &uf, downgrade); ++ ret = do_mas_munmap(&mas, mm, start, len, &uf, downgrade); + /* + * Returning 1 indicates mmap_lock is downgraded. + * But 1 is not legal return value of vm_munmap() and munmap(), reset +@@ -2922,11 +2845,12 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, + goto out; + + if (start + size > vma->vm_end) { +- struct vm_area_struct *next; ++ VMA_ITERATOR(vmi, mm, vma->vm_end); ++ struct vm_area_struct *next, *prev = vma; + +- for (next = vma->vm_next; next; next = next->vm_next) { ++ for_each_vma_range(vmi, next, start + size) { + /* hole between vmas ? */ +- if (next->vm_start != next->vm_prev->vm_end) ++ if (next->vm_start != prev->vm_end) + goto out; + + if (next->vm_file != vma->vm_file) +@@ -2935,8 +2859,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, + if (next->vm_flags != vma->vm_flags) + goto out; + +- if (start + size <= next->vm_end) +- break; ++ prev = next; + } + + if (!next) +@@ -2966,37 +2889,53 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, + } + + /* +- * this is really a simplified "do_mmap". it only handles +- * anonymous maps. eventually we may be able to do some +- * brk-specific accounting here. ++ * brk_munmap() - Unmap a parital vma. ++ * @mas: The maple tree state. ++ * @vma: The vma to be modified ++ * @newbrk: the start of the address to unmap ++ * @oldbrk: The end of the address to unmap ++ * @uf: The userfaultfd list_head ++ * ++ * Returns: 1 on success. ++ * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if ++ * possible. + */ +-static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf) ++static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long newbrk, unsigned long oldbrk, ++ struct list_head *uf) + { +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma, *prev; +- struct rb_node **rb_link, *rb_parent; +- pgoff_t pgoff = addr >> PAGE_SHIFT; +- int error; +- unsigned long mapped_addr; +- +- /* Until we need other flags, refuse anything except VM_EXEC. */ +- if ((flags & (~VM_EXEC)) != 0) +- return -EINVAL; +- flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; +- +- mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); +- if (IS_ERR_VALUE(mapped_addr)) +- return mapped_addr; ++ struct mm_struct *mm = vma->vm_mm; ++ int ret; + +- error = mlock_future_check(mm, mm->def_flags, len); +- if (error) +- return error; ++ arch_unmap(mm, newbrk, oldbrk); ++ ret = do_mas_align_munmap(mas, vma, mm, newbrk, oldbrk, uf, true); ++ validate_mm_mt(mm); ++ return ret; ++} + +- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ +- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) +- return -ENOMEM; ++/* ++ * do_brk_flags() - Increase the brk vma if the flags match. ++ * @mas: The maple tree state. ++ * @addr: The start address ++ * @len: The length of the increase ++ * @vma: The vma, ++ * @flags: The VMA Flags ++ * ++ * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags ++ * do not match then create a new anonymous VMA. Eventually we may be able to ++ * do some brk-specific accounting here. ++ */ ++static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long addr, unsigned long len, unsigned long flags) ++{ ++ struct mm_struct *mm = current->mm; + +- /* Check against address space limits *after* clearing old maps... */ ++ validate_mm_mt(mm); ++ /* ++ * Check against address space limits by the changed size ++ * Note: This happens *after* clearing old mappings in some code paths. ++ */ ++ flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; + if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) + return -ENOMEM; + +@@ -3006,28 +2945,49 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla + if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + return -ENOMEM; + +- /* Can we just expand an old private anonymous mapping? */ +- vma = vma_merge(mm, prev, addr, addr + len, flags, +- NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); +- if (vma) +- goto out; +- + /* +- * create a vma struct for an anonymous mapping ++ * Expand the existing vma if possible; Note that singular lists do not ++ * occur after forking, so the expand will only happen on new VMAs. + */ +- vma = vm_area_alloc(mm); +- if (!vma) { +- vm_unacct_memory(len >> PAGE_SHIFT); +- return -ENOMEM; ++ if (vma && ++ (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) && ++ ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) { ++ mas->index = vma->vm_start; ++ mas->last = addr + len - 1; ++ vma_adjust_trans_huge(vma, addr, addr + len, 0); ++ if (vma->anon_vma) { ++ anon_vma_lock_write(vma->anon_vma); ++ anon_vma_interval_tree_pre_update_vma(vma); ++ } ++ vma->vm_end = addr + len; ++ vma->vm_flags |= VM_SOFTDIRTY; ++ if (mas_store_gfp(mas, vma, GFP_KERNEL)) ++ goto mas_expand_failed; ++ ++ if (vma->anon_vma) { ++ anon_vma_interval_tree_post_update_vma(vma); ++ anon_vma_unlock_write(vma->anon_vma); ++ } ++ khugepaged_enter_vma(vma, flags); ++ goto out; + } + ++ /* create a vma struct for an anonymous mapping */ ++ vma = vm_area_alloc(mm); ++ if (!vma) ++ goto vma_alloc_fail; ++ + vma_set_anonymous(vma); + vma->vm_start = addr; + vma->vm_end = addr + len; +- vma->vm_pgoff = pgoff; ++ vma->vm_pgoff = addr >> PAGE_SHIFT; + vma->vm_flags = flags; + vma->vm_page_prot = vm_get_page_prot(flags); +- vma_link(mm, vma, prev, rb_link, rb_parent); ++ mas_set_range(mas, vma->vm_start, addr + len - 1); ++ if (mas_store_gfp(mas, vma, GFP_KERNEL)) ++ goto mas_store_fail; ++ ++ mm->map_count++; + out: + perf_event_mmap(vma); + mm->total_vm += len >> PAGE_SHIFT; +@@ -3035,16 +2995,32 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla + if (flags & VM_LOCKED) + mm->locked_vm += (len >> PAGE_SHIFT); + vma->vm_flags |= VM_SOFTDIRTY; ++ validate_mm(mm); + return 0; ++ ++mas_store_fail: ++ vm_area_free(vma); ++vma_alloc_fail: ++ vm_unacct_memory(len >> PAGE_SHIFT); ++ return -ENOMEM; ++ ++mas_expand_failed: ++ if (vma->anon_vma) { ++ anon_vma_interval_tree_post_update_vma(vma); ++ anon_vma_unlock_write(vma->anon_vma); ++ } ++ return -ENOMEM; + } + + int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) + { + struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma = NULL; + unsigned long len; + int ret; + bool populate; + LIST_HEAD(uf); ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + + len = PAGE_ALIGN(request); + if (len < request) +@@ -3055,13 +3031,36 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) + if (mmap_write_lock_killable(mm)) + return -EINTR; + +- ret = do_brk_flags(addr, len, flags, &uf); ++ /* Until we need other flags, refuse anything except VM_EXEC. */ ++ if ((flags & (~VM_EXEC)) != 0) ++ return -EINVAL; ++ ++ ret = check_brk_limits(addr, len); ++ if (ret) ++ goto limits_failed; ++ ++ ret = do_mas_munmap(&mas, mm, addr, len, &uf, 0); ++ if (ret) ++ goto munmap_failed; ++ ++ vma = mas_prev(&mas, 0); ++ if (!vma || vma->vm_end != addr || vma_policy(vma) || ++ !can_vma_merge_after(vma, flags, NULL, NULL, ++ addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) ++ vma = NULL; ++ ++ ret = do_brk_flags(&mas, vma, addr, len, flags); + populate = ((mm->def_flags & VM_LOCKED) != 0); + mmap_write_unlock(mm); + userfaultfd_unmap_complete(mm, &uf); + if (populate && !ret) + mm_populate(addr, len); + return ret; ++ ++munmap_failed: ++limits_failed: ++ mmap_write_unlock(mm); ++ return ret; + } + EXPORT_SYMBOL(vm_brk_flags); + +@@ -3077,6 +3076,8 @@ void exit_mmap(struct mm_struct *mm) + struct mmu_gather tlb; + struct vm_area_struct *vma; + unsigned long nr_accounted = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ int count = 0; + + /* mm's last user has gone, and its about to be pulled down */ + mmu_notifier_release(mm); +@@ -3101,7 +3102,7 @@ void exit_mmap(struct mm_struct *mm) + mmap_write_lock(mm); + arch_exit_mmap(mm); + +- vma = mm->mmap; ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) { + /* Can happen if dup_mmap() received an OOM */ + mmap_write_unlock(mm); +@@ -3112,19 +3113,29 @@ void exit_mmap(struct mm_struct *mm) + flush_cache_mm(mm); + tlb_gather_mmu_fullmm(&tlb, mm); + /* update_hiwater_rss(mm) here? but nobody should be looking */ +- /* Use -1 here to ensure all VMAs in the mm are unmapped */ +- unmap_vmas(&tlb, vma, 0, -1); +- free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); ++ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ ++ unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX); ++ free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS, ++ USER_PGTABLES_CEILING); + tlb_finish_mmu(&tlb); + +- /* Walk the list again, actually closing and freeing it. */ +- while (vma) { ++ /* ++ * Walk the list again, actually closing and freeing it, with preemption ++ * enabled, without holding any MM locks besides the unreachable ++ * mmap_write_lock. ++ */ ++ do { + if (vma->vm_flags & VM_ACCOUNT) + nr_accounted += vma_pages(vma); +- vma = remove_vma(vma); ++ remove_vma(vma); ++ count++; + cond_resched(); +- } +- mm->mmap = NULL; ++ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); ++ ++ BUG_ON(count != mm->map_count); ++ ++ trace_exit_mmap(mm); ++ __mt_destroy(&mm->mm_mt); + mmap_write_unlock(mm); + vm_unacct_memory(nr_accounted); + } +@@ -3135,14 +3146,14 @@ void exit_mmap(struct mm_struct *mm) + */ + int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) + { +- struct vm_area_struct *prev; +- struct rb_node **rb_link, *rb_parent; ++ unsigned long charged = vma_pages(vma); ++ + +- if (find_vma_links(mm, vma->vm_start, vma->vm_end, +- &prev, &rb_link, &rb_parent)) ++ if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) + return -ENOMEM; ++ + if ((vma->vm_flags & VM_ACCOUNT) && +- security_vm_enough_memory_mm(mm, vma_pages(vma))) ++ security_vm_enough_memory_mm(mm, charged)) + return -ENOMEM; + + /* +@@ -3162,7 +3173,11 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) + vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + } + +- vma_link(mm, vma, prev, rb_link, rb_parent); ++ if (vma_link(mm, vma)) { ++ vm_unacct_memory(charged); ++ return -ENOMEM; ++ } ++ + return 0; + } + +@@ -3178,9 +3193,9 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + unsigned long vma_start = vma->vm_start; + struct mm_struct *mm = vma->vm_mm; + struct vm_area_struct *new_vma, *prev; +- struct rb_node **rb_link, *rb_parent; + bool faulted_in_anon_vma = true; + ++ validate_mm_mt(mm); + /* + * If anonymous vma has not yet been faulted, update new pgoff + * to match new location, to increase its chance of merging. +@@ -3190,8 +3205,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + faulted_in_anon_vma = false; + } + +- if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) ++ new_vma = find_vma_prev(mm, addr, &prev); ++ if (new_vma && new_vma->vm_start < addr + len) + return NULL; /* should never get here */ ++ + new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, + vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), + vma->vm_userfaultfd_ctx, anon_vma_name(vma)); +@@ -3232,16 +3249,22 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + get_file(new_vma->vm_file); + if (new_vma->vm_ops && new_vma->vm_ops->open) + new_vma->vm_ops->open(new_vma); +- vma_link(mm, new_vma, prev, rb_link, rb_parent); ++ if (vma_link(mm, new_vma)) ++ goto out_vma_link; + *need_rmap_locks = false; + } ++ validate_mm_mt(mm); + return new_vma; + ++out_vma_link: ++ if (new_vma->vm_ops && new_vma->vm_ops->close) ++ new_vma->vm_ops->close(new_vma); + out_free_mempol: + mpol_put(vma_policy(new_vma)); + out_free_vma: + vm_area_free(new_vma); + out: ++ validate_mm_mt(mm); + return NULL; + } + +@@ -3378,6 +3401,7 @@ static struct vm_area_struct *__install_special_mapping( + int ret; + struct vm_area_struct *vma; + ++ validate_mm_mt(mm); + vma = vm_area_alloc(mm); + if (unlikely(vma == NULL)) + return ERR_PTR(-ENOMEM); +@@ -3400,10 +3424,12 @@ static struct vm_area_struct *__install_special_mapping( + + perf_event_mmap(vma); + ++ validate_mm_mt(mm); + return vma; + + out: + vm_area_free(vma); ++ validate_mm_mt(mm); + return ERR_PTR(ret); + } + +@@ -3528,12 +3554,13 @@ int mm_take_all_locks(struct mm_struct *mm) + { + struct vm_area_struct *vma; + struct anon_vma_chain *avc; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + mmap_assert_write_locked(mm); + + mutex_lock(&mm_all_locks_mutex); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (signal_pending(current)) + goto out_unlock; + if (vma->vm_file && vma->vm_file->f_mapping && +@@ -3541,7 +3568,8 @@ int mm_take_all_locks(struct mm_struct *mm) + vm_lock_mapping(mm, vma->vm_file->f_mapping); + } + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_set(&mas, 0); ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (signal_pending(current)) + goto out_unlock; + if (vma->vm_file && vma->vm_file->f_mapping && +@@ -3549,7 +3577,8 @@ int mm_take_all_locks(struct mm_struct *mm) + vm_lock_mapping(mm, vma->vm_file->f_mapping); + } + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_set(&mas, 0); ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (signal_pending(current)) + goto out_unlock; + if (vma->anon_vma) +@@ -3608,11 +3637,12 @@ void mm_drop_all_locks(struct mm_struct *mm) + { + struct vm_area_struct *vma; + struct anon_vma_chain *avc; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + mmap_assert_write_locked(mm); + BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (vma->anon_vma) + list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) + vm_unlock_anon_vma(avc->anon_vma); +diff --git a/mm/mprotect.c b/mm/mprotect.c +index bc6bddd156ca..bf030bbe7e97 100644 +--- a/mm/mprotect.c ++++ b/mm/mprotect.c +@@ -669,6 +669,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + const bool rier = (current->personality & READ_IMPLIES_EXEC) && + (prot & PROT_READ); + struct mmu_gather tlb; ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + start = untagged_addr(start); + +@@ -700,7 +701,8 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) + goto out; + +- vma = find_vma(current->mm, start); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + error = -ENOMEM; + if (!vma) + goto out; +@@ -726,7 +728,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + if (start > vma->vm_start) + prev = vma; + else +- prev = vma->vm_prev; ++ prev = mas_prev(&mas, 0); + + tlb_gather_mmu(&tlb, current->mm); + for (nstart = start ; ; ) { +@@ -789,7 +791,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + if (nstart >= end) + break; + +- vma = prev->vm_next; ++ vma = find_vma(current->mm, prev->vm_end); + if (!vma || vma->vm_start != nstart) { + error = -ENOMEM; + break; +diff --git a/mm/mremap.c b/mm/mremap.c +index b522cd0259a0..8644ff278f02 100644 +--- a/mm/mremap.c ++++ b/mm/mremap.c +@@ -716,7 +716,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, + if (excess) { + vma->vm_flags |= VM_ACCOUNT; + if (split) +- vma->vm_next->vm_flags |= VM_ACCOUNT; ++ find_vma(mm, vma->vm_end)->vm_flags |= VM_ACCOUNT; + } + + return new_addr; +@@ -866,9 +866,10 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, + static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) + { + unsigned long end = vma->vm_end + delta; ++ + if (end < vma->vm_end) /* overflow */ + return 0; +- if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ ++ if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) + return 0; + if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, + 0, MAP_FIXED) & ~PAGE_MASK) +@@ -975,20 +976,23 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, + /* + * Always allow a shrinking remap: that just unmaps + * the unnecessary pages.. +- * __do_munmap does all the needed commit accounting, and ++ * do_mas_munmap does all the needed commit accounting, and + * downgrades mmap_lock to read if so directed. + */ + if (old_len >= new_len) { + int retval; ++ MA_STATE(mas, &mm->mm_mt, addr + new_len, addr + new_len); + +- retval = __do_munmap(mm, addr+new_len, old_len - new_len, +- &uf_unmap, true); +- if (retval < 0 && old_len != new_len) { +- ret = retval; +- goto out; ++ retval = do_mas_munmap(&mas, mm, addr + new_len, ++ old_len - new_len, &uf_unmap, true); + /* Returning 1 indicates mmap_lock is downgraded to read. */ +- } else if (retval == 1) ++ if (retval == 1) { + downgraded = true; ++ } else if (retval < 0 && old_len != new_len) { ++ ret = retval; ++ goto out; ++ } ++ + ret = addr; + goto out; + } +diff --git a/mm/msync.c b/mm/msync.c +index 137d1c104f3e..ac4c9bfea2e7 100644 +--- a/mm/msync.c ++++ b/mm/msync.c +@@ -104,7 +104,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) + error = 0; + goto out_unlock; + } +- vma = vma->vm_next; ++ vma = find_vma(mm, vma->vm_end); + } + } + out_unlock: +diff --git a/mm/nommu.c b/mm/nommu.c +index e819cbc21b39..214c70e1d059 100644 +--- a/mm/nommu.c ++++ b/mm/nommu.c +@@ -19,7 +19,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -545,26 +544,27 @@ static void put_nommu_region(struct vm_region *region) + __put_nommu_region(region); + } + +-/* +- * add a VMA into a process's mm_struct in the appropriate place in the list +- * and tree and add to the address space's page tree also if not an anonymous +- * page +- * - should be called with mm->mmap_lock held writelocked +- */ +-static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) ++void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) + { +- struct vm_area_struct *pvma, *prev; +- struct address_space *mapping; +- struct rb_node **p, *parent, *rb_prev; ++ mas_set_range(mas, vma->vm_start, vma->vm_end - 1); ++ mas_store_prealloc(mas, vma); ++} + +- BUG_ON(!vma->vm_region); ++void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) ++{ ++ mas->index = vma->vm_start; ++ mas->last = vma->vm_end - 1; ++ mas_store_prealloc(mas, NULL); ++} + ++static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) ++{ + mm->map_count++; + vma->vm_mm = mm; + + /* add the VMA to the mapping */ + if (vma->vm_file) { +- mapping = vma->vm_file->f_mapping; ++ struct address_space *mapping = vma->vm_file->f_mapping; + + i_mmap_lock_write(mapping); + flush_dcache_mmap_lock(mapping); +@@ -572,67 +572,51 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) + flush_dcache_mmap_unlock(mapping); + i_mmap_unlock_write(mapping); + } ++} + +- /* add the VMA to the tree */ +- parent = rb_prev = NULL; +- p = &mm->mm_rb.rb_node; +- while (*p) { +- parent = *p; +- pvma = rb_entry(parent, struct vm_area_struct, vm_rb); +- +- /* sort by: start addr, end addr, VMA struct addr in that order +- * (the latter is necessary as we may get identical VMAs) */ +- if (vma->vm_start < pvma->vm_start) +- p = &(*p)->rb_left; +- else if (vma->vm_start > pvma->vm_start) { +- rb_prev = parent; +- p = &(*p)->rb_right; +- } else if (vma->vm_end < pvma->vm_end) +- p = &(*p)->rb_left; +- else if (vma->vm_end > pvma->vm_end) { +- rb_prev = parent; +- p = &(*p)->rb_right; +- } else if (vma < pvma) +- p = &(*p)->rb_left; +- else if (vma > pvma) { +- rb_prev = parent; +- p = &(*p)->rb_right; +- } else +- BUG(); +- } +- +- rb_link_node(&vma->vm_rb, parent, p); +- rb_insert_color(&vma->vm_rb, &mm->mm_rb); ++/* ++ * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm(). ++ * @mas: The maple state with preallocations. ++ * @mm: The mm_struct ++ * @vma: The vma to add ++ * ++ */ ++static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm, ++ struct vm_area_struct *vma) ++{ ++ BUG_ON(!vma->vm_region); + +- /* add VMA to the VMA list also */ +- prev = NULL; +- if (rb_prev) +- prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); ++ setup_vma_to_mm(vma, mm); + +- __vma_link_list(mm, vma, prev); ++ /* add the VMA to the tree */ ++ vma_mas_store(vma, mas); + } + + /* +- * delete a VMA from its owning mm_struct and address space ++ * add a VMA into a process's mm_struct in the appropriate place in the list ++ * and tree and add to the address space's page tree also if not an anonymous ++ * page ++ * - should be called with mm->mmap_lock held writelocked + */ +-static void delete_vma_from_mm(struct vm_area_struct *vma) +-{ +- int i; +- struct address_space *mapping; +- struct mm_struct *mm = vma->vm_mm; +- struct task_struct *curr = current; +- +- mm->map_count--; +- for (i = 0; i < VMACACHE_SIZE; i++) { +- /* if the vma is cached, invalidate the entire cache */ +- if (curr->vmacache.vmas[i] == vma) { +- vmacache_invalidate(mm); +- break; +- } ++static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) ++{ ++ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); ++ ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ pr_warn("Allocation of vma tree for process %d failed\n", ++ current->pid); ++ return -ENOMEM; + } ++ mas_add_vma_to_mm(&mas, mm, vma); ++ return 0; ++} + ++static void cleanup_vma_from_mm(struct vm_area_struct *vma) ++{ ++ vma->vm_mm->map_count--; + /* remove the VMA from the mapping */ + if (vma->vm_file) { ++ struct address_space *mapping; + mapping = vma->vm_file->f_mapping; + + i_mmap_lock_write(mapping); +@@ -641,11 +625,24 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) + flush_dcache_mmap_unlock(mapping); + i_mmap_unlock_write(mapping); + } ++} ++/* ++ * delete a VMA from its owning mm_struct and address space ++ */ ++static int delete_vma_from_mm(struct vm_area_struct *vma) ++{ ++ MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0); + +- /* remove from the MM's tree and list */ +- rb_erase(&vma->vm_rb, &mm->mm_rb); ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ pr_warn("Allocation of vma tree for process %d failed\n", ++ current->pid); ++ return -ENOMEM; ++ } ++ cleanup_vma_from_mm(vma); + +- __vma_unlink_list(mm, vma); ++ /* remove from the MM's tree and list */ ++ vma_mas_remove(vma, &mas); ++ return 0; + } + + /* +@@ -661,31 +658,26 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) + vm_area_free(vma); + } + ++struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, ++ unsigned long start_addr, ++ unsigned long end_addr) ++{ ++ unsigned long index = start_addr; ++ ++ mmap_assert_locked(mm); ++ return mt_find(&mm->mm_mt, &index, end_addr - 1); ++} ++EXPORT_SYMBOL(find_vma_intersection); ++ + /* + * look up the first VMA in which addr resides, NULL if none + * - should be called with mm->mmap_lock at least held readlocked + */ + struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) + { +- struct vm_area_struct *vma; ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + +- /* check the cache first */ +- vma = vmacache_find(mm, addr); +- if (likely(vma)) +- return vma; +- +- /* trawl the list (there may be multiple mappings in which addr +- * resides) */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) { +- if (vma->vm_start > addr) +- return NULL; +- if (vma->vm_end > addr) { +- vmacache_update(addr, vma); +- return vma; +- } +- } +- +- return NULL; ++ return mas_walk(&mas); + } + EXPORT_SYMBOL(find_vma); + +@@ -717,26 +709,17 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, + { + struct vm_area_struct *vma; + unsigned long end = addr + len; ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + +- /* check the cache first */ +- vma = vmacache_find_exact(mm, addr, end); +- if (vma) +- return vma; +- +- /* trawl the list (there may be multiple mappings in which addr +- * resides) */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) { +- if (vma->vm_start < addr) +- continue; +- if (vma->vm_start > addr) +- return NULL; +- if (vma->vm_end == end) { +- vmacache_update(addr, vma); +- return vma; +- } +- } ++ vma = mas_walk(&mas); ++ if (!vma) ++ return NULL; ++ if (vma->vm_start != addr) ++ return NULL; ++ if (vma->vm_end != end) ++ return NULL; + +- return NULL; ++ return vma; + } + + /* +@@ -1069,6 +1052,7 @@ unsigned long do_mmap(struct file *file, + vm_flags_t vm_flags; + unsigned long capabilities, result; + int ret; ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + *populate = 0; + +@@ -1087,6 +1071,7 @@ unsigned long do_mmap(struct file *file, + * now know into VMA flags */ + vm_flags = determine_vm_flags(file, prot, flags, capabilities); + ++ + /* we're going to need to record the mapping */ + region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); + if (!region) +@@ -1096,6 +1081,9 @@ unsigned long do_mmap(struct file *file, + if (!vma) + goto error_getting_vma; + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ goto error_maple_preallocate; ++ + region->vm_usage = 1; + region->vm_flags = vm_flags; + region->vm_pgoff = pgoff; +@@ -1236,7 +1224,7 @@ unsigned long do_mmap(struct file *file, + current->mm->total_vm += len >> PAGE_SHIFT; + + share: +- add_vma_to_mm(current->mm, vma); ++ mas_add_vma_to_mm(&mas, current->mm, vma); + + /* we flush the region from the icache only when the first executable + * mapping of it is made */ +@@ -1262,6 +1250,7 @@ unsigned long do_mmap(struct file *file, + + sharing_violation: + up_write(&nommu_region_sem); ++ mas_destroy(&mas); + pr_warn("Attempt to share mismatched mappings\n"); + ret = -EINVAL; + goto error; +@@ -1278,6 +1267,14 @@ unsigned long do_mmap(struct file *file, + len, current->pid); + show_free_areas(0, NULL); + return -ENOMEM; ++ ++error_maple_preallocate: ++ kmem_cache_free(vm_region_jar, region); ++ vm_area_free(vma); ++ pr_warn("Allocation of vma tree for process %d failed\n", current->pid); ++ show_free_areas(0, NULL); ++ return -ENOMEM; ++ + } + + unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +@@ -1343,6 +1340,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *new; + struct vm_region *region; + unsigned long npages; ++ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); + + /* we're only permitted to split anonymous regions (these should have + * only a single usage on the region) */ +@@ -1357,9 +1355,13 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + return -ENOMEM; + + new = vm_area_dup(vma); +- if (!new) { +- kmem_cache_free(vm_region_jar, region); +- return -ENOMEM; ++ if (!new) ++ goto err_vma_dup; ++ ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ pr_warn("Allocation of vma tree for process %d failed\n", ++ current->pid); ++ goto err_mas_preallocate; + } + + /* most fields are the same, copy all, and then fixup */ +@@ -1378,7 +1380,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + if (new->vm_ops && new->vm_ops->open) + new->vm_ops->open(new); + +- delete_vma_from_mm(vma); + down_write(&nommu_region_sem); + delete_nommu_region(vma->vm_region); + if (new_below) { +@@ -1391,9 +1392,19 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + add_nommu_region(vma->vm_region); + add_nommu_region(new->vm_region); + up_write(&nommu_region_sem); +- add_vma_to_mm(mm, vma); +- add_vma_to_mm(mm, new); ++ ++ setup_vma_to_mm(vma, mm); ++ setup_vma_to_mm(new, mm); ++ mas_set_range(&mas, vma->vm_start, vma->vm_end - 1); ++ mas_store(&mas, vma); ++ vma_mas_store(new, &mas); + return 0; ++ ++err_mas_preallocate: ++ vm_area_free(new); ++err_vma_dup: ++ kmem_cache_free(vm_region_jar, region); ++ return -ENOMEM; + } + + /* +@@ -1408,12 +1419,14 @@ static int shrink_vma(struct mm_struct *mm, + + /* adjust the VMA's pointers, which may reposition it in the MM's tree + * and list */ +- delete_vma_from_mm(vma); ++ if (delete_vma_from_mm(vma)) ++ return -ENOMEM; + if (from > vma->vm_start) + vma->vm_end = from; + else + vma->vm_start = to; +- add_vma_to_mm(mm, vma); ++ if (add_vma_to_mm(mm, vma)) ++ return -ENOMEM; + + /* cut the backing region down to size */ + region = vma->vm_region; +@@ -1441,9 +1454,10 @@ static int shrink_vma(struct mm_struct *mm, + */ + int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) + { ++ MA_STATE(mas, &mm->mm_mt, start, start); + struct vm_area_struct *vma; + unsigned long end; +- int ret; ++ int ret = 0; + + len = PAGE_ALIGN(len); + if (len == 0) +@@ -1452,7 +1466,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list + end = start + len; + + /* find the first potentially overlapping VMA */ +- vma = find_vma(mm, start); ++ vma = mas_find(&mas, end - 1); + if (!vma) { + static int limit; + if (limit < 5) { +@@ -1471,7 +1485,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list + return -EINVAL; + if (end == vma->vm_end) + goto erase_whole_vma; +- vma = vma->vm_next; ++ vma = mas_next(&mas, end - 1); + } while (vma); + return -EINVAL; + } else { +@@ -1493,9 +1507,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list + } + + erase_whole_vma: +- delete_vma_from_mm(vma); ++ if (delete_vma_from_mm(vma)) ++ ret = -ENOMEM; + delete_vma(mm, vma); +- return 0; ++ return ret; + } + + int vm_munmap(unsigned long addr, size_t len) +@@ -1520,6 +1535,7 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) + */ + void exit_mmap(struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + + if (!mm) +@@ -1527,12 +1543,18 @@ void exit_mmap(struct mm_struct *mm) + + mm->total_vm = 0; + +- while ((vma = mm->mmap)) { +- mm->mmap = vma->vm_next; +- delete_vma_from_mm(vma); ++ /* ++ * Lock the mm to avoid assert complaining even though this is the only ++ * user of the mm ++ */ ++ mmap_write_lock(mm); ++ for_each_vma(vmi, vma) { ++ cleanup_vma_from_mm(vma); + delete_vma(mm, vma); + cond_resched(); + } ++ __mt_destroy(&mm->mm_mt); ++ mmap_write_unlock(mm); + } + + int vm_brk(unsigned long addr, unsigned long len) +diff --git a/mm/oom_kill.c b/mm/oom_kill.c +index 3c6cf9e3cd66..3996301450e8 100644 +--- a/mm/oom_kill.c ++++ b/mm/oom_kill.c +@@ -513,6 +513,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) + { + struct vm_area_struct *vma; + bool ret = true; ++ VMA_ITERATOR(vmi, mm, 0); + + /* + * Tell all users of get_user/copy_from_user etc... that the content +@@ -522,7 +523,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) + */ + set_bit(MMF_UNSTABLE, &mm->flags); + +- for (vma = mm->mmap ; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) + continue; + +diff --git a/mm/pagewalk.c b/mm/pagewalk.c +index fa7a3d21a751..c09bb9c01f7e 100644 +--- a/mm/pagewalk.c ++++ b/mm/pagewalk.c +@@ -460,7 +460,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, + } else { /* inside vma */ + walk.vma = vma; + next = min(end, vma->vm_end); +- vma = vma->vm_next; ++ vma = find_vma(mm, vma->vm_end); + + err = walk_page_test(start, next, &walk); + if (err > 0) { +diff --git a/mm/swapfile.c b/mm/swapfile.c +index 1fdccd2f1422..5c8681a3f1d9 100644 +--- a/mm/swapfile.c ++++ b/mm/swapfile.c +@@ -1990,14 +1990,16 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type) + { + struct vm_area_struct *vma; + int ret = 0; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (vma->anon_vma) { + ret = unuse_vma(vma, type); + if (ret) + break; + } ++ + cond_resched(); + } + mmap_read_unlock(mm); +diff --git a/mm/util.c b/mm/util.c +index 346e40177bc6..50427596f208 100644 +--- a/mm/util.c ++++ b/mm/util.c +@@ -272,38 +272,6 @@ void *memdup_user_nul(const void __user *src, size_t len) + } + EXPORT_SYMBOL(memdup_user_nul); + +-void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev) +-{ +- struct vm_area_struct *next; +- +- vma->vm_prev = prev; +- if (prev) { +- next = prev->vm_next; +- prev->vm_next = vma; +- } else { +- next = mm->mmap; +- mm->mmap = vma; +- } +- vma->vm_next = next; +- if (next) +- next->vm_prev = vma; +-} +- +-void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma) +-{ +- struct vm_area_struct *prev, *next; +- +- next = vma->vm_next; +- prev = vma->vm_prev; +- if (prev) +- prev->vm_next = next; +- else +- mm->mmap = next; +- if (next) +- next->vm_prev = prev; +-} +- + /* Check if the vma is being used as a stack by this task */ + int vma_is_stack_for_current(struct vm_area_struct *vma) + { +diff --git a/mm/vmacache.c b/mm/vmacache.c +deleted file mode 100644 +index 01a6e6688ec1..000000000000 +--- a/mm/vmacache.c ++++ /dev/null +@@ -1,117 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0 +-/* +- * Copyright (C) 2014 Davidlohr Bueso. +- */ +-#include +-#include +-#include +-#include +- +-/* +- * Hash based on the pmd of addr if configured with MMU, which provides a good +- * hit rate for workloads with spatial locality. Otherwise, use pages. +- */ +-#ifdef CONFIG_MMU +-#define VMACACHE_SHIFT PMD_SHIFT +-#else +-#define VMACACHE_SHIFT PAGE_SHIFT +-#endif +-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK) +- +-/* +- * This task may be accessing a foreign mm via (for example) +- * get_user_pages()->find_vma(). The vmacache is task-local and this +- * task's vmacache pertains to a different mm (ie, its own). There is +- * nothing we can do here. +- * +- * Also handle the case where a kernel thread has adopted this mm via +- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm. +- */ +-static inline bool vmacache_valid_mm(struct mm_struct *mm) +-{ +- return current->mm == mm && !(current->flags & PF_KTHREAD); +-} +- +-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma) +-{ +- if (vmacache_valid_mm(newvma->vm_mm)) +- current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma; +-} +- +-static bool vmacache_valid(struct mm_struct *mm) +-{ +- struct task_struct *curr; +- +- if (!vmacache_valid_mm(mm)) +- return false; +- +- curr = current; +- if (mm->vmacache_seqnum != curr->vmacache.seqnum) { +- /* +- * First attempt will always be invalid, initialize +- * the new cache for this task here. +- */ +- curr->vmacache.seqnum = mm->vmacache_seqnum; +- vmacache_flush(curr); +- return false; +- } +- return true; +-} +- +-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr) +-{ +- int idx = VMACACHE_HASH(addr); +- int i; +- +- count_vm_vmacache_event(VMACACHE_FIND_CALLS); +- +- if (!vmacache_valid(mm)) +- return NULL; +- +- for (i = 0; i < VMACACHE_SIZE; i++) { +- struct vm_area_struct *vma = current->vmacache.vmas[idx]; +- +- if (vma) { +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- if (WARN_ON_ONCE(vma->vm_mm != mm)) +- break; +-#endif +- if (vma->vm_start <= addr && vma->vm_end > addr) { +- count_vm_vmacache_event(VMACACHE_FIND_HITS); +- return vma; +- } +- } +- if (++idx == VMACACHE_SIZE) +- idx = 0; +- } +- +- return NULL; +-} +- +-#ifndef CONFIG_MMU +-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, +- unsigned long start, +- unsigned long end) +-{ +- int idx = VMACACHE_HASH(start); +- int i; +- +- count_vm_vmacache_event(VMACACHE_FIND_CALLS); +- +- if (!vmacache_valid(mm)) +- return NULL; +- +- for (i = 0; i < VMACACHE_SIZE; i++) { +- struct vm_area_struct *vma = current->vmacache.vmas[idx]; +- +- if (vma && vma->vm_start == start && vma->vm_end == end) { +- count_vm_vmacache_event(VMACACHE_FIND_HITS); +- return vma; +- } +- if (++idx == VMACACHE_SIZE) +- idx = 0; +- } +- +- return NULL; +-} +-#endif +diff --git a/mm/vmscan.c b/mm/vmscan.c +index bedf4ae55c31..1e66f8cfc03d 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -3749,23 +3749,14 @@ static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk + { + unsigned long start = round_up(*vm_end, size); + unsigned long end = (start | ~mask) + 1; ++ VMA_ITERATOR(vmi, args->mm, start); + + VM_WARN_ON_ONCE(mask & size); + VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); + +- while (args->vma) { +- if (start >= args->vma->vm_end) { +- args->vma = args->vma->vm_next; ++ for_each_vma_range(vmi, args->vma, end) { ++ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) + continue; +- } +- +- if (end && end <= args->vma->vm_start) +- return false; +- +- if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { +- args->vma = args->vma->vm_next; +- continue; +- } + + *vm_start = max(start, args->vma->vm_start); + *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 90af9a8572f5..4b85d3d5f7d7 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -1389,10 +1389,6 @@ const char * const vmstat_text[] = { + "nr_tlb_local_flush_one", + #endif /* CONFIG_DEBUG_TLBFLUSH */ + +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- "vmacache_find_calls", +- "vmacache_find_hits", +-#endif + #ifdef CONFIG_SWAP + "swap_ra", + "swap_ra_hit", +diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h +index 0616409513eb..311759ea25e9 100644 +--- a/tools/include/linux/slab.h ++++ b/tools/include/linux/slab.h +@@ -41,4 +41,8 @@ struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, + unsigned int align, unsigned int flags, + void (*ctor)(void *)); + ++void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); ++int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, ++ void **list); ++ + #endif /* _TOOLS_SLAB_H */ +diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore +index d971516401e6..c901d96dd013 100644 +--- a/tools/testing/radix-tree/.gitignore ++++ b/tools/testing/radix-tree/.gitignore +@@ -6,3 +6,5 @@ main + multiorder + radix-tree.c + xarray ++maple ++ma_xa_benchmark +diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile +index c4ea4fbb0bfc..89d613e0505b 100644 +--- a/tools/testing/radix-tree/Makefile ++++ b/tools/testing/radix-tree/Makefile +@@ -4,9 +4,9 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ + -fsanitize=undefined + LDFLAGS += -fsanitize=address -fsanitize=undefined + LDLIBS+= -lpthread -lurcu +-TARGETS = main idr-test multiorder xarray ++TARGETS = main idr-test multiorder xarray maple + CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \ +- slab.o ++ slab.o maple.o + OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ + regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ + iteration_check_2.o benchmark.o +@@ -29,6 +29,8 @@ idr-test: idr-test.o $(CORE_OFILES) + + xarray: $(CORE_OFILES) + ++maple: $(CORE_OFILES) ++ + multiorder: multiorder.o $(CORE_OFILES) + + clean: +@@ -40,6 +42,7 @@ $(OFILES): Makefile *.h */*.h generated/map-shift.h \ + ../../include/linux/*.h \ + ../../include/asm/*.h \ + ../../../include/linux/xarray.h \ ++ ../../../include/linux/maple_tree.h \ + ../../../include/linux/radix-tree.h \ + ../../../include/linux/idr.h + +@@ -51,6 +54,8 @@ idr.c: ../../../lib/idr.c + + xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c + ++maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c ++ + generated/map-shift.h: + @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ + echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ +diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h +index 2218b3cc184e..e7da80350236 100644 +--- a/tools/testing/radix-tree/generated/autoconf.h ++++ b/tools/testing/radix-tree/generated/autoconf.h +@@ -1 +1,2 @@ + #define CONFIG_XARRAY_MULTI 1 ++#define CONFIG_64BIT 1 +diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c +index d5c1bcba86fe..2048d12c31df 100644 +--- a/tools/testing/radix-tree/linux.c ++++ b/tools/testing/radix-tree/linux.c +@@ -23,15 +23,47 @@ struct kmem_cache { + int nr_objs; + void *objs; + void (*ctor)(void *); ++ unsigned int non_kernel; ++ unsigned long nr_allocated; ++ unsigned long nr_tallocated; + }; + ++void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) ++{ ++ cachep->non_kernel = val; ++} ++ ++unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) ++{ ++ return cachep->size * cachep->nr_allocated; ++} ++ ++unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) ++{ ++ return cachep->nr_allocated; ++} ++ ++unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) ++{ ++ return cachep->nr_tallocated; ++} ++ ++void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) ++{ ++ cachep->nr_tallocated = 0; ++} ++ + void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, + int gfp) + { + void *p; + +- if (!(gfp & __GFP_DIRECT_RECLAIM)) +- return NULL; ++ if (!(gfp & __GFP_DIRECT_RECLAIM)) { ++ if (!cachep->non_kernel) ++ return NULL; ++ ++ cachep->non_kernel--; ++ } + + pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs) { +@@ -53,19 +85,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, + memset(p, 0, cachep->size); + } + ++ uatomic_inc(&cachep->nr_allocated); + uatomic_inc(&nr_allocated); ++ uatomic_inc(&cachep->nr_tallocated); + if (kmalloc_verbose) + printf("Allocating %p from slab\n", p); + return p; + } + +-void kmem_cache_free(struct kmem_cache *cachep, void *objp) ++void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) + { + assert(objp); + uatomic_dec(&nr_allocated); ++ uatomic_dec(&cachep->nr_allocated); + if (kmalloc_verbose) + printf("Freeing %p to slab\n", objp); +- pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs > 10 || cachep->align) { + memset(objp, POISON_FREE, cachep->size); + free(objp); +@@ -75,9 +109,80 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) + node->parent = cachep->objs; + cachep->objs = node; + } ++} ++ ++void kmem_cache_free(struct kmem_cache *cachep, void *objp) ++{ ++ pthread_mutex_lock(&cachep->lock); ++ kmem_cache_free_locked(cachep, objp); + pthread_mutex_unlock(&cachep->lock); + } + ++void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) ++{ ++ if (kmalloc_verbose) ++ pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); ++ ++ pthread_mutex_lock(&cachep->lock); ++ for (int i = 0; i < size; i++) ++ kmem_cache_free_locked(cachep, list[i]); ++ pthread_mutex_unlock(&cachep->lock); ++} ++ ++int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, ++ void **p) ++{ ++ size_t i; ++ ++ if (kmalloc_verbose) ++ pr_debug("Bulk alloc %lu\n", size); ++ ++ if (!(gfp & __GFP_DIRECT_RECLAIM)) { ++ if (cachep->non_kernel < size) ++ return 0; ++ ++ cachep->non_kernel -= size; ++ } ++ ++ pthread_mutex_lock(&cachep->lock); ++ if (cachep->nr_objs >= size) { ++ struct radix_tree_node *node; ++ ++ for (i = 0; i < size; i++) { ++ node = cachep->objs; ++ cachep->nr_objs--; ++ cachep->objs = node->parent; ++ p[i] = node; ++ node->parent = NULL; ++ } ++ pthread_mutex_unlock(&cachep->lock); ++ } else { ++ pthread_mutex_unlock(&cachep->lock); ++ for (i = 0; i < size; i++) { ++ if (cachep->align) { ++ posix_memalign(&p[i], cachep->align, ++ cachep->size * size); ++ } else { ++ p[i] = malloc(cachep->size * size); ++ } ++ if (cachep->ctor) ++ cachep->ctor(p[i]); ++ else if (gfp & __GFP_ZERO) ++ memset(p[i], 0, cachep->size); ++ } ++ } ++ ++ for (i = 0; i < size; i++) { ++ uatomic_inc(&nr_allocated); ++ uatomic_inc(&cachep->nr_allocated); ++ uatomic_inc(&cachep->nr_tallocated); ++ if (kmalloc_verbose) ++ printf("Allocating %p from slab\n", p[i]); ++ } ++ ++ return size; ++} ++ + struct kmem_cache * + kmem_cache_create(const char *name, unsigned int size, unsigned int align, + unsigned int flags, void (*ctor)(void *)) +@@ -88,7 +193,54 @@ kmem_cache_create(const char *name, unsigned int size, unsigned int align, + ret->size = size; + ret->align = align; + ret->nr_objs = 0; ++ ret->nr_allocated = 0; ++ ret->nr_tallocated = 0; + ret->objs = NULL; + ret->ctor = ctor; ++ ret->non_kernel = 0; + return ret; + } ++ ++/* ++ * Test the test infrastructure for kem_cache_alloc/free and bulk counterparts. ++ */ ++void test_kmem_cache_bulk(void) ++{ ++ int i; ++ void *list[12]; ++ static struct kmem_cache *test_cache, *test_cache2; ++ ++ /* ++ * Testing the bulk allocators without aligned kmem_cache to force the ++ * bulk alloc/free to reuse ++ */ ++ test_cache = kmem_cache_create("test_cache", 256, 0, SLAB_PANIC, NULL); ++ ++ for (i = 0; i < 5; i++) ++ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); ++ ++ for (i = 0; i < 5; i++) ++ kmem_cache_free(test_cache, list[i]); ++ assert(test_cache->nr_objs == 5); ++ ++ kmem_cache_alloc_bulk(test_cache, __GFP_DIRECT_RECLAIM, 5, list); ++ kmem_cache_free_bulk(test_cache, 5, list); ++ ++ for (i = 0; i < 12 ; i++) ++ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); ++ ++ for (i = 0; i < 12; i++) ++ kmem_cache_free(test_cache, list[i]); ++ ++ /* The last free will not be kept around */ ++ assert(test_cache->nr_objs == 11); ++ ++ /* Aligned caches will immediately free */ ++ test_cache2 = kmem_cache_create("test_cache2", 128, 128, SLAB_PANIC, NULL); ++ ++ kmem_cache_alloc_bulk(test_cache2, __GFP_DIRECT_RECLAIM, 10, list); ++ kmem_cache_free_bulk(test_cache2, 10, list); ++ assert(!test_cache2->nr_objs); ++ ++ ++} +diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h +index 39867fd80c8f..c5c9d05f29da 100644 +--- a/tools/testing/radix-tree/linux/kernel.h ++++ b/tools/testing/radix-tree/linux/kernel.h +@@ -14,6 +14,7 @@ + #include "../../../include/linux/kconfig.h" + + #define printk printf ++#define pr_err printk + #define pr_info printk + #define pr_debug printk + #define pr_cont printk +diff --git a/tools/testing/radix-tree/linux/lockdep.h b/tools/testing/radix-tree/linux/lockdep.h +index 016cff473cfc..62473ab57f99 100644 +--- a/tools/testing/radix-tree/linux/lockdep.h ++++ b/tools/testing/radix-tree/linux/lockdep.h +@@ -11,4 +11,6 @@ static inline void lockdep_set_class(spinlock_t *lock, + struct lock_class_key *key) + { + } ++ ++extern int lockdep_is_held(const void *); + #endif /* _LINUX_LOCKDEP_H */ +diff --git a/tools/testing/radix-tree/linux/maple_tree.h b/tools/testing/radix-tree/linux/maple_tree.h +new file mode 100644 +index 000000000000..7d8d1f445b89 +--- /dev/null ++++ b/tools/testing/radix-tree/linux/maple_tree.h +@@ -0,0 +1,7 @@ ++/* SPDX-License-Identifier: GPL-2.0+ */ ++#define atomic_t int32_t ++#include "../../../../include/linux/maple_tree.h" ++#define atomic_inc(x) uatomic_inc(x) ++#define atomic_read(x) uatomic_read(x) ++#define atomic_set(x, y) do {} while (0) ++#define U8_MAX UCHAR_MAX +diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c +new file mode 100644 +index 000000000000..35082671928a +--- /dev/null ++++ b/tools/testing/radix-tree/maple.c +@@ -0,0 +1,59 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++/* ++ * maple_tree.c: Userspace shim for maple tree test-suite ++ * Copyright (c) 2018 Liam R. Howlett ++ */ ++ ++#define CONFIG_DEBUG_MAPLE_TREE ++#define CONFIG_MAPLE_SEARCH ++#include "test.h" ++ ++#define module_init(x) ++#define module_exit(x) ++#define MODULE_AUTHOR(x) ++#define MODULE_LICENSE(x) ++#define dump_stack() assert(0) ++ ++#include "../../../lib/maple_tree.c" ++#undef CONFIG_DEBUG_MAPLE_TREE ++#include "../../../lib/test_maple_tree.c" ++ ++void farmer_tests(void) ++{ ++ struct maple_node *node; ++ DEFINE_MTREE(tree); ++ ++ mt_dump(&tree); ++ ++ tree.ma_root = xa_mk_value(0); ++ mt_dump(&tree); ++ ++ node = mt_alloc_one(GFP_KERNEL); ++ node->parent = (void *)((unsigned long)(&tree) | 1); ++ node->slot[0] = xa_mk_value(0); ++ node->slot[1] = xa_mk_value(1); ++ node->mr64.pivot[0] = 0; ++ node->mr64.pivot[1] = 1; ++ node->mr64.pivot[2] = 0; ++ tree.ma_root = mt_mk_node(node, maple_leaf_64); ++ mt_dump(&tree); ++ ++ ma_free_rcu(node); ++} ++ ++void maple_tree_tests(void) ++{ ++ farmer_tests(); ++ maple_tree_seed(); ++ maple_tree_harvest(); ++} ++ ++int __weak main(void) ++{ ++ maple_tree_init(); ++ maple_tree_tests(); ++ rcu_barrier(); ++ if (nr_allocated) ++ printf("nr_allocated = %d\n", nr_allocated); ++ return 0; ++} +diff --git a/tools/testing/radix-tree/trace/events/maple_tree.h b/tools/testing/radix-tree/trace/events/maple_tree.h +new file mode 100644 +index 000000000000..97d0e1ddcf08 +--- /dev/null ++++ b/tools/testing/radix-tree/trace/events/maple_tree.h +@@ -0,0 +1,5 @@ ++/* SPDX-License-Identifier: GPL-2.0+ */ ++ ++#define trace_ma_op(a, b) do {} while (0) ++#define trace_ma_read(a, b) do {} while (0) ++#define trace_ma_write(a, b, c, d) do {} while (0) +-- +2.38.0 + +From bc98fba95c9e9b188bcae7ab9cdb292751810fe8 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:26:01 +0200 +Subject: [PATCH 07/17] mm/demotion: Memory tiers and demotion + +The current kernel has the basic memory tiering support: Inactive pages on +a higher tier NUMA node can be migrated (demoted) to a lower tier NUMA +node to make room for new allocations on the higher tier NUMA node. +Frequently accessed pages on a lower tier NUMA node can be migrated +(promoted) to a higher tier NUMA node to improve the performance. + +In the current kernel, memory tiers are defined implicitly via a demotion +path relationship between NUMA nodes, which is created during the kernel +initialization and updated when a NUMA node is hot-added or hot-removed. +The current implementation puts all nodes with CPU into the highest tier, +and builds the tier hierarchy tier-by-tier by establishing the per-node +demotion targets based on the distances between nodes. + +This current memory tier kernel implementation needs to be improved for +several important use cases: + +* The current tier initialization code always initializes each + memory-only NUMA node into a lower tier. But a memory-only NUMA node + may have a high performance memory device (e.g. a DRAM-backed + memory-only node on a virtual machine) and that should be put into a + higher tier. + +* The current tier hierarchy always puts CPU nodes into the top tier. + But on a system with HBM (e.g. GPU memory) devices, these memory-only + HBM NUMA nodes should be in the top tier, and DRAM nodes with CPUs are + better to be placed into the next lower tier. + +* Also because the current tier hierarchy always puts CPU nodes into the + top tier, when a CPU is hot-added (or hot-removed) and triggers a memory + node from CPU-less into a CPU node (or vice versa), the memory tier + hierarchy gets changed, even though no memory node is added or removed. + This can make the tier hierarchy unstable and make it difficult to + support tier-based memory accounting. + +* A higher tier node can only be demoted to nodes with shortest distance + on the next lower tier as defined by the demotion path, not any other + node from any lower tier. This strict, demotion order does not work in + all use cases (e.g. some use cases may want to allow cross-socket + demotion to another node in the same demotion tier as a fallback when + the preferred demotion node is out of space), and has resulted in the + feature request for an interface to override the system-wide, per-node + demotion order from the userspace. This demotion order is also + inconsistent with the page allocation fallback order when all the nodes + in a higher tier are out of space: The page allocation can fall back to + any node from any lower tier, whereas the demotion order doesn't allow + that. + +This patch series make the creation of memory tiers explicit under the +control of device driver. + +Memory Tier Initialization +========================== + +Linux kernel presents memory devices as NUMA nodes and each memory device +is of a specific type. The memory type of a device is represented by its +abstract distance. A memory tier corresponds to a range of abstract +distance. This allows for classifying memory devices with a specific +performance range into a memory tier. + +By default, all memory nodes are assigned to the default tier with +abstract distance 512. + +A device driver can move its memory nodes from the default tier. For +example, PMEM can move its memory nodes below the default tier, whereas +GPU can move its memory nodes above the default tier. + +The kernel initialization code makes the decision on which exact tier a +memory node should be assigned to based on the requests from the device +drivers as well as the memory device hardware information provided by the +firmware. + +Hot-adding/removing CPUs doesn't affect memory tier hierarchy. + +This patch (of 10): + +In the current kernel, memory tiers are defined implicitly via a demotion +path relationship between NUMA nodes, which is created during the kernel +initialization and updated when a NUMA node is hot-added or hot-removed. +The current implementation puts all nodes with CPU into the highest tier, +and builds the tier hierarchy by establishing the per-node demotion +targets based on the distances between nodes. + +This current memory tier kernel implementation needs to be improved for +several important use cases, + +The current tier initialization code always initializes each memory-only +NUMA node into a lower tier. But a memory-only NUMA node may have a high +performance memory device (e.g. a DRAM-backed memory-only node on a +virtual machine) that should be put into a higher tier. + +The current tier hierarchy always puts CPU nodes into the top tier. But +on a system with HBM or GPU devices, the memory-only NUMA nodes mapping +these devices should be in the top tier, and DRAM nodes with CPUs are +better to be placed into the next lower tier. + +With current kernel higher tier node can only be demoted to nodes with +shortest distance on the next lower tier as defined by the demotion path, +not any other node from any lower tier. This strict, demotion order does +not work in all use cases (e.g. some use cases may want to allow +cross-socket demotion to another node in the same demotion tier as a +fallback when the preferred demotion node is out of space), This demotion +order is also inconsistent with the page allocation fallback order when +all the nodes in a higher tier are out of space: The page allocation can +fall back to any node from any lower tier, whereas the demotion order +doesn't allow that. + +This patch series address the above by defining memory tiers explicitly. + +Linux kernel presents memory devices as NUMA nodes and each memory device +is of a specific type. The memory type of a device is represented by its +abstract distance. A memory tier corresponds to a range of abstract +distance. This allows for classifying memory devices with a specific +performance range into a memory tier. + +This patch configures the range/chunk size to be 128. The default DRAM +abstract distance is 512. We can have 4 memory tiers below the default +DRAM with abstract distance range 0 - 127, 127 - 255, 256- 383, 384 - 511. +Faster memory devices can be placed in these faster(higher) memory tiers. +Slower memory devices like persistent memory will have abstract distance +higher than the default DRAM level. + +Signed-off-by: Peter Jung +--- + .../ABI/testing/sysfs-kernel-mm-memory-tiers | 25 + + drivers/dax/kmem.c | 42 +- + include/linux/memory-tiers.h | 102 +++ + include/linux/migrate.h | 15 - + include/linux/mmzone.h | 3 + + include/linux/node.h | 5 - + include/linux/nodemask.h | 15 +- + kernel/sched/fair.c | 1 + + mm/Makefile | 1 + + mm/huge_memory.c | 1 + + mm/memory-tiers.c | 732 ++++++++++++++++++ + mm/memory.c | 1 + + mm/migrate.c | 453 +---------- + mm/mprotect.c | 1 + + mm/vmscan.c | 59 +- + mm/vmstat.c | 4 - + 16 files changed, 963 insertions(+), 497 deletions(-) + create mode 100644 Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers + create mode 100644 include/linux/memory-tiers.h + create mode 100644 mm/memory-tiers.c + +diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +new file mode 100644 +index 000000000000..45985e411f13 +--- /dev/null ++++ b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +@@ -0,0 +1,25 @@ ++What: /sys/devices/virtual/memory_tiering/ ++Date: August 2022 ++Contact: Linux memory management mailing list ++Description: A collection of all the memory tiers allocated. ++ ++ Individual memory tier details are contained in subdirectories ++ named by the abstract distance of the memory tier. ++ ++ /sys/devices/virtual/memory_tiering/memory_tierN/ ++ ++ ++What: /sys/devices/virtual/memory_tiering/memory_tierN/ ++ /sys/devices/virtual/memory_tiering/memory_tierN/nodes ++Date: August 2022 ++Contact: Linux memory management mailing list ++Description: Directory with details of a specific memory tier ++ ++ This is the directory containing information about a particular ++ memory tier, memtierN, where N is derived based on abstract distance. ++ ++ A smaller value of N implies a higher (faster) memory tier in the ++ hierarchy. ++ ++ nodes: NUMA nodes that are part of this memory tier. ++ +diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c +index a37622060fff..4852a2dbdb27 100644 +--- a/drivers/dax/kmem.c ++++ b/drivers/dax/kmem.c +@@ -11,9 +11,17 @@ + #include + #include + #include ++#include + #include "dax-private.h" + #include "bus.h" + ++/* ++ * Default abstract distance assigned to the NUMA node onlined ++ * by DAX/kmem if the low level platform driver didn't initialize ++ * one for this NUMA node. ++ */ ++#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) ++ + /* Memory resource name used for add_memory_driver_managed(). */ + static const char *kmem_name; + /* Set if any memory will remain added when the driver will be unloaded. */ +@@ -41,6 +49,7 @@ struct dax_kmem_data { + struct resource *res[]; + }; + ++static struct memory_dev_type *dax_slowmem_type; + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + { + struct device *dev = &dev_dax->dev; +@@ -79,11 +88,13 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + return -EINVAL; + } + ++ init_node_memory_type(numa_node, dax_slowmem_type); ++ ++ rc = -ENOMEM; + data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL); + if (!data) +- return -ENOMEM; ++ goto err_dax_kmem_data; + +- rc = -ENOMEM; + data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); + if (!data->res_name) + goto err_res_name; +@@ -155,6 +166,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + kfree(data->res_name); + err_res_name: + kfree(data); ++err_dax_kmem_data: ++ clear_node_memory_type(numa_node, dax_slowmem_type); + return rc; + } + +@@ -162,6 +175,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + static void dev_dax_kmem_remove(struct dev_dax *dev_dax) + { + int i, success = 0; ++ int node = dev_dax->target_node; + struct device *dev = &dev_dax->dev; + struct dax_kmem_data *data = dev_get_drvdata(dev); + +@@ -198,6 +212,14 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) + kfree(data->res_name); + kfree(data); + dev_set_drvdata(dev, NULL); ++ /* ++ * Clear the memtype association on successful unplug. ++ * If not, we have memory blocks left which can be ++ * offlined/onlined later. We need to keep memory_dev_type ++ * for that. This implies this reference will be around ++ * till next reboot. ++ */ ++ clear_node_memory_type(node, dax_slowmem_type); + } + } + #else +@@ -228,9 +250,22 @@ static int __init dax_kmem_init(void) + if (!kmem_name) + return -ENOMEM; + ++ dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE); ++ if (IS_ERR(dax_slowmem_type)) { ++ rc = PTR_ERR(dax_slowmem_type); ++ goto err_dax_slowmem_type; ++ } ++ + rc = dax_driver_register(&device_dax_kmem_driver); + if (rc) +- kfree_const(kmem_name); ++ goto error_dax_driver; ++ ++ return rc; ++ ++error_dax_driver: ++ destroy_memory_type(dax_slowmem_type); ++err_dax_slowmem_type: ++ kfree_const(kmem_name); + return rc; + } + +@@ -239,6 +274,7 @@ static void __exit dax_kmem_exit(void) + dax_driver_unregister(&device_dax_kmem_driver); + if (!any_hotremove_failed) + kfree_const(kmem_name); ++ destroy_memory_type(dax_slowmem_type); + } + + MODULE_AUTHOR("Intel Corporation"); +diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h +new file mode 100644 +index 000000000000..965009aa01d7 +--- /dev/null ++++ b/include/linux/memory-tiers.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_MEMORY_TIERS_H ++#define _LINUX_MEMORY_TIERS_H ++ ++#include ++#include ++#include ++#include ++/* ++ * Each tier cover a abstrace distance chunk size of 128 ++ */ ++#define MEMTIER_CHUNK_BITS 7 ++#define MEMTIER_CHUNK_SIZE (1 << MEMTIER_CHUNK_BITS) ++/* ++ * Smaller abstract distance values imply faster (higher) memory tiers. Offset ++ * the DRAM adistance so that we can accommodate devices with a slightly lower ++ * adistance value (slightly faster) than default DRAM adistance to be part of ++ * the same memory tier. ++ */ ++#define MEMTIER_ADISTANCE_DRAM ((4 * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1)) ++#define MEMTIER_HOTPLUG_PRIO 100 ++ ++struct memory_tier; ++struct memory_dev_type { ++ /* list of memory types that are part of same tier as this type */ ++ struct list_head tier_sibiling; ++ /* abstract distance for this specific memory type */ ++ int adistance; ++ /* Nodes of same abstract distance */ ++ nodemask_t nodes; ++ struct kref kref; ++}; ++ ++#ifdef CONFIG_NUMA ++extern bool numa_demotion_enabled; ++struct memory_dev_type *alloc_memory_type(int adistance); ++void destroy_memory_type(struct memory_dev_type *memtype); ++void init_node_memory_type(int node, struct memory_dev_type *default_type); ++void clear_node_memory_type(int node, struct memory_dev_type *memtype); ++#ifdef CONFIG_MIGRATION ++int next_demotion_node(int node); ++void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); ++bool node_is_toptier(int node); ++#else ++static inline int next_demotion_node(int node) ++{ ++ return NUMA_NO_NODE; ++} ++ ++static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ *targets = NODE_MASK_NONE; ++} ++ ++static inline bool node_is_toptier(int node) ++{ ++ return true; ++} ++#endif ++ ++#else ++ ++#define numa_demotion_enabled false ++/* ++ * CONFIG_NUMA implementation returns non NULL error. ++ */ ++static inline struct memory_dev_type *alloc_memory_type(int adistance) ++{ ++ return NULL; ++} ++ ++static inline void destroy_memory_type(struct memory_dev_type *memtype) ++{ ++ ++} ++ ++static inline void init_node_memory_type(int node, struct memory_dev_type *default_type) ++{ ++ ++} ++ ++static inline void clear_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ ++} ++ ++static inline int next_demotion_node(int node) ++{ ++ return NUMA_NO_NODE; ++} ++ ++static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ *targets = NODE_MASK_NONE; ++} ++ ++static inline bool node_is_toptier(int node) ++{ ++ return true; ++} ++#endif /* CONFIG_NUMA */ ++#endif /* _LINUX_MEMORY_TIERS_H */ +diff --git a/include/linux/migrate.h b/include/linux/migrate.h +index 22c0a0cf5e0c..704a04f5a074 100644 +--- a/include/linux/migrate.h ++++ b/include/linux/migrate.h +@@ -100,21 +100,6 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping, + + #endif /* CONFIG_MIGRATION */ + +-#if defined(CONFIG_MIGRATION) && defined(CONFIG_NUMA) +-extern void set_migration_target_nodes(void); +-extern void migrate_on_reclaim_init(void); +-extern bool numa_demotion_enabled; +-extern int next_demotion_node(int node); +-#else +-static inline void set_migration_target_nodes(void) {} +-static inline void migrate_on_reclaim_init(void) {} +-static inline int next_demotion_node(int node) +-{ +- return NUMA_NO_NODE; +-} +-#define numa_demotion_enabled false +-#endif +- + #ifdef CONFIG_COMPACTION + bool PageMovable(struct page *page); + void __SetPageMovable(struct page *page, const struct movable_operations *ops); +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index 1543001feba9..e5c240eed6af 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -1228,6 +1228,9 @@ typedef struct pglist_data { + /* Per-node vmstats */ + struct per_cpu_nodestat __percpu *per_cpu_nodestats; + atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; ++#ifdef CONFIG_NUMA ++ struct memory_tier __rcu *memtier; ++#endif + } pg_data_t; + + #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) +diff --git a/include/linux/node.h b/include/linux/node.h +index 40d641a8bfb0..9ec680dd607f 100644 +--- a/include/linux/node.h ++++ b/include/linux/node.h +@@ -185,9 +185,4 @@ static inline void register_hugetlbfs_with_node(node_registration_func_t reg, + + #define to_node(device) container_of(device, struct node, dev) + +-static inline bool node_is_toptier(int node) +-{ +- return node_state(node, N_CPU); +-} +- + #endif /* _LINUX_NODE_H_ */ +diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h +index 3a0eec9f2faa..e66742db741c 100644 +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -505,12 +505,21 @@ static inline int num_node_state(enum node_states state) + static inline int node_random(const nodemask_t *maskp) + { + #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) +- int w, bit = NUMA_NO_NODE; ++ int w, bit; + + w = nodes_weight(*maskp); +- if (w) ++ switch (w) { ++ case 0: ++ bit = NUMA_NO_NODE; ++ break; ++ case 1: ++ bit = first_node(*maskp); ++ break; ++ default: + bit = bitmap_ord_to_pos(maskp->bits, +- get_random_int() % w, MAX_NUMNODES); ++ get_random_int() % w, MAX_NUMNODES); ++ break; ++ } + return bit; + #else + return 0; +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index 67b8e93ea742..f7899278e979 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -40,6 +40,7 @@ + + #include + #include ++#include + #include + #include + #include +diff --git a/mm/Makefile b/mm/Makefile +index 8083fa85a348..a731d1decbb1 100644 +--- a/mm/Makefile ++++ b/mm/Makefile +@@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/ + obj-$(CONFIG_FAILSLAB) += failslab.o + obj-$(CONFIG_MEMTEST) += memtest.o + obj-$(CONFIG_MIGRATION) += migrate.o ++obj-$(CONFIG_NUMA) += memory-tiers.o + obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o + obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o + obj-$(CONFIG_PAGE_COUNTER) += page_counter.o +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index a50ab1bf4038..2a5c7f420098 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include + + #include + #include +diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c +new file mode 100644 +index 000000000000..f116b7b6333e +--- /dev/null ++++ b/mm/memory-tiers.c +@@ -0,0 +1,732 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "internal.h" ++ ++struct memory_tier { ++ /* hierarchy of memory tiers */ ++ struct list_head list; ++ /* list of all memory types part of this tier */ ++ struct list_head memory_types; ++ /* ++ * start value of abstract distance. memory tier maps ++ * an abstract distance range, ++ * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE ++ */ ++ int adistance_start; ++ struct device dev; ++ /* All the nodes that are part of all the lower memory tiers. */ ++ nodemask_t lower_tier_mask; ++}; ++ ++struct demotion_nodes { ++ nodemask_t preferred; ++}; ++ ++struct node_memory_type_map { ++ struct memory_dev_type *memtype; ++ int map_count; ++}; ++ ++static DEFINE_MUTEX(memory_tier_lock); ++static LIST_HEAD(memory_tiers); ++static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; ++static struct memory_dev_type *default_dram_type; ++ ++static struct bus_type memory_tier_subsys = { ++ .name = "memory_tiering", ++ .dev_name = "memory_tier", ++}; ++ ++#ifdef CONFIG_MIGRATION ++static int top_tier_adistance; ++/* ++ * node_demotion[] examples: ++ * ++ * Example 1: ++ * ++ * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes. ++ * ++ * node distances: ++ * node 0 1 2 3 ++ * 0 10 20 30 40 ++ * 1 20 10 40 30 ++ * 2 30 40 10 40 ++ * 3 40 30 40 10 ++ * ++ * memory_tiers0 = 0-1 ++ * memory_tiers1 = 2-3 ++ * ++ * node_demotion[0].preferred = 2 ++ * node_demotion[1].preferred = 3 ++ * node_demotion[2].preferred = ++ * node_demotion[3].preferred = ++ * ++ * Example 2: ++ * ++ * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node. ++ * ++ * node distances: ++ * node 0 1 2 ++ * 0 10 20 30 ++ * 1 20 10 30 ++ * 2 30 30 10 ++ * ++ * memory_tiers0 = 0-2 ++ * ++ * node_demotion[0].preferred = ++ * node_demotion[1].preferred = ++ * node_demotion[2].preferred = ++ * ++ * Example 3: ++ * ++ * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node. ++ * ++ * node distances: ++ * node 0 1 2 ++ * 0 10 20 30 ++ * 1 20 10 40 ++ * 2 30 40 10 ++ * ++ * memory_tiers0 = 1 ++ * memory_tiers1 = 0 ++ * memory_tiers2 = 2 ++ * ++ * node_demotion[0].preferred = 2 ++ * node_demotion[1].preferred = 0 ++ * node_demotion[2].preferred = ++ * ++ */ ++static struct demotion_nodes *node_demotion __read_mostly; ++#endif /* CONFIG_MIGRATION */ ++ ++static inline struct memory_tier *to_memory_tier(struct device *device) ++{ ++ return container_of(device, struct memory_tier, dev); ++} ++ ++static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier) ++{ ++ nodemask_t nodes = NODE_MASK_NONE; ++ struct memory_dev_type *memtype; ++ ++ list_for_each_entry(memtype, &memtier->memory_types, tier_sibiling) ++ nodes_or(nodes, nodes, memtype->nodes); ++ ++ return nodes; ++} ++ ++static void memory_tier_device_release(struct device *dev) ++{ ++ struct memory_tier *tier = to_memory_tier(dev); ++ /* ++ * synchronize_rcu in clear_node_memory_tier makes sure ++ * we don't have rcu access to this memory tier. ++ */ ++ kfree(tier); ++} ++ ++static ssize_t nodes_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ int ret; ++ nodemask_t nmask; ++ ++ mutex_lock(&memory_tier_lock); ++ nmask = get_memtier_nodemask(to_memory_tier(dev)); ++ ret = sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&nmask)); ++ mutex_unlock(&memory_tier_lock); ++ return ret; ++} ++static DEVICE_ATTR_RO(nodes); ++ ++static struct attribute *memtier_dev_attrs[] = { ++ &dev_attr_nodes.attr, ++ NULL ++}; ++ ++static const struct attribute_group memtier_dev_group = { ++ .attrs = memtier_dev_attrs, ++}; ++ ++static const struct attribute_group *memtier_dev_groups[] = { ++ &memtier_dev_group, ++ NULL ++}; ++ ++static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype) ++{ ++ int ret; ++ bool found_slot = false; ++ struct memory_tier *memtier, *new_memtier; ++ int adistance = memtype->adistance; ++ unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE; ++ ++ lockdep_assert_held_once(&memory_tier_lock); ++ ++ adistance = round_down(adistance, memtier_adistance_chunk_size); ++ /* ++ * If the memtype is already part of a memory tier, ++ * just return that. ++ */ ++ if (!list_empty(&memtype->tier_sibiling)) { ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ if (adistance == memtier->adistance_start) ++ return memtier; ++ } ++ WARN_ON(1); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ if (adistance == memtier->adistance_start) { ++ goto link_memtype; ++ } else if (adistance < memtier->adistance_start) { ++ found_slot = true; ++ break; ++ } ++ } ++ ++ new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL); ++ if (!new_memtier) ++ return ERR_PTR(-ENOMEM); ++ ++ new_memtier->adistance_start = adistance; ++ INIT_LIST_HEAD(&new_memtier->list); ++ INIT_LIST_HEAD(&new_memtier->memory_types); ++ if (found_slot) ++ list_add_tail(&new_memtier->list, &memtier->list); ++ else ++ list_add_tail(&new_memtier->list, &memory_tiers); ++ ++ new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS; ++ new_memtier->dev.bus = &memory_tier_subsys; ++ new_memtier->dev.release = memory_tier_device_release; ++ new_memtier->dev.groups = memtier_dev_groups; ++ ++ ret = device_register(&new_memtier->dev); ++ if (ret) { ++ list_del(&memtier->list); ++ put_device(&memtier->dev); ++ return ERR_PTR(ret); ++ } ++ memtier = new_memtier; ++ ++link_memtype: ++ list_add(&memtype->tier_sibiling, &memtier->memory_types); ++ return memtier; ++} ++ ++static struct memory_tier *__node_get_memory_tier(int node) ++{ ++ pg_data_t *pgdat; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return NULL; ++ /* ++ * Since we hold memory_tier_lock, we can avoid ++ * RCU read locks when accessing the details. No ++ * parallel updates are possible here. ++ */ ++ return rcu_dereference_check(pgdat->memtier, ++ lockdep_is_held(&memory_tier_lock)); ++} ++ ++#ifdef CONFIG_MIGRATION ++bool node_is_toptier(int node) ++{ ++ bool toptier; ++ pg_data_t *pgdat; ++ struct memory_tier *memtier; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return false; ++ ++ rcu_read_lock(); ++ memtier = rcu_dereference(pgdat->memtier); ++ if (!memtier) { ++ toptier = true; ++ goto out; ++ } ++ if (memtier->adistance_start <= top_tier_adistance) ++ toptier = true; ++ else ++ toptier = false; ++out: ++ rcu_read_unlock(); ++ return toptier; ++} ++ ++void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ struct memory_tier *memtier; ++ ++ /* ++ * pg_data_t.memtier updates includes a synchronize_rcu() ++ * which ensures that we either find NULL or a valid memtier ++ * in NODE_DATA. protect the access via rcu_read_lock(); ++ */ ++ rcu_read_lock(); ++ memtier = rcu_dereference(pgdat->memtier); ++ if (memtier) ++ *targets = memtier->lower_tier_mask; ++ else ++ *targets = NODE_MASK_NONE; ++ rcu_read_unlock(); ++} ++ ++/** ++ * next_demotion_node() - Get the next node in the demotion path ++ * @node: The starting node to lookup the next node ++ * ++ * Return: node id for next memory node in the demotion path hierarchy ++ * from @node; NUMA_NO_NODE if @node is terminal. This does not keep ++ * @node online or guarantee that it *continues* to be the next demotion ++ * target. ++ */ ++int next_demotion_node(int node) ++{ ++ struct demotion_nodes *nd; ++ int target; ++ ++ if (!node_demotion) ++ return NUMA_NO_NODE; ++ ++ nd = &node_demotion[node]; ++ ++ /* ++ * node_demotion[] is updated without excluding this ++ * function from running. ++ * ++ * Make sure to use RCU over entire code blocks if ++ * node_demotion[] reads need to be consistent. ++ */ ++ rcu_read_lock(); ++ /* ++ * If there are multiple target nodes, just select one ++ * target node randomly. ++ * ++ * In addition, we can also use round-robin to select ++ * target node, but we should introduce another variable ++ * for node_demotion[] to record last selected target node, ++ * that may cause cache ping-pong due to the changing of ++ * last target node. Or introducing per-cpu data to avoid ++ * caching issue, which seems more complicated. So selecting ++ * target node randomly seems better until now. ++ */ ++ target = node_random(&nd->preferred); ++ rcu_read_unlock(); ++ ++ return target; ++} ++ ++static void disable_all_demotion_targets(void) ++{ ++ struct memory_tier *memtier; ++ int node; ++ ++ for_each_node_state(node, N_MEMORY) { ++ node_demotion[node].preferred = NODE_MASK_NONE; ++ /* ++ * We are holding memory_tier_lock, it is safe ++ * to access pgda->memtier. ++ */ ++ memtier = __node_get_memory_tier(node); ++ if (memtier) ++ memtier->lower_tier_mask = NODE_MASK_NONE; ++ } ++ /* ++ * Ensure that the "disable" is visible across the system. ++ * Readers will see either a combination of before+disable ++ * state or disable+after. They will never see before and ++ * after state together. ++ */ ++ synchronize_rcu(); ++} ++ ++/* ++ * Find an automatic demotion target for all memory ++ * nodes. Failing here is OK. It might just indicate ++ * being at the end of a chain. ++ */ ++static void establish_demotion_targets(void) ++{ ++ struct memory_tier *memtier; ++ struct demotion_nodes *nd; ++ int target = NUMA_NO_NODE, node; ++ int distance, best_distance; ++ nodemask_t tier_nodes, lower_tier; ++ ++ lockdep_assert_held_once(&memory_tier_lock); ++ ++ if (!node_demotion || !IS_ENABLED(CONFIG_MIGRATION)) ++ return; ++ ++ disable_all_demotion_targets(); ++ ++ for_each_node_state(node, N_MEMORY) { ++ best_distance = -1; ++ nd = &node_demotion[node]; ++ ++ memtier = __node_get_memory_tier(node); ++ if (!memtier || list_is_last(&memtier->list, &memory_tiers)) ++ continue; ++ /* ++ * Get the lower memtier to find the demotion node list. ++ */ ++ memtier = list_next_entry(memtier, list); ++ tier_nodes = get_memtier_nodemask(memtier); ++ /* ++ * find_next_best_node, use 'used' nodemask as a skip list. ++ * Add all memory nodes except the selected memory tier ++ * nodelist to skip list so that we find the best node from the ++ * memtier nodelist. ++ */ ++ nodes_andnot(tier_nodes, node_states[N_MEMORY], tier_nodes); ++ ++ /* ++ * Find all the nodes in the memory tier node list of same best distance. ++ * add them to the preferred mask. We randomly select between nodes ++ * in the preferred mask when allocating pages during demotion. ++ */ ++ do { ++ target = find_next_best_node(node, &tier_nodes); ++ if (target == NUMA_NO_NODE) ++ break; ++ ++ distance = node_distance(node, target); ++ if (distance == best_distance || best_distance == -1) { ++ best_distance = distance; ++ node_set(target, nd->preferred); ++ } else { ++ break; ++ } ++ } while (1); ++ } ++ /* ++ * Promotion is allowed from a memory tier to higher ++ * memory tier only if the memory tier doesn't include ++ * compute. We want to skip promotion from a memory tier, ++ * if any node that is part of the memory tier have CPUs. ++ * Once we detect such a memory tier, we consider that tier ++ * as top tiper from which promotion is not allowed. ++ */ ++ list_for_each_entry_reverse(memtier, &memory_tiers, list) { ++ tier_nodes = get_memtier_nodemask(memtier); ++ nodes_and(tier_nodes, node_states[N_CPU], tier_nodes); ++ if (!nodes_empty(tier_nodes)) { ++ /* ++ * abstract distance below the max value of this memtier ++ * is considered toptier. ++ */ ++ top_tier_adistance = memtier->adistance_start + ++ MEMTIER_CHUNK_SIZE - 1; ++ break; ++ } ++ } ++ /* ++ * Now build the lower_tier mask for each node collecting node mask from ++ * all memory tier below it. This allows us to fallback demotion page ++ * allocation to a set of nodes that is closer the above selected ++ * perferred node. ++ */ ++ lower_tier = node_states[N_MEMORY]; ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ /* ++ * Keep removing current tier from lower_tier nodes, ++ * This will remove all nodes in current and above ++ * memory tier from the lower_tier mask. ++ */ ++ tier_nodes = get_memtier_nodemask(memtier); ++ nodes_andnot(lower_tier, lower_tier, tier_nodes); ++ memtier->lower_tier_mask = lower_tier; ++ } ++} ++ ++#else ++static inline void disable_all_demotion_targets(void) {} ++static inline void establish_demotion_targets(void) {} ++#endif /* CONFIG_MIGRATION */ ++ ++static inline void __init_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ if (!node_memory_types[node].memtype) ++ node_memory_types[node].memtype = memtype; ++ /* ++ * for each device getting added in the same NUMA node ++ * with this specific memtype, bump the map count. We ++ * Only take memtype device reference once, so that ++ * changing a node memtype can be done by droping the ++ * only reference count taken here. ++ */ ++ ++ if (node_memory_types[node].memtype == memtype) { ++ if (!node_memory_types[node].map_count++) ++ kref_get(&memtype->kref); ++ } ++} ++ ++static struct memory_tier *set_node_memory_tier(int node) ++{ ++ struct memory_tier *memtier; ++ struct memory_dev_type *memtype; ++ pg_data_t *pgdat = NODE_DATA(node); ++ ++ ++ lockdep_assert_held_once(&memory_tier_lock); ++ ++ if (!node_state(node, N_MEMORY)) ++ return ERR_PTR(-EINVAL); ++ ++ __init_node_memory_type(node, default_dram_type); ++ ++ memtype = node_memory_types[node].memtype; ++ node_set(node, memtype->nodes); ++ memtier = find_create_memory_tier(memtype); ++ if (!IS_ERR(memtier)) ++ rcu_assign_pointer(pgdat->memtier, memtier); ++ return memtier; ++} ++ ++static void destroy_memory_tier(struct memory_tier *memtier) ++{ ++ list_del(&memtier->list); ++ device_unregister(&memtier->dev); ++} ++ ++static bool clear_node_memory_tier(int node) ++{ ++ bool cleared = false; ++ pg_data_t *pgdat; ++ struct memory_tier *memtier; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return false; ++ ++ /* ++ * Make sure that anybody looking at NODE_DATA who finds ++ * a valid memtier finds memory_dev_types with nodes still ++ * linked to the memtier. We achieve this by waiting for ++ * rcu read section to finish using synchronize_rcu. ++ * This also enables us to free the destroyed memory tier ++ * with kfree instead of kfree_rcu ++ */ ++ memtier = __node_get_memory_tier(node); ++ if (memtier) { ++ struct memory_dev_type *memtype; ++ ++ rcu_assign_pointer(pgdat->memtier, NULL); ++ synchronize_rcu(); ++ memtype = node_memory_types[node].memtype; ++ node_clear(node, memtype->nodes); ++ if (nodes_empty(memtype->nodes)) { ++ list_del_init(&memtype->tier_sibiling); ++ if (list_empty(&memtier->memory_types)) ++ destroy_memory_tier(memtier); ++ } ++ cleared = true; ++ } ++ return cleared; ++} ++ ++static void release_memtype(struct kref *kref) ++{ ++ struct memory_dev_type *memtype; ++ ++ memtype = container_of(kref, struct memory_dev_type, kref); ++ kfree(memtype); ++} ++ ++struct memory_dev_type *alloc_memory_type(int adistance) ++{ ++ struct memory_dev_type *memtype; ++ ++ memtype = kmalloc(sizeof(*memtype), GFP_KERNEL); ++ if (!memtype) ++ return ERR_PTR(-ENOMEM); ++ ++ memtype->adistance = adistance; ++ INIT_LIST_HEAD(&memtype->tier_sibiling); ++ memtype->nodes = NODE_MASK_NONE; ++ kref_init(&memtype->kref); ++ return memtype; ++} ++EXPORT_SYMBOL_GPL(alloc_memory_type); ++ ++void destroy_memory_type(struct memory_dev_type *memtype) ++{ ++ kref_put(&memtype->kref, release_memtype); ++} ++EXPORT_SYMBOL_GPL(destroy_memory_type); ++ ++void init_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ ++ mutex_lock(&memory_tier_lock); ++ __init_node_memory_type(node, memtype); ++ mutex_unlock(&memory_tier_lock); ++} ++EXPORT_SYMBOL_GPL(init_node_memory_type); ++ ++void clear_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ mutex_lock(&memory_tier_lock); ++ if (node_memory_types[node].memtype == memtype) ++ node_memory_types[node].map_count--; ++ /* ++ * If we umapped all the attached devices to this node, ++ * clear the node memory type. ++ */ ++ if (!node_memory_types[node].map_count) { ++ node_memory_types[node].memtype = NULL; ++ kref_put(&memtype->kref, release_memtype); ++ } ++ mutex_unlock(&memory_tier_lock); ++} ++EXPORT_SYMBOL_GPL(clear_node_memory_type); ++ ++static int __meminit memtier_hotplug_callback(struct notifier_block *self, ++ unsigned long action, void *_arg) ++{ ++ struct memory_tier *memtier; ++ struct memory_notify *arg = _arg; ++ ++ /* ++ * Only update the node migration order when a node is ++ * changing status, like online->offline. ++ */ ++ if (arg->status_change_nid < 0) ++ return notifier_from_errno(0); ++ ++ switch (action) { ++ case MEM_OFFLINE: ++ mutex_lock(&memory_tier_lock); ++ if (clear_node_memory_tier(arg->status_change_nid)) ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ break; ++ case MEM_ONLINE: ++ mutex_lock(&memory_tier_lock); ++ memtier = set_node_memory_tier(arg->status_change_nid); ++ if (!IS_ERR(memtier)) ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ break; ++ } ++ ++ return notifier_from_errno(0); ++} ++ ++static int __init memory_tier_init(void) ++{ ++ int ret, node; ++ struct memory_tier *memtier; ++ ++ ret = subsys_virtual_register(&memory_tier_subsys, NULL); ++ if (ret) ++ panic("%s() failed to register memory tier subsystem\n", __func__); ++ ++#ifdef CONFIG_MIGRATION ++ node_demotion = kcalloc(nr_node_ids, sizeof(struct demotion_nodes), ++ GFP_KERNEL); ++ WARN_ON(!node_demotion); ++#endif ++ mutex_lock(&memory_tier_lock); ++ /* ++ * For now we can have 4 faster memory tiers with smaller adistance ++ * than default DRAM tier. ++ */ ++ default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM); ++ if (!default_dram_type) ++ panic("%s() failed to allocate default DRAM tier\n", __func__); ++ ++ /* ++ * Look at all the existing N_MEMORY nodes and add them to ++ * default memory tier or to a tier if we already have memory ++ * types assigned. ++ */ ++ for_each_node_state(node, N_MEMORY) { ++ memtier = set_node_memory_tier(node); ++ if (IS_ERR(memtier)) ++ /* ++ * Continue with memtiers we are able to setup ++ */ ++ break; ++ } ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ ++ hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRIO); ++ return 0; ++} ++subsys_initcall(memory_tier_init); ++ ++bool numa_demotion_enabled = false; ++ ++#ifdef CONFIG_MIGRATION ++#ifdef CONFIG_SYSFS ++static ssize_t numa_demotion_enabled_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%s\n", ++ numa_demotion_enabled ? "true" : "false"); ++} ++ ++static ssize_t numa_demotion_enabled_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ ssize_t ret; ++ ++ ret = kstrtobool(buf, &numa_demotion_enabled); ++ if (ret) ++ return ret; ++ ++ return count; ++} ++ ++static struct kobj_attribute numa_demotion_enabled_attr = ++ __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, ++ numa_demotion_enabled_store); ++ ++static struct attribute *numa_attrs[] = { ++ &numa_demotion_enabled_attr.attr, ++ NULL, ++}; ++ ++static const struct attribute_group numa_attr_group = { ++ .attrs = numa_attrs, ++}; ++ ++static int __init numa_init_sysfs(void) ++{ ++ int err; ++ struct kobject *numa_kobj; ++ ++ numa_kobj = kobject_create_and_add("numa", mm_kobj); ++ if (!numa_kobj) { ++ pr_err("failed to create numa kobject\n"); ++ return -ENOMEM; ++ } ++ err = sysfs_create_group(numa_kobj, &numa_attr_group); ++ if (err) { ++ pr_err("failed to register numa group\n"); ++ goto delete_obj; ++ } ++ return 0; ++ ++delete_obj: ++ kobject_put(numa_kobj); ++ return err; ++} ++subsys_initcall(numa_init_sysfs); ++#endif /* CONFIG_SYSFS */ ++#endif +diff --git a/mm/memory.c b/mm/memory.c +index 6a416cac46d3..daeeca7bc3b5 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -66,6 +66,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/mm/migrate.c b/mm/migrate.c +index 6a1597c92261..55e7718cfe45 100644 +--- a/mm/migrate.c ++++ b/mm/migrate.c +@@ -50,6 +50,7 @@ + #include + #include + #include ++#include + + #include + +@@ -2170,456 +2171,4 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, + return 0; + } + #endif /* CONFIG_NUMA_BALANCING */ +- +-/* +- * node_demotion[] example: +- * +- * Consider a system with two sockets. Each socket has +- * three classes of memory attached: fast, medium and slow. +- * Each memory class is placed in its own NUMA node. The +- * CPUs are placed in the node with the "fast" memory. The +- * 6 NUMA nodes (0-5) might be split among the sockets like +- * this: +- * +- * Socket A: 0, 1, 2 +- * Socket B: 3, 4, 5 +- * +- * When Node 0 fills up, its memory should be migrated to +- * Node 1. When Node 1 fills up, it should be migrated to +- * Node 2. The migration path start on the nodes with the +- * processors (since allocations default to this node) and +- * fast memory, progress through medium and end with the +- * slow memory: +- * +- * 0 -> 1 -> 2 -> stop +- * 3 -> 4 -> 5 -> stop +- * +- * This is represented in the node_demotion[] like this: +- * +- * { nr=1, nodes[0]=1 }, // Node 0 migrates to 1 +- * { nr=1, nodes[0]=2 }, // Node 1 migrates to 2 +- * { nr=0, nodes[0]=-1 }, // Node 2 does not migrate +- * { nr=1, nodes[0]=4 }, // Node 3 migrates to 4 +- * { nr=1, nodes[0]=5 }, // Node 4 migrates to 5 +- * { nr=0, nodes[0]=-1 }, // Node 5 does not migrate +- * +- * Moreover some systems may have multiple slow memory nodes. +- * Suppose a system has one socket with 3 memory nodes, node 0 +- * is fast memory type, and node 1/2 both are slow memory +- * type, and the distance between fast memory node and slow +- * memory node is same. So the migration path should be: +- * +- * 0 -> 1/2 -> stop +- * +- * This is represented in the node_demotion[] like this: +- * { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2 +- * { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate +- * { nr=0, nodes[0]=-1, }, // Node 2 does not migrate +- */ +- +-/* +- * Writes to this array occur without locking. Cycles are +- * not allowed: Node X demotes to Y which demotes to X... +- * +- * If multiple reads are performed, a single rcu_read_lock() +- * must be held over all reads to ensure that no cycles are +- * observed. +- */ +-#define DEFAULT_DEMOTION_TARGET_NODES 15 +- +-#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES +-#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1) +-#else +-#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES +-#endif +- +-struct demotion_nodes { +- unsigned short nr; +- short nodes[DEMOTION_TARGET_NODES]; +-}; +- +-static struct demotion_nodes *node_demotion __read_mostly; +- +-/** +- * next_demotion_node() - Get the next node in the demotion path +- * @node: The starting node to lookup the next node +- * +- * Return: node id for next memory node in the demotion path hierarchy +- * from @node; NUMA_NO_NODE if @node is terminal. This does not keep +- * @node online or guarantee that it *continues* to be the next demotion +- * target. +- */ +-int next_demotion_node(int node) +-{ +- struct demotion_nodes *nd; +- unsigned short target_nr, index; +- int target; +- +- if (!node_demotion) +- return NUMA_NO_NODE; +- +- nd = &node_demotion[node]; +- +- /* +- * node_demotion[] is updated without excluding this +- * function from running. RCU doesn't provide any +- * compiler barriers, so the READ_ONCE() is required +- * to avoid compiler reordering or read merging. +- * +- * Make sure to use RCU over entire code blocks if +- * node_demotion[] reads need to be consistent. +- */ +- rcu_read_lock(); +- target_nr = READ_ONCE(nd->nr); +- +- switch (target_nr) { +- case 0: +- target = NUMA_NO_NODE; +- goto out; +- case 1: +- index = 0; +- break; +- default: +- /* +- * If there are multiple target nodes, just select one +- * target node randomly. +- * +- * In addition, we can also use round-robin to select +- * target node, but we should introduce another variable +- * for node_demotion[] to record last selected target node, +- * that may cause cache ping-pong due to the changing of +- * last target node. Or introducing per-cpu data to avoid +- * caching issue, which seems more complicated. So selecting +- * target node randomly seems better until now. +- */ +- index = get_random_int() % target_nr; +- break; +- } +- +- target = READ_ONCE(nd->nodes[index]); +- +-out: +- rcu_read_unlock(); +- return target; +-} +- +-/* Disable reclaim-based migration. */ +-static void __disable_all_migrate_targets(void) +-{ +- int node, i; +- +- if (!node_demotion) +- return; +- +- for_each_online_node(node) { +- node_demotion[node].nr = 0; +- for (i = 0; i < DEMOTION_TARGET_NODES; i++) +- node_demotion[node].nodes[i] = NUMA_NO_NODE; +- } +-} +- +-static void disable_all_migrate_targets(void) +-{ +- __disable_all_migrate_targets(); +- +- /* +- * Ensure that the "disable" is visible across the system. +- * Readers will see either a combination of before+disable +- * state or disable+after. They will never see before and +- * after state together. +- * +- * The before+after state together might have cycles and +- * could cause readers to do things like loop until this +- * function finishes. This ensures they can only see a +- * single "bad" read and would, for instance, only loop +- * once. +- */ +- synchronize_rcu(); +-} +- +-/* +- * Find an automatic demotion target for 'node'. +- * Failing here is OK. It might just indicate +- * being at the end of a chain. +- */ +-static int establish_migrate_target(int node, nodemask_t *used, +- int best_distance) +-{ +- int migration_target, index, val; +- struct demotion_nodes *nd; +- +- if (!node_demotion) +- return NUMA_NO_NODE; +- +- nd = &node_demotion[node]; +- +- migration_target = find_next_best_node(node, used); +- if (migration_target == NUMA_NO_NODE) +- return NUMA_NO_NODE; +- +- /* +- * If the node has been set a migration target node before, +- * which means it's the best distance between them. Still +- * check if this node can be demoted to other target nodes +- * if they have a same best distance. +- */ +- if (best_distance != -1) { +- val = node_distance(node, migration_target); +- if (val > best_distance) +- goto out_clear; +- } +- +- index = nd->nr; +- if (WARN_ONCE(index >= DEMOTION_TARGET_NODES, +- "Exceeds maximum demotion target nodes\n")) +- goto out_clear; +- +- nd->nodes[index] = migration_target; +- nd->nr++; +- +- return migration_target; +-out_clear: +- node_clear(migration_target, *used); +- return NUMA_NO_NODE; +-} +- +-/* +- * When memory fills up on a node, memory contents can be +- * automatically migrated to another node instead of +- * discarded at reclaim. +- * +- * Establish a "migration path" which will start at nodes +- * with CPUs and will follow the priorities used to build the +- * page allocator zonelists. +- * +- * The difference here is that cycles must be avoided. If +- * node0 migrates to node1, then neither node1, nor anything +- * node1 migrates to can migrate to node0. Also one node can +- * be migrated to multiple nodes if the target nodes all have +- * a same best-distance against the source node. +- * +- * This function can run simultaneously with readers of +- * node_demotion[]. However, it can not run simultaneously +- * with itself. Exclusion is provided by memory hotplug events +- * being single-threaded. +- */ +-static void __set_migration_target_nodes(void) +-{ +- nodemask_t next_pass; +- nodemask_t this_pass; +- nodemask_t used_targets = NODE_MASK_NONE; +- int node, best_distance; +- +- /* +- * Avoid any oddities like cycles that could occur +- * from changes in the topology. This will leave +- * a momentary gap when migration is disabled. +- */ +- disable_all_migrate_targets(); +- +- /* +- * Allocations go close to CPUs, first. Assume that +- * the migration path starts at the nodes with CPUs. +- */ +- next_pass = node_states[N_CPU]; +-again: +- this_pass = next_pass; +- next_pass = NODE_MASK_NONE; +- /* +- * To avoid cycles in the migration "graph", ensure +- * that migration sources are not future targets by +- * setting them in 'used_targets'. Do this only +- * once per pass so that multiple source nodes can +- * share a target node. +- * +- * 'used_targets' will become unavailable in future +- * passes. This limits some opportunities for +- * multiple source nodes to share a destination. +- */ +- nodes_or(used_targets, used_targets, this_pass); +- +- for_each_node_mask(node, this_pass) { +- best_distance = -1; +- +- /* +- * Try to set up the migration path for the node, and the target +- * migration nodes can be multiple, so doing a loop to find all +- * the target nodes if they all have a best node distance. +- */ +- do { +- int target_node = +- establish_migrate_target(node, &used_targets, +- best_distance); +- +- if (target_node == NUMA_NO_NODE) +- break; +- +- if (best_distance == -1) +- best_distance = node_distance(node, target_node); +- +- /* +- * Visit targets from this pass in the next pass. +- * Eventually, every node will have been part of +- * a pass, and will become set in 'used_targets'. +- */ +- node_set(target_node, next_pass); +- } while (1); +- } +- /* +- * 'next_pass' contains nodes which became migration +- * targets in this pass. Make additional passes until +- * no more migrations targets are available. +- */ +- if (!nodes_empty(next_pass)) +- goto again; +-} +- +-/* +- * For callers that do not hold get_online_mems() already. +- */ +-void set_migration_target_nodes(void) +-{ +- get_online_mems(); +- __set_migration_target_nodes(); +- put_online_mems(); +-} +- +-/* +- * This leaves migrate-on-reclaim transiently disabled between +- * the MEM_GOING_OFFLINE and MEM_OFFLINE events. This runs +- * whether reclaim-based migration is enabled or not, which +- * ensures that the user can turn reclaim-based migration at +- * any time without needing to recalculate migration targets. +- * +- * These callbacks already hold get_online_mems(). That is why +- * __set_migration_target_nodes() can be used as opposed to +- * set_migration_target_nodes(). +- */ +-#ifdef CONFIG_MEMORY_HOTPLUG +-static int __meminit migrate_on_reclaim_callback(struct notifier_block *self, +- unsigned long action, void *_arg) +-{ +- struct memory_notify *arg = _arg; +- +- /* +- * Only update the node migration order when a node is +- * changing status, like online->offline. This avoids +- * the overhead of synchronize_rcu() in most cases. +- */ +- if (arg->status_change_nid < 0) +- return notifier_from_errno(0); +- +- switch (action) { +- case MEM_GOING_OFFLINE: +- /* +- * Make sure there are not transient states where +- * an offline node is a migration target. This +- * will leave migration disabled until the offline +- * completes and the MEM_OFFLINE case below runs. +- */ +- disable_all_migrate_targets(); +- break; +- case MEM_OFFLINE: +- case MEM_ONLINE: +- /* +- * Recalculate the target nodes once the node +- * reaches its final state (online or offline). +- */ +- __set_migration_target_nodes(); +- break; +- case MEM_CANCEL_OFFLINE: +- /* +- * MEM_GOING_OFFLINE disabled all the migration +- * targets. Reenable them. +- */ +- __set_migration_target_nodes(); +- break; +- case MEM_GOING_ONLINE: +- case MEM_CANCEL_ONLINE: +- break; +- } +- +- return notifier_from_errno(0); +-} +-#endif +- +-void __init migrate_on_reclaim_init(void) +-{ +- node_demotion = kcalloc(nr_node_ids, +- sizeof(struct demotion_nodes), +- GFP_KERNEL); +- WARN_ON(!node_demotion); +-#ifdef CONFIG_MEMORY_HOTPLUG +- hotplug_memory_notifier(migrate_on_reclaim_callback, 100); +-#endif +- /* +- * At this point, all numa nodes with memory/CPus have their state +- * properly set, so we can build the demotion order now. +- * Let us hold the cpu_hotplug lock just, as we could possibily have +- * CPU hotplug events during boot. +- */ +- cpus_read_lock(); +- set_migration_target_nodes(); +- cpus_read_unlock(); +-} +- +-bool numa_demotion_enabled = false; +- +-#ifdef CONFIG_SYSFS +-static ssize_t numa_demotion_enabled_show(struct kobject *kobj, +- struct kobj_attribute *attr, char *buf) +-{ +- return sysfs_emit(buf, "%s\n", +- numa_demotion_enabled ? "true" : "false"); +-} +- +-static ssize_t numa_demotion_enabled_store(struct kobject *kobj, +- struct kobj_attribute *attr, +- const char *buf, size_t count) +-{ +- ssize_t ret; +- +- ret = kstrtobool(buf, &numa_demotion_enabled); +- if (ret) +- return ret; +- +- return count; +-} +- +-static struct kobj_attribute numa_demotion_enabled_attr = +- __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, +- numa_demotion_enabled_store); +- +-static struct attribute *numa_attrs[] = { +- &numa_demotion_enabled_attr.attr, +- NULL, +-}; +- +-static const struct attribute_group numa_attr_group = { +- .attrs = numa_attrs, +-}; +- +-static int __init numa_init_sysfs(void) +-{ +- int err; +- struct kobject *numa_kobj; +- +- numa_kobj = kobject_create_and_add("numa", mm_kobj); +- if (!numa_kobj) { +- pr_err("failed to create numa kobject\n"); +- return -ENOMEM; +- } +- err = sysfs_create_group(numa_kobj, &numa_attr_group); +- if (err) { +- pr_err("failed to register numa group\n"); +- goto delete_obj; +- } +- return 0; +- +-delete_obj: +- kobject_put(numa_kobj); +- return err; +-} +-subsys_initcall(numa_init_sysfs); +-#endif /* CONFIG_SYSFS */ + #endif /* CONFIG_NUMA */ +diff --git a/mm/mprotect.c b/mm/mprotect.c +index bf030bbe7e97..2bb8613cd295 100644 +--- a/mm/mprotect.c ++++ b/mm/mprotect.c +@@ -31,6 +31,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/mm/vmscan.c b/mm/vmscan.c +index 1e66f8cfc03d..856d6fd1b57a 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -43,6 +43,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -1536,21 +1537,34 @@ static void folio_check_dirty_writeback(struct folio *folio, + mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); + } + +-static struct page *alloc_demote_page(struct page *page, unsigned long node) ++static struct page *alloc_demote_page(struct page *page, unsigned long private) + { +- struct migration_target_control mtc = { +- /* +- * Allocate from 'node', or fail quickly and quietly. +- * When this happens, 'page' will likely just be discarded +- * instead of migrated. +- */ +- .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | +- __GFP_THISNODE | __GFP_NOWARN | +- __GFP_NOMEMALLOC | GFP_NOWAIT, +- .nid = node +- }; ++ struct page *target_page; ++ nodemask_t *allowed_mask; ++ struct migration_target_control *mtc; ++ ++ mtc = (struct migration_target_control *)private; ++ ++ allowed_mask = mtc->nmask; ++ /* ++ * make sure we allocate from the target node first also trying to ++ * demote or reclaim pages from the target node via kswapd if we are ++ * low on free memory on target node. If we don't do this and if ++ * we have free memory on the slower(lower) memtier, we would start ++ * allocating pages from slower(lower) memory tiers without even forcing ++ * a demotion of cold pages from the target memtier. This can result ++ * in the kernel placing hot pages in slower(lower) memory tiers. ++ */ ++ mtc->nmask = NULL; ++ mtc->gfp_mask |= __GFP_THISNODE; ++ target_page = alloc_migration_target(page, (unsigned long)mtc); ++ if (target_page) ++ return target_page; + +- return alloc_migration_target(page, (unsigned long)&mtc); ++ mtc->gfp_mask &= ~__GFP_THISNODE; ++ mtc->nmask = allowed_mask; ++ ++ return alloc_migration_target(page, (unsigned long)mtc); + } + + /* +@@ -1563,6 +1577,19 @@ static unsigned int demote_page_list(struct list_head *demote_pages, + { + int target_nid = next_demotion_node(pgdat->node_id); + unsigned int nr_succeeded; ++ nodemask_t allowed_mask; ++ ++ struct migration_target_control mtc = { ++ /* ++ * Allocate from 'node', or fail quickly and quietly. ++ * When this happens, 'page' will likely just be discarded ++ * instead of migrated. ++ */ ++ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | ++ __GFP_NOMEMALLOC | GFP_NOWAIT, ++ .nid = target_nid, ++ .nmask = &allowed_mask ++ }; + + if (list_empty(demote_pages)) + return 0; +@@ -1570,10 +1597,12 @@ static unsigned int demote_page_list(struct list_head *demote_pages, + if (target_nid == NUMA_NO_NODE) + return 0; + ++ node_get_allowed_targets(pgdat, &allowed_mask); ++ + /* Demotion ignores all cpuset and mempolicy settings */ + migrate_pages(demote_pages, alloc_demote_page, NULL, +- target_nid, MIGRATE_ASYNC, MR_DEMOTION, +- &nr_succeeded); ++ (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, ++ &nr_succeeded); + + if (current_is_kswapd()) + __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded); +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 4b85d3d5f7d7..33091a67627e 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -28,7 +28,6 @@ + #include + #include + #include +-#include + + #include "internal.h" + +@@ -2063,7 +2062,6 @@ static int vmstat_cpu_online(unsigned int cpu) + + if (!node_state(cpu_to_node(cpu), N_CPU)) { + node_set_state(cpu_to_node(cpu), N_CPU); +- set_migration_target_nodes(); + } + + return 0; +@@ -2088,7 +2086,6 @@ static int vmstat_cpu_dead(unsigned int cpu) + return 0; + + node_clear_state(node, N_CPU); +- set_migration_target_nodes(); + + return 0; + } +@@ -2121,7 +2118,6 @@ void __init init_mm_internals(void) + + start_shepherd_timer(); + #endif +- migrate_on_reclaim_init(); + #ifdef CONFIG_PROC_FS + proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); + proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); +-- +2.38.0 + +From 628ea87b03b347640307037c86e57664cd6bb47a Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Fri, 19 Aug 2022 17:06:47 +0200 +Subject: [PATCH 08/17] rtw88 + +Signed-off-by: Peter Jung +--- + drivers/net/wireless/realtek/rtw88/Kconfig | 47 + + drivers/net/wireless/realtek/rtw88/Makefile | 14 + + drivers/net/wireless/realtek/rtw88/coex.c | 3 +- + drivers/net/wireless/realtek/rtw88/debug.c | 15 + + drivers/net/wireless/realtek/rtw88/fw.c | 13 +- + drivers/net/wireless/realtek/rtw88/hci.h | 9 +- + drivers/net/wireless/realtek/rtw88/mac.c | 3 + + drivers/net/wireless/realtek/rtw88/mac80211.c | 2 +- + drivers/net/wireless/realtek/rtw88/main.c | 9 +- + drivers/net/wireless/realtek/rtw88/main.h | 11 +- + drivers/net/wireless/realtek/rtw88/phy.c | 6 +- + drivers/net/wireless/realtek/rtw88/ps.c | 2 +- + drivers/net/wireless/realtek/rtw88/reg.h | 1 + + drivers/net/wireless/realtek/rtw88/rtw8723d.c | 19 + + drivers/net/wireless/realtek/rtw88/rtw8723d.h | 1 + + .../net/wireless/realtek/rtw88/rtw8723du.c | 40 + + .../net/wireless/realtek/rtw88/rtw8723du.h | 13 + + drivers/net/wireless/realtek/rtw88/rtw8821c.c | 23 + + drivers/net/wireless/realtek/rtw88/rtw8821c.h | 21 + + .../net/wireless/realtek/rtw88/rtw8821cu.c | 69 ++ + .../net/wireless/realtek/rtw88/rtw8821cu.h | 15 + + drivers/net/wireless/realtek/rtw88/rtw8822b.c | 19 + + .../net/wireless/realtek/rtw88/rtw8822bu.c | 62 + + .../net/wireless/realtek/rtw88/rtw8822bu.h | 15 + + drivers/net/wireless/realtek/rtw88/rtw8822c.c | 24 + + .../net/wireless/realtek/rtw88/rtw8822cu.c | 40 + + .../net/wireless/realtek/rtw88/rtw8822cu.h | 15 + + drivers/net/wireless/realtek/rtw88/tx.h | 31 + + drivers/net/wireless/realtek/rtw88/usb.c | 1051 +++++++++++++++++ + drivers/net/wireless/realtek/rtw88/usb.h | 109 ++ + drivers/net/wireless/realtek/rtw88/util.c | 92 ++ + drivers/net/wireless/realtek/rtw88/util.h | 12 +- + 32 files changed, 1770 insertions(+), 36 deletions(-) + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8723du.c + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8723du.h + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8821cu.c + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8821cu.h + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822bu.c + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822bu.h + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822cu.c + create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822cu.h + create mode 100644 drivers/net/wireless/realtek/rtw88/usb.c + create mode 100644 drivers/net/wireless/realtek/rtw88/usb.h + +diff --git a/drivers/net/wireless/realtek/rtw88/Kconfig b/drivers/net/wireless/realtek/rtw88/Kconfig +index e3d7cb6c1290..651ab56d9c6b 100644 +--- a/drivers/net/wireless/realtek/rtw88/Kconfig ++++ b/drivers/net/wireless/realtek/rtw88/Kconfig +@@ -16,6 +16,9 @@ config RTW88_CORE + config RTW88_PCI + tristate + ++config RTW88_USB ++ tristate ++ + config RTW88_8822B + tristate + +@@ -39,6 +42,17 @@ config RTW88_8822BE + + 802.11ac PCIe wireless network adapter + ++config RTW88_8822BU ++ tristate "Realtek 8822BU USB wireless network adapter" ++ depends on USB ++ select RTW88_CORE ++ select RTW88_USB ++ select RTW88_8822B ++ help ++ Select this option will enable support for 8822BU chipset ++ ++ 802.11ac USB wireless network adapter ++ + config RTW88_8822CE + tristate "Realtek 8822CE PCI wireless network adapter" + depends on PCI +@@ -50,6 +64,17 @@ config RTW88_8822CE + + 802.11ac PCIe wireless network adapter + ++config RTW88_8822CU ++ tristate "Realtek 8822CU USB wireless network adapter" ++ depends on USB ++ select RTW88_CORE ++ select RTW88_USB ++ select RTW88_8822C ++ help ++ Select this option will enable support for 8822CU chipset ++ ++ 802.11ac USB wireless network adapter ++ + config RTW88_8723DE + tristate "Realtek 8723DE PCI wireless network adapter" + depends on PCI +@@ -61,6 +86,17 @@ config RTW88_8723DE + + 802.11n PCIe wireless network adapter + ++config RTW88_8723DU ++ tristate "Realtek 8723DU USB wireless network adapter" ++ depends on USB ++ select RTW88_CORE ++ select RTW88_USB ++ select RTW88_8723D ++ help ++ Select this option will enable support for 8723DU chipset ++ ++ 802.11n USB wireless network adapter ++ + config RTW88_8821CE + tristate "Realtek 8821CE PCI wireless network adapter" + depends on PCI +@@ -72,6 +108,17 @@ config RTW88_8821CE + + 802.11ac PCIe wireless network adapter + ++config RTW88_8821CU ++ tristate "Realtek 8821CU USB wireless network adapter" ++ depends on USB ++ select RTW88_CORE ++ select RTW88_USB ++ select RTW88_8821C ++ help ++ Select this option will enable support for 8821CU chipset ++ ++ 802.11ac USB wireless network adapter ++ + config RTW88_DEBUG + bool "Realtek rtw88 debug support" + depends on RTW88_CORE +diff --git a/drivers/net/wireless/realtek/rtw88/Makefile b/drivers/net/wireless/realtek/rtw88/Makefile +index 834c66ec0af9..e0950dbc2565 100644 +--- a/drivers/net/wireless/realtek/rtw88/Makefile ++++ b/drivers/net/wireless/realtek/rtw88/Makefile +@@ -26,23 +26,37 @@ rtw88_8822b-objs := rtw8822b.o rtw8822b_table.o + obj-$(CONFIG_RTW88_8822BE) += rtw88_8822be.o + rtw88_8822be-objs := rtw8822be.o + ++obj-$(CONFIG_RTW88_8822BU) += rtw88_8822bu.o ++rtw88_8822bu-objs := rtw8822bu.o ++ + obj-$(CONFIG_RTW88_8822C) += rtw88_8822c.o + rtw88_8822c-objs := rtw8822c.o rtw8822c_table.o + + obj-$(CONFIG_RTW88_8822CE) += rtw88_8822ce.o + rtw88_8822ce-objs := rtw8822ce.o + ++obj-$(CONFIG_RTW88_8822CU) += rtw88_8822cu.o ++rtw88_8822cu-objs := rtw8822cu.o ++ + obj-$(CONFIG_RTW88_8723D) += rtw88_8723d.o + rtw88_8723d-objs := rtw8723d.o rtw8723d_table.o + + obj-$(CONFIG_RTW88_8723DE) += rtw88_8723de.o + rtw88_8723de-objs := rtw8723de.o + ++obj-$(CONFIG_RTW88_8723DU) += rtw88_8723du.o ++rtw88_8723du-objs := rtw8723du.o ++ + obj-$(CONFIG_RTW88_8821C) += rtw88_8821c.o + rtw88_8821c-objs := rtw8821c.o rtw8821c_table.o + + obj-$(CONFIG_RTW88_8821CE) += rtw88_8821ce.o + rtw88_8821ce-objs := rtw8821ce.o + ++obj-$(CONFIG_RTW88_8821CU) += rtw88_8821cu.o ++rtw88_8821cu-objs := rtw8821cu.o ++ + obj-$(CONFIG_RTW88_PCI) += rtw88_pci.o ++obj-$(CONFIG_RTW88_USB) += rtw88_usb.o + rtw88_pci-objs := pci.o ++rtw88_usb-objs := usb.o +diff --git a/drivers/net/wireless/realtek/rtw88/coex.c b/drivers/net/wireless/realtek/rtw88/coex.c +index cac053f485c3..b156f8c48ffb 100644 +--- a/drivers/net/wireless/realtek/rtw88/coex.c ++++ b/drivers/net/wireless/realtek/rtw88/coex.c +@@ -633,7 +633,7 @@ static struct sk_buff *rtw_coex_info_request(struct rtw_dev *rtwdev, + struct rtw_coex *coex = &rtwdev->coex; + struct sk_buff *skb_resp = NULL; + +- mutex_lock(&coex->mutex); ++ lockdep_assert_held(&rtwdev->mutex); + + rtw_fw_query_bt_mp_info(rtwdev, req); + +@@ -650,7 +650,6 @@ static struct sk_buff *rtw_coex_info_request(struct rtw_dev *rtwdev, + } + + out: +- mutex_unlock(&coex->mutex); + return skb_resp; + } + +diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c +index 7cde6bcf253b..28db952be4ff 100644 +--- a/drivers/net/wireless/realtek/rtw88/debug.c ++++ b/drivers/net/wireless/realtek/rtw88/debug.c +@@ -144,7 +144,9 @@ static int rtw_debugfs_get_rf_read(struct seq_file *m, void *v) + addr = debugfs_priv->rf_addr; + mask = debugfs_priv->rf_mask; + ++ mutex_lock(&rtwdev->mutex); + val = rtw_read_rf(rtwdev, path, addr, mask); ++ mutex_unlock(&rtwdev->mutex); + + seq_printf(m, "rf_read path:%d addr:0x%08x mask:0x%08x val=0x%08x\n", + path, addr, mask, val); +@@ -390,7 +392,9 @@ static ssize_t rtw_debugfs_set_h2c(struct file *filp, + return -EINVAL; + } + ++ mutex_lock(&rtwdev->mutex); + rtw_fw_h2c_cmd_dbg(rtwdev, param); ++ mutex_unlock(&rtwdev->mutex); + + return count; + } +@@ -414,7 +418,9 @@ static ssize_t rtw_debugfs_set_rf_write(struct file *filp, + return count; + } + ++ mutex_lock(&rtwdev->mutex); + rtw_write_rf(rtwdev, path, addr, mask, val); ++ mutex_unlock(&rtwdev->mutex); + rtw_dbg(rtwdev, RTW_DBG_DEBUGFS, + "write_rf path:%d addr:0x%08x mask:0x%08x, val:0x%08x\n", + path, addr, mask, val); +@@ -519,6 +525,8 @@ static int rtw_debug_get_rf_dump(struct seq_file *m, void *v) + u32 addr, offset, data; + u8 path; + ++ mutex_lock(&rtwdev->mutex); ++ + for (path = 0; path < rtwdev->hal.rf_path_num; path++) { + seq_printf(m, "RF path:%d\n", path); + for (addr = 0; addr < 0x100; addr += 4) { +@@ -533,6 +541,8 @@ static int rtw_debug_get_rf_dump(struct seq_file *m, void *v) + seq_puts(m, "\n"); + } + ++ mutex_unlock(&rtwdev->mutex); ++ + return 0; + } + +@@ -828,7 +838,9 @@ static int rtw_debugfs_get_coex_info(struct seq_file *m, void *v) + struct rtw_debugfs_priv *debugfs_priv = m->private; + struct rtw_dev *rtwdev = debugfs_priv->rtwdev; + ++ mutex_lock(&rtwdev->mutex); + rtw_coex_display_coex_info(rtwdev, m); ++ mutex_unlock(&rtwdev->mutex); + + return 0; + } +@@ -1023,6 +1035,8 @@ static void dump_gapk_status(struct rtw_dev *rtwdev, struct seq_file *m) + dm_info->dm_flags & BIT(RTW_DM_CAP_TXGAPK) ? '-' : '+', + rtw_dm_cap_strs[RTW_DM_CAP_TXGAPK]); + ++ mutex_lock(&rtwdev->mutex); ++ + for (path = 0; path < rtwdev->hal.rf_path_num; path++) { + val = rtw_read_rf(rtwdev, path, RF_GAINTX, RFREG_MASK); + seq_printf(m, "path %d:\n0x%x = 0x%x\n", path, RF_GAINTX, val); +@@ -1032,6 +1046,7 @@ static void dump_gapk_status(struct rtw_dev *rtwdev, struct seq_file *m) + txgapk->rf3f_fs[path][i], i); + seq_puts(m, "\n"); + } ++ mutex_unlock(&rtwdev->mutex); + } + + static int rtw_debugfs_get_dm_cap(struct seq_file *m, void *v) +diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c +index 4fdab0329695..5e912421408e 100644 +--- a/drivers/net/wireless/realtek/rtw88/fw.c ++++ b/drivers/net/wireless/realtek/rtw88/fw.c +@@ -320,7 +320,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, + h2c[3], h2c[2], h2c[1], h2c[0], + h2c[7], h2c[6], h2c[5], h2c[4]); + +- spin_lock(&rtwdev->h2c.lock); ++ lockdep_assert_held(&rtwdev->mutex); + + box = rtwdev->h2c.last_box_num; + switch (box) { +@@ -342,7 +342,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, + break; + default: + WARN(1, "invalid h2c mail box number\n"); +- goto out; ++ return; + } + + ret = read_poll_timeout_atomic(rtw_read8, box_state, +@@ -351,7 +351,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, + + if (ret) { + rtw_err(rtwdev, "failed to send h2c command\n"); +- goto out; ++ return; + } + + for (idx = 0; idx < 4; idx++) +@@ -361,9 +361,6 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, + + if (++rtwdev->h2c.last_box_num >= 4) + rtwdev->h2c.last_box_num = 0; +- +-out: +- spin_unlock(&rtwdev->h2c.lock); + } + + void rtw_fw_h2c_cmd_dbg(struct rtw_dev *rtwdev, u8 *h2c) +@@ -375,15 +372,13 @@ static void rtw_fw_send_h2c_packet(struct rtw_dev *rtwdev, u8 *h2c_pkt) + { + int ret; + +- spin_lock(&rtwdev->h2c.lock); ++ lockdep_assert_held(&rtwdev->mutex); + + FW_OFFLOAD_H2C_SET_SEQ_NUM(h2c_pkt, rtwdev->h2c.seq); + ret = rtw_hci_write_data_h2c(rtwdev, h2c_pkt, H2C_PKT_SIZE); + if (ret) + rtw_err(rtwdev, "failed to send h2c packet\n"); + rtwdev->h2c.seq++; +- +- spin_unlock(&rtwdev->h2c.lock); + } + + void +diff --git a/drivers/net/wireless/realtek/rtw88/hci.h b/drivers/net/wireless/realtek/rtw88/hci.h +index 4c6fc6fb3f83..830d7532f2a3 100644 +--- a/drivers/net/wireless/realtek/rtw88/hci.h ++++ b/drivers/net/wireless/realtek/rtw88/hci.h +@@ -166,12 +166,11 @@ static inline u32 + rtw_read_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, + u32 addr, u32 mask) + { +- unsigned long flags; + u32 val; + +- spin_lock_irqsave(&rtwdev->rf_lock, flags); ++ lockdep_assert_held(&rtwdev->mutex); ++ + val = rtwdev->chip->ops->read_rf(rtwdev, rf_path, addr, mask); +- spin_unlock_irqrestore(&rtwdev->rf_lock, flags); + + return val; + } +@@ -180,11 +179,9 @@ static inline void + rtw_write_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, + u32 addr, u32 mask, u32 data) + { +- unsigned long flags; ++ lockdep_assert_held(&rtwdev->mutex); + +- spin_lock_irqsave(&rtwdev->rf_lock, flags); + rtwdev->chip->ops->write_rf(rtwdev, rf_path, addr, mask, data); +- spin_unlock_irqrestore(&rtwdev->rf_lock, flags); + } + + static inline u32 +diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c +index caf2603da2d6..c3c24850a427 100644 +--- a/drivers/net/wireless/realtek/rtw88/mac.c ++++ b/drivers/net/wireless/realtek/rtw88/mac.c +@@ -1032,6 +1032,9 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev) + if (rtw_chip_wcpu_11ac(rtwdev)) + rtw_write32(rtwdev, REG_H2CQ_CSR, BIT_H2CQ_FULL); + ++ if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB) ++ rtw_write8_set(rtwdev, REG_TXDMA_PQ_MAP, BIT_RXDMA_ARBBW_EN); ++ + return 0; + } + +diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c +index c7b98a0599d5..bc30400e67a1 100644 +--- a/drivers/net/wireless/realtek/rtw88/mac80211.c ++++ b/drivers/net/wireless/realtek/rtw88/mac80211.c +@@ -483,8 +483,8 @@ static int rtw_ops_sta_remove(struct ieee80211_hw *hw, + { + struct rtw_dev *rtwdev = hw->priv; + +- rtw_fw_beacon_filter_config(rtwdev, false, vif); + mutex_lock(&rtwdev->mutex); ++ rtw_fw_beacon_filter_config(rtwdev, false, vif); + rtw_sta_remove(rtwdev, sta, true); + mutex_unlock(&rtwdev->mutex); + +diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c +index 76dc9da88f6c..0edf22d9c1ef 100644 +--- a/drivers/net/wireless/realtek/rtw88/main.c ++++ b/drivers/net/wireless/realtek/rtw88/main.c +@@ -1733,6 +1733,10 @@ static int rtw_chip_parameter_setup(struct rtw_dev *rtwdev) + rtwdev->hci.rpwm_addr = 0x03d9; + rtwdev->hci.cpwm_addr = 0x03da; + break; ++ case RTW_HCI_TYPE_USB: ++ rtwdev->hci.rpwm_addr = 0xfe58; ++ rtwdev->hci.cpwm_addr = 0xfe57; ++ break; + default: + rtw_err(rtwdev, "unsupported hci type\n"); + return -EINVAL; +@@ -2017,13 +2021,10 @@ int rtw_core_init(struct rtw_dev *rtwdev) + skb_queue_head_init(&rtwdev->coex.queue); + skb_queue_head_init(&rtwdev->tx_report.queue); + +- spin_lock_init(&rtwdev->rf_lock); +- spin_lock_init(&rtwdev->h2c.lock); + spin_lock_init(&rtwdev->txq_lock); + spin_lock_init(&rtwdev->tx_report.q_lock); + + mutex_init(&rtwdev->mutex); +- mutex_init(&rtwdev->coex.mutex); + mutex_init(&rtwdev->hal.tx_power_mutex); + + init_waitqueue_head(&rtwdev->coex.wait); +@@ -2091,7 +2092,6 @@ void rtw_core_deinit(struct rtw_dev *rtwdev) + } + + mutex_destroy(&rtwdev->mutex); +- mutex_destroy(&rtwdev->coex.mutex); + mutex_destroy(&rtwdev->hal.tx_power_mutex); + } + EXPORT_SYMBOL(rtw_core_deinit); +@@ -2132,6 +2132,7 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) + hw->wiphy->available_antennas_rx = hal->antenna_rx; + + hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS | ++ WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL | + WIPHY_FLAG_TDLS_EXTERNAL_SETUP; + + hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR; +diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h +index 7db627fc26be..663b8dc453da 100644 +--- a/drivers/net/wireless/realtek/rtw88/main.h ++++ b/drivers/net/wireless/realtek/rtw88/main.h +@@ -875,6 +875,10 @@ struct rtw_chip_ops { + bool is_tx2_path); + void (*config_txrx_mode)(struct rtw_dev *rtwdev, u8 tx_path, + u8 rx_path, bool is_tx2_path); ++ /* for USB/SDIO only */ ++ void (*fill_txdesc_checksum)(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *txdesc); + + /* for coex */ + void (*coex_set_init)(struct rtw_dev *rtwdev); +@@ -1504,8 +1508,6 @@ struct rtw_coex_stat { + }; + + struct rtw_coex { +- /* protects coex info request section */ +- struct mutex mutex; + struct sk_buff_head queue; + wait_queue_head_t wait; + +@@ -1994,9 +1996,6 @@ struct rtw_dev { + /* ensures exclusive access from mac80211 callbacks */ + struct mutex mutex; + +- /* read/write rf register */ +- spinlock_t rf_lock; +- + /* watch dog every 2 sec */ + struct delayed_work watch_dog_work; + u32 watch_dog_cnt; +@@ -2022,8 +2021,6 @@ struct rtw_dev { + struct { + /* incicate the mail box to use with fw */ + u8 last_box_num; +- /* protect to send h2c to fw */ +- spinlock_t lock; + u32 seq; + } h2c; + +diff --git a/drivers/net/wireless/realtek/rtw88/phy.c b/drivers/net/wireless/realtek/rtw88/phy.c +index 8982e0c98dac..f6de6568f962 100644 +--- a/drivers/net/wireless/realtek/rtw88/phy.c ++++ b/drivers/net/wireless/realtek/rtw88/phy.c +@@ -300,7 +300,7 @@ static void rtw_phy_stat_rssi(struct rtw_dev *rtwdev) + + data.rtwdev = rtwdev; + data.min_rssi = U8_MAX; +- rtw_iterate_stas_atomic(rtwdev, rtw_phy_stat_rssi_iter, &data); ++ rtw_iterate_stas(rtwdev, rtw_phy_stat_rssi_iter, &data); + + dm_info->pre_min_rssi = dm_info->min_rssi; + dm_info->min_rssi = data.min_rssi; +@@ -544,7 +544,7 @@ static void rtw_phy_ra_info_update(struct rtw_dev *rtwdev) + if (rtwdev->watch_dog_cnt & 0x3) + return; + +- rtw_iterate_stas_atomic(rtwdev, rtw_phy_ra_info_update_iter, rtwdev); ++ rtw_iterate_stas(rtwdev, rtw_phy_ra_info_update_iter, rtwdev); + } + + static u32 rtw_phy_get_rrsr_mask(struct rtw_dev *rtwdev, u8 rate_idx) +@@ -597,7 +597,7 @@ static void rtw_phy_rrsr_update(struct rtw_dev *rtwdev) + struct rtw_dm_info *dm_info = &rtwdev->dm_info; + + dm_info->rrsr_mask_min = RRSR_RATE_ORDER_MAX; +- rtw_iterate_stas_atomic(rtwdev, rtw_phy_rrsr_mask_min_iter, rtwdev); ++ rtw_iterate_stas(rtwdev, rtw_phy_rrsr_mask_min_iter, rtwdev); + rtw_write32(rtwdev, REG_RRSR, dm_info->rrsr_val_init & dm_info->rrsr_mask_min); + } + +diff --git a/drivers/net/wireless/realtek/rtw88/ps.c b/drivers/net/wireless/realtek/rtw88/ps.c +index bfa64c038f5f..a7213ff2c224 100644 +--- a/drivers/net/wireless/realtek/rtw88/ps.c ++++ b/drivers/net/wireless/realtek/rtw88/ps.c +@@ -58,7 +58,7 @@ int rtw_leave_ips(struct rtw_dev *rtwdev) + return ret; + } + +- rtw_iterate_vifs_atomic(rtwdev, rtw_restore_port_cfg_iter, rtwdev); ++ rtw_iterate_vifs(rtwdev, rtw_restore_port_cfg_iter, rtwdev); + + rtw_coex_ips_notify(rtwdev, COEX_IPS_LEAVE); + +diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h +index 03bd8dc53f72..8852b24d6c2a 100644 +--- a/drivers/net/wireless/realtek/rtw88/reg.h ++++ b/drivers/net/wireless/realtek/rtw88/reg.h +@@ -184,6 +184,7 @@ + #define BIT_TXDMA_VIQ_MAP(x) \ + (((x) & BIT_MASK_TXDMA_VIQ_MAP) << BIT_SHIFT_TXDMA_VIQ_MAP) + #define REG_TXDMA_PQ_MAP 0x010C ++#define BIT_RXDMA_ARBBW_EN BIT(0) + #define BIT_SHIFT_TXDMA_BEQ_MAP 8 + #define BIT_MASK_TXDMA_BEQ_MAP 0x3 + #define BIT_TXDMA_BEQ_MAP(x) \ +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.c b/drivers/net/wireless/realtek/rtw88/rtw8723d.c +index 993bd6b1d723..d99c10f34f92 100644 +--- a/drivers/net/wireless/realtek/rtw88/rtw8723d.c ++++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.c +@@ -210,6 +210,12 @@ static void rtw8723de_efuse_parsing(struct rtw_efuse *efuse, + ether_addr_copy(efuse->addr, map->e.mac_addr); + } + ++static void rtw8723du_efuse_parsing(struct rtw_efuse *efuse, ++ struct rtw8723d_efuse *map) ++{ ++ ether_addr_copy(efuse->addr, map->u.mac_addr); ++} ++ + static int rtw8723d_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + { + struct rtw_efuse *efuse = &rtwdev->efuse; +@@ -239,6 +245,9 @@ static int rtw8723d_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + case RTW_HCI_TYPE_PCIE: + rtw8723de_efuse_parsing(efuse, map); + break; ++ case RTW_HCI_TYPE_USB: ++ rtw8723du_efuse_parsing(efuse, map); ++ break; + default: + /* unsupported now */ + return -ENOTSUPP; +@@ -1945,6 +1954,15 @@ static void rtw8723d_pwr_track(struct rtw_dev *rtwdev) + dm_info->pwr_trk_triggered = false; + } + ++static void rtw8723d_fill_txdesc_checksum(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *txdesc) ++{ ++ size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */ ++ ++ fill_txdesc_checksum_common(txdesc, words); ++} ++ + static struct rtw_chip_ops rtw8723d_ops = { + .phy_set_param = rtw8723d_phy_set_param, + .read_efuse = rtw8723d_read_efuse, +@@ -1965,6 +1983,7 @@ static struct rtw_chip_ops rtw8723d_ops = { + .config_bfee = NULL, + .set_gid_table = NULL, + .cfg_csi_rate = NULL, ++ .fill_txdesc_checksum = rtw8723d_fill_txdesc_checksum, + + .coex_set_init = rtw8723d_coex_cfg_init, + .coex_set_ant_switch = NULL, +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.h b/drivers/net/wireless/realtek/rtw88/rtw8723d.h +index 4641f6e047b4..7d9fc3dde97d 100644 +--- a/drivers/net/wireless/realtek/rtw88/rtw8723d.h ++++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.h +@@ -70,6 +70,7 @@ struct rtw8723d_efuse { + u8 country_code[2]; + u8 res[3]; + struct rtw8723de_efuse e; ++ struct rtw8723de_efuse u; + }; + + extern const struct rtw_chip_info rtw8723d_hw_spec; +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723du.c b/drivers/net/wireless/realtek/rtw88/rtw8723du.c +new file mode 100644 +index 000000000000..910f64c16813 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8723du.c +@@ -0,0 +1,40 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#include ++#include ++#include "main.h" ++#include "rtw8723du.h" ++#include "usb.h" ++ ++static const struct usb_device_id rtw_8723du_id_table[] = { ++ /* ++ * ULLI : ++ * ID found in rtw8822bu sources ++ */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xD723, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8723d_hw_spec) }, /* 8723DU 1*1 */ ++ { }, ++}; ++MODULE_DEVICE_TABLE(usb, rtw_8723du_id_table); ++ ++static int rtw8723du_probe(struct usb_interface *intf, ++ const struct usb_device_id *id) ++{ ++ return rtw_usb_probe(intf, id); ++} ++ ++static struct usb_driver rtw_8723du_driver = { ++ .name = "rtw_8723du", ++ .id_table = rtw_8723du_id_table, ++ .probe = rtw8723du_probe, ++ .disconnect = rtw_usb_disconnect, ++}; ++module_usb_driver(rtw_8723du_driver); ++ ++MODULE_AUTHOR("Hans Ulli Kroll "); ++MODULE_DESCRIPTION("Realtek 802.11n wireless 8723du driver"); ++MODULE_LICENSE("Dual BSD/GPL"); +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723du.h b/drivers/net/wireless/realtek/rtw88/rtw8723du.h +new file mode 100644 +index 000000000000..2e069f65c055 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8723du.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#ifndef __RTW_8723DU_H_ ++#define __RTW_8723DU_H_ ++ ++/* USB Vendor/Product IDs */ ++#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA ++ ++extern struct rtw_chip_info rtw8723d_hw_spec; ++ ++#endif +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.c b/drivers/net/wireless/realtek/rtw88/rtw8821c.c +index 025262a8970e..5b4ea9990b78 100644 +--- a/drivers/net/wireless/realtek/rtw88/rtw8821c.c ++++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.c +@@ -26,6 +26,12 @@ static void rtw8821ce_efuse_parsing(struct rtw_efuse *efuse, + ether_addr_copy(efuse->addr, map->e.mac_addr); + } + ++static void rtw8821cu_efuse_parsing(struct rtw_efuse *efuse, ++ struct rtw8821c_efuse *map) ++{ ++ ether_addr_copy(efuse->addr, map->u.mac_addr); ++} ++ + enum rtw8821ce_rf_set { + SWITCH_TO_BTG, + SWITCH_TO_WLG, +@@ -68,6 +74,9 @@ static int rtw8821c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + case RTW_HCI_TYPE_PCIE: + rtw8821ce_efuse_parsing(efuse, map); + break; ++ case RTW_HCI_TYPE_USB: ++ rtw8821cu_efuse_parsing(efuse, map); ++ break; + default: + /* unsupported now */ + return -ENOTSUPP; +@@ -1148,6 +1157,18 @@ static void rtw8821c_phy_cck_pd_set(struct rtw_dev *rtwdev, u8 new_lvl) + dm_info->cck_pd_default + new_lvl * 2); + } + ++static void rtw8821c_fill_txdesc_checksum(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *txdesc) ++{ ++ struct rtw_chip_info *chip = rtwdev->chip; ++ size_t words; ++ ++ words = (pkt_info->pkt_offset * 8 + chip->tx_pkt_desc_sz) / 2; ++ ++ fill_txdesc_checksum_common(txdesc, words); ++} ++ + static struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8821c[] = { + {0x0086, + RTW_PWR_CUT_ALL_MSK, +@@ -1521,6 +1542,7 @@ static const struct rtw_rfe_def rtw8821c_rfe_defs[] = { + [2] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2), + [4] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2), + [6] = RTW_DEF_RFE(8821c, 0, 0), ++ [34] = RTW_DEF_RFE(8821c, 0, 0), + }; + + static struct rtw_hw_reg rtw8821c_dig[] = { +@@ -1595,6 +1617,7 @@ static struct rtw_chip_ops rtw8821c_ops = { + .config_bfee = rtw8821c_bf_config_bfee, + .set_gid_table = rtw_bf_set_gid_table, + .cfg_csi_rate = rtw_bf_cfg_csi_rate, ++ .fill_txdesc_checksum = rtw8821c_fill_txdesc_checksum, + + .coex_set_init = rtw8821c_coex_cfg_init, + .coex_set_ant_switch = rtw8821c_coex_cfg_ant_switch, +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.h b/drivers/net/wireless/realtek/rtw88/rtw8821c.h +index 2698801fc35d..71ce01623d49 100644 +--- a/drivers/net/wireless/realtek/rtw88/rtw8821c.h ++++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.h +@@ -9,6 +9,26 @@ + + #define RCR_VHT_ACK BIT(26) + ++struct rtw8821cu_efuse { ++ u8 res4[4]; /* 0xd0 */ ++ u8 usb_optional_function; ++ u8 res5[0x1e]; ++ u8 res6[2]; ++ u8 serial[0x0b]; /* 0xf5 */ ++ u8 vid; /* 0x100 */ ++ u8 res7; ++ u8 pid; ++ u8 res8[4]; ++ u8 mac_addr[ETH_ALEN]; /* 0x107 */ ++ u8 res9[2]; ++ u8 vendor_name[0x07]; ++ u8 res10[2]; ++ u8 device_name[0x14]; ++ u8 res11[0xcf]; ++ u8 package_type; /* 0x1fb */ ++ u8 res12[0x4]; ++}; ++ + struct rtw8821ce_efuse { + u8 mac_addr[ETH_ALEN]; /* 0xd0 */ + u8 vender_id[2]; +@@ -73,6 +93,7 @@ struct rtw8821c_efuse { + u8 res[3]; + union { + struct rtw8821ce_efuse e; ++ struct rtw8821cu_efuse u; + }; + }; + +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c +new file mode 100644 +index 000000000000..e6710c5ebdfc +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c +@@ -0,0 +1,69 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#include ++#include ++#include "main.h" ++#include "rtw8821cu.h" ++#include "usb.h" ++ ++static const struct usb_device_id rtw_8821cu_id_table[] = { ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xb82b, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xb820, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xC821, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xC820, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xC82A, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xC82B, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0xC811, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8811CU */ ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ 0x8811, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8811CU */ ++ /*=== Customer ID ===*/ ++ { USB_DEVICE(0x0bda, 0x2006), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Totolink */ ++ { USB_DEVICE(0x0bda, 0xc811), ++ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Simplecom NW602 */ ++ {}, ++}; ++MODULE_DEVICE_TABLE(usb, rtw_8821cu_id_table); ++ ++static int rtw_8821cu_probe(struct usb_interface *intf, ++ const struct usb_device_id *id) ++{ ++ return rtw_usb_probe(intf, id); ++} ++ ++static struct usb_driver rtw_8821cu_driver = { ++ .name = "rtw_8821cu", ++ .id_table = rtw_8821cu_id_table, ++ .probe = rtw_8821cu_probe, ++ .disconnect = rtw_usb_disconnect, ++}; ++module_usb_driver(rtw_8821cu_driver); ++ ++MODULE_AUTHOR("Hans Ulli Kroll "); ++MODULE_DESCRIPTION("Realtek 802.11ac wireless 8821cu driver"); ++MODULE_LICENSE("Dual BSD/GPL"); +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.h b/drivers/net/wireless/realtek/rtw88/rtw8821cu.h +new file mode 100644 +index 000000000000..bddbd96aa45f +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#ifndef __RTW_8821CU_H_ ++#define __RTW_8821CU_H_ ++ ++/* USB Vendor/Product IDs */ ++#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA ++#define RTW_USB_PRODUCT_ID_REALTEK_8811C 0xC811 ++#define RTW_USB_PRODUCT_ID_REALTEK_8821C 0xC81C ++ ++extern struct rtw_chip_info rtw8821c_hw_spec; ++ ++#endif +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c +index 321848870561..fd1e9a47ccef 100644 +--- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c ++++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c +@@ -26,6 +26,12 @@ static void rtw8822be_efuse_parsing(struct rtw_efuse *efuse, + ether_addr_copy(efuse->addr, map->e.mac_addr); + } + ++static void rtw8822bu_efuse_parsing(struct rtw_efuse *efuse, ++ struct rtw8822b_efuse *map) ++{ ++ ether_addr_copy(efuse->addr, map->u.mac_addr); ++} ++ + static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + { + struct rtw_efuse *efuse = &rtwdev->efuse; +@@ -56,6 +62,9 @@ static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + case RTW_HCI_TYPE_PCIE: + rtw8822be_efuse_parsing(efuse, map); + break; ++ case RTW_HCI_TYPE_USB: ++ rtw8822bu_efuse_parsing(efuse, map); ++ break; + default: + /* unsupported now */ + return -ENOTSUPP; +@@ -1588,6 +1597,15 @@ static void rtw8822b_adaptivity(struct rtw_dev *rtwdev) + rtw_phy_set_edcca_th(rtwdev, l2h, h2l); + } + ++static void rtw8822b_fill_txdesc_checksum(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *txdesc) ++{ ++ size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */ ++ ++ fill_txdesc_checksum_common(txdesc, words); ++} ++ + static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8822b[] = { + {0x0086, + RTW_PWR_CUT_ALL_MSK, +@@ -2163,6 +2181,7 @@ static struct rtw_chip_ops rtw8822b_ops = { + .cfg_csi_rate = rtw_bf_cfg_csi_rate, + .adaptivity_init = rtw8822b_adaptivity_init, + .adaptivity = rtw8822b_adaptivity, ++ .fill_txdesc_checksum = rtw8822b_fill_txdesc_checksum, + + .coex_set_init = rtw8822b_coex_cfg_init, + .coex_set_ant_switch = rtw8822b_coex_cfg_ant_switch, +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bu.c b/drivers/net/wireless/realtek/rtw88/rtw8822bu.c +new file mode 100644 +index 000000000000..5becebdc3247 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8822bu.c +@@ -0,0 +1,62 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#include ++#include ++#include "main.h" ++#include "rtw8822bu.h" ++#include "usb.h" ++ ++#define RTW_USB_VENDER_ID_EDIMAX 0x7392 ++ ++static const struct usb_device_id rtw_8822bu_id_table[] = { ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ RTW_USB_PRODUCT_ID_REALTEK_8812B, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ RTW_USB_PRODUCT_ID_REALTEK_8822B, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDER_ID_EDIMAX, ++ 0xB822, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDER_ID_EDIMAX, ++ 0xC822, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE(0x0b05, 0x184c), /* ASUS AC53 Nano */ ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE(0x0b05, 0x1841), /* ASUS AC55 B1 */ ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE(0x2001, 0x331c), /* D-Link DWA-182 rev D1 */ ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE(0x13b1, 0x0043), /* Linksys WUSB6400M */ ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE(0x2357, 0x012D), /* TP-Link AC1300 T3U */ ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ { USB_DEVICE(0x2357, 0x0138), /* TP-Link AC1300 T3U */ ++ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(usb, rtw_8822bu_id_table); ++ ++static int rtw8822bu_probe(struct usb_interface *intf, ++ const struct usb_device_id *id) ++{ ++ return rtw_usb_probe(intf, id); ++} ++ ++static struct usb_driver rtw_8822bu_driver = { ++ .name = "rtw_8822bu", ++ .id_table = rtw_8822bu_id_table, ++ .probe = rtw8822bu_probe, ++ .disconnect = rtw_usb_disconnect, ++}; ++module_usb_driver(rtw_8822bu_driver); ++ ++MODULE_AUTHOR("Realtek Corporation"); ++MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822bu driver"); ++MODULE_LICENSE("Dual BSD/GPL"); +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bu.h b/drivers/net/wireless/realtek/rtw88/rtw8822bu.h +new file mode 100644 +index 000000000000..20f01ecd7441 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8822bu.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#ifndef __RTW_8822BU_H_ ++#define __RTW_8822BU_H_ ++ ++/* USB Vendor/Product IDs */ ++#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA ++#define RTW_USB_PRODUCT_ID_REALTEK_8812B 0xB812 ++#define RTW_USB_PRODUCT_ID_REALTEK_8822B 0xB82C ++ ++extern struct rtw_chip_info rtw8822b_hw_spec; ++ ++#endif +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c +index 09f9e4adcf34..c0283c8278da 100644 +--- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c ++++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c +@@ -29,6 +29,12 @@ static void rtw8822ce_efuse_parsing(struct rtw_efuse *efuse, + ether_addr_copy(efuse->addr, map->e.mac_addr); + } + ++static void rtw8822cu_efuse_parsing(struct rtw_efuse *efuse, ++ struct rtw8822c_efuse *map) ++{ ++ ether_addr_copy(efuse->addr, map->u.mac_addr); ++} ++ + static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + { + struct rtw_efuse *efuse = &rtwdev->efuse; +@@ -58,6 +64,9 @@ static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) + case RTW_HCI_TYPE_PCIE: + rtw8822ce_efuse_parsing(efuse, map); + break; ++ case RTW_HCI_TYPE_USB: ++ rtw8822cu_efuse_parsing(efuse, map); ++ break; + default: + /* unsupported now */ + return -ENOTSUPP; +@@ -4557,6 +4566,18 @@ static void rtw8822c_adaptivity(struct rtw_dev *rtwdev) + rtw_phy_set_edcca_th(rtwdev, l2h, h2l); + } + ++static void rtw8822c_fill_txdesc_checksum(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *txdesc) ++{ ++ struct rtw_chip_info *chip = rtwdev->chip; ++ size_t words; ++ ++ words = (pkt_info->pkt_offset * 8 + chip->tx_pkt_desc_sz) / 2; ++ ++ fill_txdesc_checksum_common(txdesc, words); ++} ++ + static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8822c[] = { + {0x0086, + RTW_PWR_CUT_ALL_MSK, +@@ -4895,6 +4916,8 @@ static const struct rtw_rfe_def rtw8822c_rfe_defs[] = { + [0] = RTW_DEF_RFE(8822c, 0, 0), + [1] = RTW_DEF_RFE(8822c, 0, 0), + [2] = RTW_DEF_RFE(8822c, 0, 0), ++ [3] = RTW_DEF_RFE(8822c, 0, 0), ++ [4] = RTW_DEF_RFE(8822c, 0, 0), + [5] = RTW_DEF_RFE(8822c, 0, 5), + [6] = RTW_DEF_RFE(8822c, 0, 0), + }; +@@ -4978,6 +5001,7 @@ static struct rtw_chip_ops rtw8822c_ops = { + .cfo_track = rtw8822c_cfo_track, + .config_tx_path = rtw8822c_config_tx_path, + .config_txrx_mode = rtw8822c_config_trx_mode, ++ .fill_txdesc_checksum = rtw8822c_fill_txdesc_checksum, + + .coex_set_init = rtw8822c_coex_cfg_init, + .coex_set_ant_switch = NULL, +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.c b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c +new file mode 100644 +index 000000000000..36dc734f76eb +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c +@@ -0,0 +1,40 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#include ++#include ++#include "main.h" ++#include "rtw8822cu.h" ++#include "usb.h" ++ ++static const struct usb_device_id rtw_8822cu_id_table[] = { ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ RTW_USB_PRODUCT_ID_REALTEK_8822C, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, ++ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, ++ RTW_USB_PRODUCT_ID_REALTEK_8812C, ++ 0xff, 0xff, 0xff), ++ .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(usb, rtw_8822cu_id_table); ++ ++static int rtw8822bu_probe(struct usb_interface *intf, ++ const struct usb_device_id *id) ++{ ++ return rtw_usb_probe(intf, id); ++} ++ ++static struct usb_driver rtw_8822cu_driver = { ++ .name = "rtw_8822cu", ++ .id_table = rtw_8822cu_id_table, ++ .probe = rtw8822bu_probe, ++ .disconnect = rtw_usb_disconnect, ++}; ++module_usb_driver(rtw_8822cu_driver); ++ ++MODULE_AUTHOR("Realtek Corporation"); ++MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822cu driver"); ++MODULE_LICENSE("Dual BSD/GPL"); +diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.h b/drivers/net/wireless/realtek/rtw88/rtw8822cu.h +new file mode 100644 +index 000000000000..16afe22a8216 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#ifndef __RTW_8822CU_H_ ++#define __RTW_8822CU_H_ ++ ++/* USB Vendor/Product IDs */ ++#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA ++#define RTW_USB_PRODUCT_ID_REALTEK_8812C 0xC812 ++#define RTW_USB_PRODUCT_ID_REALTEK_8822C 0xC82C ++ ++extern struct rtw_chip_info rtw8822c_hw_spec; ++ ++#endif +diff --git a/drivers/net/wireless/realtek/rtw88/tx.h b/drivers/net/wireless/realtek/rtw88/tx.h +index 8419603adce4..d283bb05d1dd 100644 +--- a/drivers/net/wireless/realtek/rtw88/tx.h ++++ b/drivers/net/wireless/realtek/rtw88/tx.h +@@ -71,6 +71,14 @@ + le32p_replace_bits((__le32 *)(txdesc) + 0x03, value, BIT(15)) + #define SET_TX_DESC_BT_NULL(txdesc, value) \ + le32p_replace_bits((__le32 *)(txdesc) + 0x02, value, BIT(23)) ++#define SET_TX_DESC_TXDESC_CHECKSUM(txdesc, value) \ ++ le32p_replace_bits((__le32 *)(txdesc) + 0x07, value, GENMASK(15, 0)) ++#define SET_TX_DESC_DMA_TXAGG_NUM(txdesc, value) \ ++ le32p_replace_bits((__le32 *)(txdesc) + 0x07, value, GENMASK(31, 24)) ++#define GET_TX_DESC_PKT_OFFSET(txdesc) \ ++ le32_get_bits(*((__le32 *)(txdesc) + 0x01), GENMASK(28, 24)) ++#define GET_TX_DESC_QSEL(txdesc) \ ++ le32_get_bits(*((__le32 *)(txdesc) + 0x01), GENMASK(12, 8)) + + enum rtw_tx_desc_queue_select { + TX_DESC_QSEL_TID0 = 0, +@@ -123,4 +131,27 @@ rtw_tx_write_data_h2c_get(struct rtw_dev *rtwdev, + struct rtw_tx_pkt_info *pkt_info, + u8 *buf, u32 size); + ++static inline ++void fill_txdesc_checksum_common(u8 *txdesc, size_t words) ++{ ++ __le16 chksum = 0; ++ __le16 *data = (__le16 *)(txdesc); ++ ++ SET_TX_DESC_TXDESC_CHECKSUM(txdesc, 0x0000); ++ ++ while (words--) ++ chksum ^= *data++; ++ ++ SET_TX_DESC_TXDESC_CHECKSUM(txdesc, __le16_to_cpu(chksum)); ++} ++ ++static inline void rtw_tx_fill_txdesc_checksum(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *txdesc) ++{ ++ struct rtw_chip_info *chip = rtwdev->chip; ++ ++ chip->ops->fill_txdesc_checksum(rtwdev, pkt_info, txdesc); ++} ++ + #endif +diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c +new file mode 100644 +index 000000000000..7641ea6f6ad1 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/usb.c +@@ -0,0 +1,1051 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#include ++#include ++#include ++#include "main.h" ++#include "debug.h" ++#include "reg.h" ++#include "tx.h" ++#include "rx.h" ++#include "fw.h" ++#include "ps.h" ++#include "usb.h" ++ ++#define RTW_USB_MAX_RXQ_LEN 128 ++ ++struct rtw_usb_txcb { ++ struct rtw_dev *rtwdev; ++ struct sk_buff_head tx_ack_queue; ++}; ++ ++static void rtw_usb_fill_tx_checksum(struct rtw_usb *rtwusb, ++ struct sk_buff *skb, int agg_num) ++{ ++ struct rtw_dev *rtwdev = rtwusb->rtwdev; ++ struct rtw_tx_pkt_info pkt_info; ++ ++ SET_TX_DESC_DMA_TXAGG_NUM(skb->data, agg_num); ++ pkt_info.pkt_offset = GET_TX_DESC_PKT_OFFSET(skb->data); ++ rtw_tx_fill_txdesc_checksum(rtwdev, &pkt_info, skb->data); ++} ++ ++static void usbctrl_async_callback(struct urb *urb) ++{ ++ /* free dr */ ++ kfree(urb->setup_packet); ++ /* free databuf */ ++ kfree(urb->transfer_buffer); ++} ++ ++static int usbctrl_vendorreq_async_write(struct usb_device *udev, u8 request, ++ u16 value, u16 index, void *pdata, ++ u16 len) ++{ ++ int rc; ++ unsigned int pipe; ++ u8 reqtype; ++ struct usb_ctrlrequest *dr; ++ struct urb *urb; ++ const u16 databuf_maxlen = RTW_USB_VENQT_MAX_BUF_SIZE; ++ u8 *databuf; ++ ++ if (WARN_ON_ONCE(len > databuf_maxlen)) ++ len = databuf_maxlen; ++ ++ pipe = usb_sndctrlpipe(udev, 0); /* write_out */ ++ reqtype = RTW_USB_CMD_WRITE; ++ ++ dr = kzalloc(sizeof(*dr), GFP_ATOMIC); ++ if (!dr) ++ return -ENOMEM; ++ ++ databuf = kmemdup(pdata, len, GFP_ATOMIC); ++ if (!databuf) { ++ kfree(dr); ++ return -ENOMEM; ++ } ++ ++ urb = usb_alloc_urb(0, GFP_ATOMIC); ++ if (!urb) { ++ kfree(databuf); ++ kfree(dr); ++ return -ENOMEM; ++ } ++ ++ dr->bRequestType = reqtype; ++ dr->bRequest = request; ++ dr->wValue = cpu_to_le16(value); ++ dr->wIndex = cpu_to_le16(index); ++ dr->wLength = cpu_to_le16(len); ++ ++ usb_fill_control_urb(urb, udev, pipe, ++ (unsigned char *)dr, databuf, len, ++ usbctrl_async_callback, NULL); ++ rc = usb_submit_urb(urb, GFP_ATOMIC); ++ if (rc < 0) { ++ kfree(databuf); ++ kfree(dr); ++ } ++ ++ usb_free_urb(urb); ++ ++ return rc; ++} ++ ++static u32 rtw_usb_read_sync(struct rtw_dev *rtwdev, u32 addr, u16 len) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct usb_device *udev = rtwusb->udev; ++ __le32 *data; ++ unsigned long flags; ++ int ret; ++ static int count; ++ ++ spin_lock_irqsave(&rtwusb->usb_lock, flags); ++ ++ if (++rtwusb->usb_data_index >= RTW_USB_MAX_RX_COUNT) ++ rtwusb->usb_data_index = 0; ++ data = &rtwusb->usb_data[rtwusb->usb_data_index]; ++ ++ spin_unlock_irqrestore(&rtwusb->usb_lock, flags); ++ ++ ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), ++ RTW_USB_CMD_REQ, RTW_USB_CMD_READ, addr, ++ RTW_USB_VENQT_CMD_IDX, data, len, 1000); ++ if (ret < 0 && ret != -ENODEV && count++ < 4) ++ rtw_err(rtwdev, "reg 0x%x, usbctrl_vendorreq failed with %d\n", ++ addr, ret); ++ ++ return le32_to_cpu(*data); ++} ++ ++static u8 rtw_usb_read8_sync(struct rtw_dev *rtwdev, u32 addr) ++{ ++ return (u8)rtw_usb_read_sync(rtwdev, addr, 1); ++} ++ ++static u16 rtw_usb_read16_sync(struct rtw_dev *rtwdev, u32 addr) ++{ ++ return (u16)rtw_usb_read_sync(rtwdev, addr, 2); ++} ++ ++static u32 rtw_usb_read32_sync(struct rtw_dev *rtwdev, u32 addr) ++{ ++ return (u32)rtw_usb_read_sync(rtwdev, addr, 4); ++} ++ ++static void rtw_usb_write_async(struct rtw_dev *rtwdev, u32 addr, u32 val, ++ u16 len) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct usb_device *udev = rtwusb->udev; ++ u8 request; ++ u16 wvalue; ++ u16 index; ++ __le32 data; ++ ++ request = RTW_USB_CMD_REQ; ++ index = RTW_USB_VENQT_CMD_IDX; /* n/a */ ++ wvalue = (u16)(addr & 0x0000ffff); ++ data = cpu_to_le32(val); ++ usbctrl_vendorreq_async_write(udev, request, wvalue, index, &data, len); ++} ++ ++static void rtw_usb_write8_async(struct rtw_dev *rtwdev, u32 addr, u8 val) ++{ ++ rtw_usb_write_async(rtwdev, addr, val, 1); ++} ++ ++static void rtw_usb_write16_async(struct rtw_dev *rtwdev, u32 addr, u16 val) ++{ ++ rtw_usb_write_async(rtwdev, addr, val, 2); ++} ++ ++static void rtw_usb_write32_async(struct rtw_dev *rtwdev, u32 addr, u32 val) ++{ ++ rtw_usb_write_async(rtwdev, addr, val, 4); ++} ++ ++static int rtw_usb_parse(struct rtw_dev *rtwdev, ++ struct usb_interface *interface) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct usb_host_interface *host_interface = &interface->altsetting[0]; ++ struct usb_interface_descriptor *interface_desc = &host_interface->desc; ++ struct usb_endpoint_descriptor *endpoint; ++ struct usb_device *usbd = interface_to_usbdev(interface); ++ int num_out_pipes = 0; ++ int i; ++ u8 num; ++ ++ for (i = 0; i < interface_desc->bNumEndpoints; i++) { ++ endpoint = &host_interface->endpoint[i].desc; ++ num = usb_endpoint_num(endpoint); ++ ++ if (usb_endpoint_dir_in(endpoint) && ++ usb_endpoint_xfer_bulk(endpoint)) { ++ if (rtwusb->pipe_in) { ++ rtw_err(rtwdev, "IN pipes overflow\n"); ++ return -EINVAL; ++ } ++ ++ rtwusb->pipe_in = num; ++ } ++ ++ if (usb_endpoint_dir_in(endpoint) && ++ usb_endpoint_xfer_int(endpoint)) { ++ if (rtwusb->pipe_interrupt) { ++ rtw_err(rtwdev, "INT pipes overflow\n"); ++ return -EINVAL; ++ } ++ ++ rtwusb->pipe_interrupt = num; ++ } ++ ++ if (usb_endpoint_dir_out(endpoint) && ++ usb_endpoint_xfer_bulk(endpoint)) { ++ if (num_out_pipes >= ARRAY_SIZE(rtwusb->out_ep)) { ++ rtw_err(rtwdev, "OUT pipes overflow\n"); ++ return -EINVAL; ++ } ++ ++ rtwusb->out_ep[num_out_pipes++] = num; ++ } ++ } ++ ++ switch (usbd->speed) { ++ case USB_SPEED_LOW: ++ case USB_SPEED_FULL: ++ rtwusb->bulkout_size = RTW_USB_FULL_SPEED_BULK_SIZE; ++ break; ++ case USB_SPEED_HIGH: ++ rtwusb->bulkout_size = RTW_USB_HIGH_SPEED_BULK_SIZE; ++ break; ++ case USB_SPEED_SUPER: ++ rtwusb->bulkout_size = RTW_USB_SUPER_SPEED_BULK_SIZE; ++ break; ++ default: ++ rtw_err(rtwdev, "failed to detect usb speed\n"); ++ return -EINVAL; ++ } ++ ++ rtwdev->hci.bulkout_num = num_out_pipes; ++ ++ switch (num_out_pipes) { ++ case 4: ++ case 3: ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 2; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 2; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 2; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 2; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID4] = 1; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID5] = 1; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID6] = 1; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID7] = 1; ++ break; ++ case 2: ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 1; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 1; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 1; ++ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 1; ++ break; ++ case 1: ++ break; ++ default: ++ rtw_err(rtwdev, "failed to get out_pipes(%d)\n", num_out_pipes); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static void rtw_usb_txcb_enqueue(struct rtw_usb_txcb *txcb, struct sk_buff *skb) ++{ ++ skb_queue_tail(&txcb->tx_ack_queue, skb); ++} ++ ++static void rtw_usb_tx_agg_skb(struct rtw_usb *rtwusb, struct sk_buff_head *list, ++ struct sk_buff *skb_head, struct sk_buff *skb, ++ struct rtw_usb_txcb *txcb) ++{ ++ struct sk_buff *skb_iter; ++ unsigned long flags; ++ u8 *data_ptr; ++ int agg_num = 0, len, max_len; ++ ++ data_ptr = skb_head->data; ++ skb_iter = skb; ++ ++ while (skb_iter) { ++ memcpy(data_ptr, skb_iter->data, skb_iter->len); ++ len = ALIGN(skb_iter->len, 8); ++ skb_put(skb_head, len); ++ data_ptr += len; ++ agg_num++; ++ ++ rtw_usb_txcb_enqueue(txcb, skb_iter); ++ ++ spin_lock_irqsave(&list->lock, flags); ++ ++ skb_iter = skb_peek(list); ++ max_len = RTW_USB_MAX_XMITBUF_SZ - skb_head->len; ++ ++ if (skb_iter && skb_iter->len < max_len) ++ __skb_unlink(skb_iter, list); ++ else ++ skb_iter = NULL; ++ spin_unlock_irqrestore(&list->lock, flags); ++ } ++ ++ if (agg_num > 1) ++ rtw_usb_fill_tx_checksum(rtwusb, skb_head, agg_num); ++} ++ ++static void rtw_usb_indicate_tx_status(struct rtw_dev *rtwdev, ++ struct sk_buff *skb) ++{ ++ struct ieee80211_hw *hw = rtwdev->hw; ++ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); ++ struct rtw_usb_tx_data *tx_data = rtw_usb_get_tx_data(skb); ++ ++ /* enqueue to wait for tx report */ ++ if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { ++ rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); ++ return; ++ } ++ ++ /* always ACK for others, then they won't be marked as drop */ ++ ieee80211_tx_info_clear_status(info); ++ if (info->flags & IEEE80211_TX_CTL_NO_ACK) ++ info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; ++ else ++ info->flags |= IEEE80211_TX_STAT_ACK; ++ ++ ieee80211_tx_status_irqsafe(hw, skb); ++} ++ ++static void rtw_usb_write_port_tx_complete(struct urb *urb) ++{ ++ struct rtw_usb_txcb *txcb = urb->context; ++ struct rtw_dev *rtwdev = txcb->rtwdev; ++ ++ while (true) { ++ struct sk_buff *skb = skb_dequeue(&txcb->tx_ack_queue); ++ if (!skb) ++ break; ++ ++ if (GET_TX_DESC_QSEL(skb->data) <= TX_DESC_QSEL_TID7) ++ rtw_usb_indicate_tx_status(rtwdev, skb); ++ else ++ dev_kfree_skb_any(skb); ++ } ++ ++ kfree(txcb); ++} ++ ++static int rtw_usb_write_port(struct rtw_dev *rtwdev, u8 qsel, struct sk_buff *skb, ++ usb_complete_t cb, void *context) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct usb_device *usbd = rtwusb->udev; ++ struct urb *urb; ++ unsigned int pipe; ++ int ret; ++ int ep = rtwusb->qsel_to_ep[qsel]; ++ ++ pipe = usb_sndbulkpipe(usbd, rtwusb->out_ep[ep]); ++ urb = usb_alloc_urb(0, GFP_ATOMIC); ++ if (!urb) ++ return -ENOMEM; ++ ++ usb_fill_bulk_urb(urb, usbd, pipe, skb->data, skb->len, cb, context); ++ ret = usb_submit_urb(urb, GFP_ATOMIC); ++ ++ usb_free_urb(urb); ++ ++ return ret; ++} ++ ++static struct sk_buff *rtw_usb_tx_agg_check(struct rtw_usb *rtwusb, ++ struct sk_buff *skb, ++ int index, ++ struct rtw_usb_txcb *txcb) ++{ ++ struct sk_buff_head *list; ++ struct sk_buff *skb_head; ++ ++ list = &rtwusb->tx_queue[index]; ++ if (skb_queue_empty(list)) ++ return NULL; ++ ++ skb_head = dev_alloc_skb(RTW_USB_MAX_XMITBUF_SZ); ++ if (!skb_head) ++ return NULL; ++ ++ rtw_usb_tx_agg_skb(rtwusb, list, skb_head, skb, txcb); ++ ++ return skb_head; ++} ++ ++static void rtw_usb_tx_agg(struct rtw_usb *rtwusb, struct sk_buff *skb, int index) ++{ ++ struct rtw_dev *rtwdev = rtwusb->rtwdev; ++ struct sk_buff *skb_head; ++ struct rtw_usb_txcb *txcb; ++ u8 qsel; ++ ++ txcb = kmalloc(sizeof(*txcb), GFP_ATOMIC); ++ if (!txcb) ++ return; ++ ++ txcb->rtwdev = rtwdev; ++ skb_queue_head_init(&txcb->tx_ack_queue); ++ ++ skb_head = rtw_usb_tx_agg_check(rtwusb, skb, index, txcb); ++ if (!skb_head) { ++ skb_head = skb; ++ rtw_usb_txcb_enqueue(txcb, skb); ++ } ++ ++ qsel = GET_TX_DESC_QSEL(skb->data); ++ ++ rtw_usb_write_port(rtwdev, qsel, skb_head, ++ rtw_usb_write_port_tx_complete, txcb); ++ ++ if (skb_head != skb) ++ dev_kfree_skb(skb_head); ++} ++ ++static void rtw_usb_tx_handler(struct work_struct *work) ++{ ++ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, tx_work); ++ struct sk_buff *skb; ++ int index, limit; ++ ++ for (index = ARRAY_SIZE(rtwusb->tx_queue) - 1; index >= 0; index--) { ++ for (limit = 0; limit < 200; limit++) { ++ skb = skb_dequeue(&rtwusb->tx_queue[index]); ++ if (skb) ++ rtw_usb_tx_agg(rtwusb, skb, index); ++ else ++ break; ++ } ++ } ++} ++ ++static void rtw_usb_tx_queue_purge(struct rtw_usb *rtwusb) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) ++ skb_queue_purge(&rtwusb->tx_queue[i]); ++} ++ ++static void rtw_usb_write_port_complete(struct urb *urb) ++{ ++ struct sk_buff *skb = urb->context; ++ ++ dev_kfree_skb_any(skb); ++} ++ ++static int rtw_usb_write_data(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ u8 *buf) ++{ ++ struct rtw_chip_info *chip = rtwdev->chip; ++ struct sk_buff *skb; ++ unsigned int desclen, headsize, size; ++ u8 qsel; ++ int ret = 0; ++ ++ size = pkt_info->tx_pkt_size; ++ qsel = pkt_info->qsel; ++ desclen = chip->tx_pkt_desc_sz; ++ headsize = pkt_info->offset ? pkt_info->offset : desclen; ++ ++ skb = dev_alloc_skb(headsize + size); ++ if (unlikely(!skb)) ++ return -ENOMEM; ++ ++ skb_reserve(skb, headsize); ++ skb_put_data(skb, buf, size); ++ skb_push(skb, headsize); ++ memset(skb->data, 0, headsize); ++ rtw_tx_fill_tx_desc(pkt_info, skb); ++ rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, skb->data); ++ ++ ret = rtw_usb_write_port(rtwdev, qsel, skb, ++ rtw_usb_write_port_complete, skb); ++ if (unlikely(ret)) ++ rtw_err(rtwdev, "failed to do USB write, ret=%d\n", ret); ++ ++ return ret; ++} ++ ++static int rtw_usb_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, ++ u32 size) ++{ ++ struct rtw_chip_info *chip = rtwdev->chip; ++ struct rtw_usb *rtwusb; ++ struct rtw_tx_pkt_info pkt_info = {0}; ++ u32 len, desclen; ++ ++ if (unlikely(!rtwdev)) ++ return -EINVAL; ++ ++ rtwusb = rtw_get_usb_priv(rtwdev); ++ if (unlikely(!rtwusb)) ++ return -EINVAL; ++ ++ pkt_info.tx_pkt_size = size; ++ pkt_info.qsel = TX_DESC_QSEL_BEACON; ++ ++ desclen = chip->tx_pkt_desc_sz; ++ len = desclen + size; ++ if (len % rtwusb->bulkout_size == 0) { ++ len += RTW_USB_PACKET_OFFSET_SZ; ++ pkt_info.offset = desclen + RTW_USB_PACKET_OFFSET_SZ; ++ pkt_info.pkt_offset = 1; ++ } else { ++ pkt_info.offset = desclen; ++ } ++ ++ return rtw_usb_write_data(rtwdev, &pkt_info, buf); ++} ++ ++static int rtw_usb_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) ++{ ++ struct rtw_tx_pkt_info pkt_info = {0}; ++ ++ pkt_info.tx_pkt_size = size; ++ pkt_info.qsel = TX_DESC_QSEL_H2C; ++ ++ return rtw_usb_write_data(rtwdev, &pkt_info, buf); ++} ++ ++static u8 rtw_usb_tx_queue_mapping_to_qsel(struct sk_buff *skb) ++{ ++ struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; ++ __le16 fc = hdr->frame_control; ++ u8 qsel; ++ ++ if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc))) ++ qsel = TX_DESC_QSEL_MGMT; ++ else if (skb_get_queue_mapping(skb) <= IEEE80211_AC_BK) ++ qsel = skb->priority; ++ else ++ qsel = TX_DESC_QSEL_BEACON; ++ ++ return qsel; ++} ++ ++static int rtw_usb_tx_write(struct rtw_dev *rtwdev, ++ struct rtw_tx_pkt_info *pkt_info, ++ struct sk_buff *skb) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct rtw_chip_info *chip = rtwdev->chip; ++ struct rtw_usb_tx_data *tx_data; ++ u8 *pkt_desc; ++ int ep; ++ ++ pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); ++ memset(pkt_desc, 0, chip->tx_pkt_desc_sz); ++ pkt_info->qsel = rtw_usb_tx_queue_mapping_to_qsel(skb); ++ ep = rtwusb->qsel_to_ep[pkt_info->qsel]; ++ rtw_tx_fill_tx_desc(pkt_info, skb); ++ rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, skb->data); ++ tx_data = rtw_usb_get_tx_data(skb); ++ tx_data->sn = pkt_info->sn; ++ ++ skb_queue_tail(&rtwusb->tx_queue[ep], skb); ++ ++ return 0; ++} ++ ++static void rtw_usb_tx_kick_off(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ queue_work(rtwusb->txwq, &rtwusb->tx_work); ++} ++ ++static void rtw_usb_rx_handler(struct work_struct *work) ++{ ++ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, rx_work); ++ struct rtw_dev *rtwdev = rtwusb->rtwdev; ++ struct rtw_chip_info *chip = rtwdev->chip; ++ struct rtw_rx_pkt_stat pkt_stat; ++ struct ieee80211_rx_status rx_status; ++ struct sk_buff *skb; ++ u32 pkt_desc_sz = chip->rx_pkt_desc_sz; ++ u32 pkt_offset; ++ u8 *rx_desc; ++ int limit; ++ ++ for (limit = 0; limit < 200; limit++) { ++ skb = skb_dequeue(&rtwusb->rx_queue); ++ if (!skb) ++ break; ++ ++ rx_desc = skb->data; ++ chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, ++ &rx_status); ++ pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + ++ pkt_stat.shift; ++ ++ if (pkt_stat.is_c2h) { ++ skb_put(skb, pkt_stat.pkt_len + pkt_offset); ++ rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, ++ skb); ++ continue; ++ } ++ ++ if (skb_queue_len(&rtwusb->rx_queue) >= RTW_USB_MAX_RXQ_LEN) { ++ rtw_err(rtwdev, "failed to get rx_queue, overflow\n"); ++ dev_kfree_skb_any(skb); ++ continue; ++ } ++ ++ skb_put(skb, pkt_stat.pkt_len); ++ skb_reserve(skb, pkt_offset); ++ memcpy(skb->cb, &rx_status, sizeof(rx_status)); ++ ieee80211_rx_irqsafe(rtwdev->hw, skb); ++ } ++} ++ ++static void rtw_usb_rx_data_put(struct rtw_usb *rtwusb, ++ struct rx_usb_ctrl_block *rxcb) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ list_move(&rxcb->list, &rtwusb->rx_data_free); ++ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); ++} ++ ++static void rtw_usb_read_port_complete(struct urb *urb) ++{ ++ struct rx_usb_ctrl_block *rxcb = urb->context; ++ struct rtw_dev *rtwdev = rxcb->rtwdev; ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct sk_buff *skb = rxcb->rx_skb; ++ ++ if (urb->status == 0) { ++ if (urb->actual_length >= RTW_USB_MAX_RECVBUF_SZ || ++ urb->actual_length < 24) { ++ rtw_err(rtwdev, "failed to get urb length:%d\n", ++ urb->actual_length); ++ if (skb) ++ dev_kfree_skb_any(skb); ++ } else { ++ skb_queue_tail(&rtwusb->rx_queue, skb); ++ queue_work(rtwusb->rxwq, &rtwusb->rx_work); ++ } ++ ++ rtw_usb_rx_data_put(rtwusb, rxcb); ++ queue_work(rtwusb->rxwq, &rtwusb->rx_refill_work); ++ } else { ++ switch (urb->status) { ++ case -EINVAL: ++ case -EPIPE: ++ case -ENODEV: ++ case -ESHUTDOWN: ++ case -ENOENT: ++ case -EPROTO: ++ case -EILSEQ: ++ case -ETIME: ++ case -ECOMM: ++ case -EOVERFLOW: ++ case -EINPROGRESS: ++ break; ++ default: ++ rtw_err(rtwdev, "status unknown=%d\n", urb->status); ++ break; ++ } ++ if (skb) ++ dev_kfree_skb_any(skb); ++ } ++} ++ ++static void rtw_usb_rx_refill_work(struct work_struct *work) ++{ ++ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, rx_refill_work); ++ struct rtw_dev *rtwdev = rtwusb->rtwdev; ++ struct rx_usb_ctrl_block *rxcb; ++ unsigned long flags; ++ int error; ++ ++ do { ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ ++ rxcb = list_first_entry_or_null(&rtwusb->rx_data_free, ++ struct rx_usb_ctrl_block, list); ++ ++ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); ++ if (!rxcb) ++ return; ++ ++ rxcb->rx_skb = alloc_skb(RTW_USB_MAX_RECVBUF_SZ, GFP_KERNEL); ++ if (!rxcb->rx_skb) { ++ rtw_err(rtwdev, "could not allocate rx skbuff\n"); ++ return; ++ } ++ ++ usb_fill_bulk_urb(rxcb->rx_urb, rtwusb->udev, ++ usb_rcvbulkpipe(rtwusb->udev, rtwusb->pipe_in), ++ rxcb->rx_skb->data, RTW_USB_MAX_RECVBUF_SZ, ++ rtw_usb_read_port_complete, rxcb); ++ ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ list_move(&rxcb->list, &rtwusb->rx_data_used); ++ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); ++ ++ error = usb_submit_urb(rxcb->rx_urb, GFP_KERNEL); ++ if (error) { ++ kfree_skb(rxcb->rx_skb); ++ if (error != -ENODEV) ++ rtw_err(rtwdev, "Err sending rx data urb %d\n", ++ error); ++ rtw_usb_rx_data_put(rtwusb, rxcb); ++ ++ return; ++ } ++ } while (true); ++} ++ ++static void rtw_usb_cancel_rx_bufs(struct rtw_usb *rtwusb) ++{ ++ struct rx_usb_ctrl_block *rxcb; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ ++ while (true) { ++ rxcb = list_first_entry_or_null(&rtwusb->rx_data_used, ++ struct rx_usb_ctrl_block, list); ++ ++ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); ++ ++ if (!rxcb) ++ break; ++ ++ usb_kill_urb(rxcb->rx_urb); ++ ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ list_move(&rxcb->list, &rtwusb->rx_data_free); ++ } ++} ++ ++static void rtw_usb_free_rx_bufs(struct rtw_usb *rtwusb) ++{ ++ struct rx_usb_ctrl_block *rxcb; ++ unsigned long flags; ++ ++ rtw_usb_cancel_rx_bufs(rtwusb); ++ ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ ++ while (true) { ++ rxcb = list_first_entry_or_null(&rtwusb->rx_data_free, struct rx_usb_ctrl_block, list); ++ ++ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); ++ ++ if (!rxcb) ++ break; ++ ++ usb_free_urb(rxcb->rx_urb); ++ ++ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); ++ list_del(&rxcb->list); ++ } ++} ++ ++static int rtw_usb_alloc_rx_bufs(struct rtw_usb *rtwusb) ++{ ++ int i; ++ ++ for (i = 0; i < RTW_USB_RXCB_NUM; i++) { ++ struct rx_usb_ctrl_block *rxcb = &rtwusb->rx_cb[i]; ++ ++ rxcb->rtwdev = rtwusb->rtwdev; ++ rxcb->rx_urb = usb_alloc_urb(0, GFP_KERNEL); ++ if (!rxcb->rx_urb) ++ goto err; ++ list_add_tail(&rxcb->list, &rtwusb->rx_data_free); ++ } ++ ++ return 0; ++err: ++ rtw_usb_free_rx_bufs(rtwusb); ++ return -ENOMEM; ++} ++ ++static int rtw_usb_setup(struct rtw_dev *rtwdev) ++{ ++ /* empty function for rtw_hci_ops */ ++ return 0; ++} ++ ++static int rtw_usb_start(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ queue_work(rtwusb->rxwq, &rtwusb->rx_refill_work); ++ ++ return 0; ++} ++ ++static void rtw_usb_stop(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ cancel_work_sync(&rtwusb->rx_refill_work); ++ rtw_usb_cancel_rx_bufs(rtwusb); ++} ++ ++static void rtw_usb_deep_ps(struct rtw_dev *rtwdev, bool enter) ++{ ++ /* empty function for rtw_hci_ops */ ++} ++ ++static void rtw_usb_link_ps(struct rtw_dev *rtwdev, bool enter) ++{ ++ /* empty function for rtw_hci_ops */ ++} ++ ++static void rtw_usb_interface_cfg(struct rtw_dev *rtwdev) ++{ ++ /* empty function for rtw_hci_ops */ ++} ++ ++static struct rtw_hci_ops rtw_usb_ops = { ++ .tx_write = rtw_usb_tx_write, ++ .tx_kick_off = rtw_usb_tx_kick_off, ++ .setup = rtw_usb_setup, ++ .start = rtw_usb_start, ++ .stop = rtw_usb_stop, ++ .deep_ps = rtw_usb_deep_ps, ++ .link_ps = rtw_usb_link_ps, ++ .interface_cfg = rtw_usb_interface_cfg, ++ ++ .write8 = rtw_usb_write8_async, ++ .write16 = rtw_usb_write16_async, ++ .write32 = rtw_usb_write32_async, ++ .read8 = rtw_usb_read8_sync, ++ .read16 = rtw_usb_read16_sync, ++ .read32 = rtw_usb_read32_sync, ++ ++ .write_data_rsvd_page = rtw_usb_write_data_rsvd_page, ++ .write_data_h2c = rtw_usb_write_data_h2c, ++}; ++ ++static int rtw_usb_init_rx(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ rtwusb->rxwq = create_singlethread_workqueue("rtw88_usb: rx wq"); ++ if (!rtwusb->rxwq) { ++ rtw_err(rtwdev, "failed to create RX work queue\n"); ++ return -ENOMEM; ++ } ++ ++ skb_queue_head_init(&rtwusb->rx_queue); ++ ++ INIT_WORK(&rtwusb->rx_work, rtw_usb_rx_handler); ++ ++ return 0; ++} ++ ++static void rtw_usb_deinit_rx(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ skb_queue_purge(&rtwusb->rx_queue); ++ ++ flush_workqueue(rtwusb->rxwq); ++ destroy_workqueue(rtwusb->rxwq); ++} ++ ++static int rtw_usb_init_tx(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ int i; ++ ++ rtwusb->txwq = create_singlethread_workqueue("rtw88_usb: tx wq"); ++ if (!rtwusb->txwq) { ++ rtw_err(rtwdev, "failed to create TX work queue\n"); ++ return -ENOMEM; ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) ++ skb_queue_head_init(&rtwusb->tx_queue[i]); ++ ++ INIT_WORK(&rtwusb->tx_work, rtw_usb_tx_handler); ++ ++ return 0; ++} ++ ++static void rtw_usb_deinit_tx(struct rtw_dev *rtwdev) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ rtw_usb_tx_queue_purge(rtwusb); ++ flush_workqueue(rtwusb->txwq); ++ destroy_workqueue(rtwusb->txwq); ++} ++ ++static int rtw_usb_intf_init(struct rtw_dev *rtwdev, ++ struct usb_interface *intf) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ struct usb_device *udev = usb_get_dev(interface_to_usbdev(intf)); ++ int ret; ++ ++ rtwusb->udev = udev; ++ ret = rtw_usb_parse(rtwdev, intf); ++ if (ret) ++ return ret; ++ ++ rtwusb->usb_data = kcalloc(RTW_USB_MAX_RX_COUNT, sizeof(u32), ++ GFP_KERNEL); ++ if (!rtwusb->usb_data) ++ return -ENOMEM; ++ ++ usb_set_intfdata(intf, rtwdev->hw); ++ ++ SET_IEEE80211_DEV(rtwdev->hw, &intf->dev); ++ spin_lock_init(&rtwusb->usb_lock); ++ ++ return 0; ++} ++ ++static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev, ++ struct usb_interface *intf) ++{ ++ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ usb_put_dev(rtwusb->udev); ++ usb_set_intfdata(intf, NULL); ++} ++ ++int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) ++{ ++ struct rtw_dev *rtwdev; ++ struct ieee80211_hw *hw; ++ struct rtw_usb *rtwusb; ++ int drv_data_size; ++ int ret; ++ ++ drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_usb); ++ hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); ++ if (!hw) ++ return -ENOMEM; ++ ++ rtwdev = hw->priv; ++ rtwdev->hw = hw; ++ rtwdev->dev = &intf->dev; ++ rtwdev->chip = (struct rtw_chip_info *)id->driver_info; ++ rtwdev->hci.ops = &rtw_usb_ops; ++ rtwdev->hci.type = RTW_HCI_TYPE_USB; ++ ++ rtwusb = rtw_get_usb_priv(rtwdev); ++ rtwusb->rtwdev = rtwdev; ++ ++ INIT_WORK(&rtwusb->rx_refill_work, rtw_usb_rx_refill_work); ++ INIT_LIST_HEAD(&rtwusb->rx_data_free); ++ INIT_LIST_HEAD(&rtwusb->rx_data_used); ++ spin_lock_init(&rtwusb->rx_data_list_lock); ++ ++ ret = rtw_usb_alloc_rx_bufs(rtwusb); ++ if (ret) ++ return ret; ++ ++ ret = rtw_core_init(rtwdev); ++ if (ret) ++ goto err_release_hw; ++ ++ ret = rtw_usb_intf_init(rtwdev, intf); ++ if (ret) { ++ rtw_err(rtwdev, "failed to init USB interface\n"); ++ goto err_deinit_core; ++ } ++ ++ ret = rtw_usb_init_tx(rtwdev); ++ if (ret) { ++ rtw_err(rtwdev, "failed to init USB TX\n"); ++ goto err_destroy_usb; ++ } ++ ++ ret = rtw_usb_init_rx(rtwdev); ++ if (ret) { ++ rtw_err(rtwdev, "failed to init USB RX\n"); ++ goto err_destroy_txwq; ++ } ++ ++ ret = rtw_chip_info_setup(rtwdev); ++ if (ret) { ++ rtw_err(rtwdev, "failed to setup chip information\n"); ++ goto err_destroy_rxwq; ++ } ++ ++ ret = rtw_register_hw(rtwdev, rtwdev->hw); ++ if (ret) { ++ rtw_err(rtwdev, "failed to register hw\n"); ++ goto err_destroy_rxwq; ++ } ++ ++ return 0; ++ ++err_destroy_rxwq: ++ rtw_usb_deinit_rx(rtwdev); ++ ++err_destroy_txwq: ++ rtw_usb_deinit_tx(rtwdev); ++ ++err_destroy_usb: ++ rtw_usb_intf_deinit(rtwdev, intf); ++ ++err_deinit_core: ++ rtw_core_deinit(rtwdev); ++ ++err_release_hw: ++ ieee80211_free_hw(hw); ++ ++ return ret; ++} ++EXPORT_SYMBOL(rtw_usb_probe); ++ ++void rtw_usb_disconnect(struct usb_interface *intf) ++{ ++ struct ieee80211_hw *hw = usb_get_intfdata(intf); ++ struct rtw_dev *rtwdev; ++ struct rtw_usb *rtwusb; ++ ++ if (!hw) ++ return; ++ ++ rtwdev = hw->priv; ++ rtwusb = rtw_get_usb_priv(rtwdev); ++ ++ rtw_unregister_hw(rtwdev, hw); ++ rtw_usb_deinit_tx(rtwdev); ++ rtw_usb_deinit_rx(rtwdev); ++ ++ if (rtwusb->udev->state != USB_STATE_NOTATTACHED) ++ usb_reset_device(rtwusb->udev); ++ ++ rtw_usb_free_rx_bufs(rtwusb); ++ ++ rtw_usb_intf_deinit(rtwdev, intf); ++ rtw_core_deinit(rtwdev); ++ ieee80211_free_hw(hw); ++} ++EXPORT_SYMBOL(rtw_usb_disconnect); ++ ++MODULE_AUTHOR("Realtek Corporation"); ++MODULE_DESCRIPTION("Realtek 802.11ac wireless USB driver"); ++MODULE_LICENSE("Dual BSD/GPL"); +diff --git a/drivers/net/wireless/realtek/rtw88/usb.h b/drivers/net/wireless/realtek/rtw88/usb.h +new file mode 100644 +index 000000000000..4d714372f265 +--- /dev/null ++++ b/drivers/net/wireless/realtek/rtw88/usb.h +@@ -0,0 +1,109 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ ++/* Copyright(c) 2018-2019 Realtek Corporation ++ */ ++ ++#ifndef __RTW_USB_H_ ++#define __RTW_USB_H_ ++ ++#define FW_8192C_START_ADDRESS 0x1000 ++#define FW_8192C_END_ADDRESS 0x5FFF ++ ++#define RTW_USB_MAX_RX_COUNT 100 ++#define RTW_USB_VENQT_MAX_BUF_SIZE 254 ++#define MAX_USBCTRL_VENDORREQ_TIMES 10 ++ ++#define RTW_USB_CMD_READ 0xc0 ++#define RTW_USB_CMD_WRITE 0x40 ++#define RTW_USB_CMD_REQ 0x05 ++ ++#define RTW_USB_VENQT_CMD_IDX 0x00 ++ ++#define RTW_USB_SUPER_SPEED_BULK_SIZE 1024 ++#define RTW_USB_HIGH_SPEED_BULK_SIZE 512 ++#define RTW_USB_FULL_SPEED_BULK_SIZE 64 ++ ++#define RTW_USB_TX_SEL_HQ BIT(0) ++#define RTW_USB_TX_SEL_LQ BIT(1) ++#define RTW_USB_TX_SEL_NQ BIT(2) ++#define RTW_USB_TX_SEL_EQ BIT(3) ++ ++#define RTW_USB_BULK_IN_ADDR 0x80 ++#define RTW_USB_INT_IN_ADDR 0x81 ++ ++#define RTW_USB_HW_QUEUE_ENTRY 8 ++ ++#define RTW_USB_PACKET_OFFSET_SZ 8 ++#define RTW_USB_MAX_XMITBUF_SZ (1592 * 3) ++#define RTW_USB_MAX_RECVBUF_SZ 32768 ++ ++#define RTW_USB_RECVBUFF_ALIGN_SZ 8 ++ ++#define RTW_USB_RXAGG_SIZE 6 ++#define RTW_USB_RXAGG_TIMEOUT 10 ++ ++#define RTW_USB_RXCB_NUM 4 ++ ++#define RTW_USB_EP_MAX 4 ++ ++#define TX_DESC_QSEL_MAX 20 ++ ++static inline struct rtw_usb *rtw_get_usb_priv(struct rtw_dev *rtwdev) ++{ ++ return (struct rtw_usb *)rtwdev->priv; ++} ++ ++struct rx_usb_ctrl_block { ++ struct rtw_dev *rtwdev; ++ struct urb *rx_urb; ++ struct sk_buff *rx_skb; ++ struct list_head list; ++}; ++ ++struct rtw_usb_tx_data { ++ u8 sn; ++}; ++ ++struct rtw_usb { ++ struct rtw_dev *rtwdev; ++ struct usb_device *udev; ++ ++ spinlock_t rx_data_list_lock; ++ struct work_struct rx_refill_work; ++ struct list_head rx_data_free; ++ struct list_head rx_data_used; ++ ++ spinlock_t usb_lock; ++ __le32 *usb_data; ++ int usb_data_index; ++ ++ u32 bulkout_size; ++ u8 pipe_interrupt; ++ u8 pipe_in; ++ u8 out_ep[RTW_USB_EP_MAX]; ++ u8 qsel_to_ep[TX_DESC_QSEL_MAX]; ++ u8 usb_txagg_num; ++ ++ struct workqueue_struct *txwq, *rxwq; ++ ++ struct sk_buff_head tx_queue[RTW_USB_EP_MAX]; ++ struct work_struct tx_work; ++ ++ struct rx_usb_ctrl_block rx_cb[RTW_USB_RXCB_NUM]; ++ struct sk_buff_head rx_queue; ++ struct work_struct rx_work; ++}; ++ ++static inline struct rtw_usb_tx_data *rtw_usb_get_tx_data(struct sk_buff *skb) ++{ ++ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); ++ ++ BUILD_BUG_ON(sizeof(struct rtw_usb_tx_data) > ++ sizeof(info->status.status_driver_data)); ++ ++ return (struct rtw_usb_tx_data *)info->status.status_driver_data; ++} ++ ++int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id); ++void rtw_usb_disconnect(struct usb_interface *intf); ++ ++#endif +diff --git a/drivers/net/wireless/realtek/rtw88/util.c b/drivers/net/wireless/realtek/rtw88/util.c +index 2c515af214e7..db55dbd5c533 100644 +--- a/drivers/net/wireless/realtek/rtw88/util.c ++++ b/drivers/net/wireless/realtek/rtw88/util.c +@@ -105,3 +105,95 @@ void rtw_desc_to_mcsrate(u16 rate, u8 *mcs, u8 *nss) + *mcs = rate - DESC_RATEMCS0; + } + } ++ ++struct rtw_stas_entry { ++ struct list_head list; ++ struct ieee80211_sta *sta; ++}; ++ ++struct rtw_iter_stas_data { ++ struct rtw_dev *rtwdev; ++ struct list_head list; ++}; ++ ++void rtw_collect_sta_iter(void *data, struct ieee80211_sta *sta) ++{ ++ struct rtw_iter_stas_data *iter_stas = data; ++ struct rtw_stas_entry *stas_entry; ++ ++ stas_entry = kmalloc(sizeof(*stas_entry), GFP_ATOMIC); ++ if (!stas_entry) ++ return; ++ ++ stas_entry->sta = sta; ++ list_add_tail(&stas_entry->list, &iter_stas->list); ++} ++ ++void rtw_iterate_stas(struct rtw_dev *rtwdev, ++ void (*iterator)(void *data, ++ struct ieee80211_sta *sta), ++ void *data) ++{ ++ struct rtw_iter_stas_data iter_data; ++ struct rtw_stas_entry *sta_entry, *tmp; ++ ++ iter_data.rtwdev = rtwdev; ++ INIT_LIST_HEAD(&iter_data.list); ++ ++ ieee80211_iterate_stations_atomic(rtwdev->hw, rtw_collect_sta_iter, ++ &iter_data); ++ ++ list_for_each_entry_safe(sta_entry, tmp, &iter_data.list, ++ list) { ++ list_del_init(&sta_entry->list); ++ iterator(data, sta_entry->sta); ++ kfree(sta_entry); ++ } ++} ++ ++struct rtw_vifs_entry { ++ struct list_head list; ++ struct ieee80211_vif *vif; ++ u8 mac[ETH_ALEN]; ++}; ++ ++struct rtw_iter_vifs_data { ++ struct rtw_dev *rtwdev; ++ struct list_head list; ++}; ++ ++void rtw_collect_vif_iter(void *data, u8 *mac, struct ieee80211_vif *vif) ++{ ++ struct rtw_iter_vifs_data *iter_stas = data; ++ struct rtw_vifs_entry *vifs_entry; ++ ++ vifs_entry = kmalloc(sizeof(*vifs_entry), GFP_ATOMIC); ++ if (!vifs_entry) ++ return; ++ ++ vifs_entry->vif = vif; ++ ether_addr_copy(vifs_entry->mac, mac); ++ list_add_tail(&vifs_entry->list, &iter_stas->list); ++} ++ ++void rtw_iterate_vifs(struct rtw_dev *rtwdev, ++ void (*iterator)(void *data, u8 *mac, ++ struct ieee80211_vif *vif), ++ void *data) ++{ ++ struct rtw_iter_vifs_data iter_data; ++ struct rtw_vifs_entry *vif_entry, *tmp; ++ ++ iter_data.rtwdev = rtwdev; ++ INIT_LIST_HEAD(&iter_data.list); ++ ++ ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, ++ IEEE80211_IFACE_ITER_NORMAL, rtw_collect_vif_iter, &iter_data); ++ ++ list_for_each_entry_safe(vif_entry, tmp, &iter_data.list, ++ list) { ++ list_del_init(&vif_entry->list); ++ iterator(data, vif_entry->mac, vif_entry->vif); ++ kfree(vif_entry); ++ } ++} +diff --git a/drivers/net/wireless/realtek/rtw88/util.h b/drivers/net/wireless/realtek/rtw88/util.h +index 0c23b5069be0..dc8965525400 100644 +--- a/drivers/net/wireless/realtek/rtw88/util.h ++++ b/drivers/net/wireless/realtek/rtw88/util.h +@@ -7,9 +7,6 @@ + + struct rtw_dev; + +-#define rtw_iterate_vifs(rtwdev, iterator, data) \ +- ieee80211_iterate_active_interfaces(rtwdev->hw, \ +- IEEE80211_IFACE_ITER_NORMAL, iterator, data) + #define rtw_iterate_vifs_atomic(rtwdev, iterator, data) \ + ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, \ + IEEE80211_IFACE_ITER_NORMAL, iterator, data) +@@ -20,6 +17,15 @@ struct rtw_dev; + #define rtw_iterate_keys_rcu(rtwdev, vif, iterator, data) \ + ieee80211_iter_keys_rcu((rtwdev)->hw, vif, iterator, data) + ++void rtw_iterate_vifs(struct rtw_dev *rtwdev, ++ void (*iterator)(void *data, u8 *mac, ++ struct ieee80211_vif *vif), ++ void *data); ++void rtw_iterate_stas(struct rtw_dev *rtwdev, ++ void (*iterator)(void *data, ++ struct ieee80211_sta *sta), ++ void *data); ++ + static inline u8 *get_hdr_bssid(struct ieee80211_hdr *hdr) + { + __le16 fc = hdr->frame_control; +-- +2.38.0 + +From f91d174a7cf3721b9c0409552903a5258220d61e Mon Sep 17 00:00:00 2001 +From: Piotr Gorski +Date: Tue, 6 Sep 2022 20:04:11 +0200 +Subject: [PATCH 09/17] lrng + +Signed-off-by: Piotr Gorski +--- + MAINTAINERS | 7 + + crypto/drbg.c | 16 +- + crypto/jitterentropy-kcapi.c | 3 +- + crypto/jitterentropy.c | 2 +- + drivers/char/Kconfig | 2 + + drivers/char/Makefile | 5 +- + drivers/char/lrng/Kconfig | 907 ++++++++++++++++++ + drivers/char/lrng/Makefile | 39 + + drivers/char/lrng/lrng_definitions.h | 151 +++ + drivers/char/lrng/lrng_drng_atomic.c | 171 ++++ + drivers/char/lrng/lrng_drng_atomic.h | 23 + + drivers/char/lrng/lrng_drng_chacha20.c | 195 ++++ + drivers/char/lrng/lrng_drng_chacha20.h | 42 + + drivers/char/lrng/lrng_drng_drbg.c | 179 ++++ + drivers/char/lrng/lrng_drng_drbg.h | 13 + + drivers/char/lrng/lrng_drng_kcapi.c | 208 ++++ + drivers/char/lrng/lrng_drng_kcapi.h | 13 + + drivers/char/lrng/lrng_drng_mgr.c | 657 +++++++++++++ + drivers/char/lrng/lrng_drng_mgr.h | 86 ++ + drivers/char/lrng/lrng_es_aux.c | 335 +++++++ + drivers/char/lrng/lrng_es_aux.h | 44 + + drivers/char/lrng/lrng_es_cpu.c | 280 ++++++ + drivers/char/lrng/lrng_es_cpu.h | 17 + + drivers/char/lrng/lrng_es_irq.c | 727 ++++++++++++++ + drivers/char/lrng/lrng_es_irq.h | 24 + + drivers/char/lrng/lrng_es_jent.c | 136 +++ + drivers/char/lrng/lrng_es_jent.h | 17 + + drivers/char/lrng/lrng_es_krng.c | 100 ++ + drivers/char/lrng/lrng_es_krng.h | 17 + + drivers/char/lrng/lrng_es_mgr.c | 460 +++++++++ + drivers/char/lrng/lrng_es_mgr.h | 51 + + drivers/char/lrng/lrng_es_mgr_cb.h | 87 ++ + drivers/char/lrng/lrng_es_sched.c | 562 +++++++++++ + drivers/char/lrng/lrng_es_sched.h | 20 + + drivers/char/lrng/lrng_es_timer_common.c | 144 +++ + drivers/char/lrng/lrng_es_timer_common.h | 83 ++ + drivers/char/lrng/lrng_hash_kcapi.c | 140 +++ + drivers/char/lrng/lrng_health.c | 420 ++++++++ + drivers/char/lrng/lrng_health.h | 42 + + drivers/char/lrng/lrng_interface_aux.c | 126 +++ + drivers/char/lrng/lrng_interface_dev.c | 35 + + drivers/char/lrng/lrng_interface_dev_common.c | 314 ++++++ + drivers/char/lrng/lrng_interface_dev_common.h | 51 + + drivers/char/lrng/lrng_interface_hwrand.c | 68 ++ + drivers/char/lrng/lrng_interface_kcapi.c | 129 +++ + .../char/lrng/lrng_interface_random_kernel.c | 214 +++++ + .../char/lrng/lrng_interface_random_kernel.h | 15 + + .../char/lrng/lrng_interface_random_user.c | 104 ++ + drivers/char/lrng/lrng_numa.c | 124 +++ + drivers/char/lrng/lrng_numa.h | 15 + + drivers/char/lrng/lrng_proc.c | 74 ++ + drivers/char/lrng/lrng_proc.h | 15 + + drivers/char/lrng/lrng_selftest.c | 397 ++++++++ + drivers/char/lrng/lrng_sha.h | 14 + + drivers/char/lrng/lrng_sha1.c | 88 ++ + drivers/char/lrng/lrng_sha256.c | 72 ++ + drivers/char/lrng/lrng_switch.c | 286 ++++++ + drivers/char/lrng/lrng_sysctl.c | 140 +++ + drivers/char/lrng/lrng_sysctl.h | 15 + + drivers/char/lrng/lrng_testing.c | 901 +++++++++++++++++ + drivers/char/lrng/lrng_testing.h | 85 ++ + include/crypto/drbg.h | 7 + + .../crypto/internal}/jitterentropy.h | 0 + include/linux/lrng.h | 251 +++++ + kernel/sched/core.c | 3 + + 65 files changed, 9958 insertions(+), 10 deletions(-) + create mode 100644 drivers/char/lrng/Kconfig + create mode 100644 drivers/char/lrng/Makefile + create mode 100644 drivers/char/lrng/lrng_definitions.h + create mode 100644 drivers/char/lrng/lrng_drng_atomic.c + create mode 100644 drivers/char/lrng/lrng_drng_atomic.h + create mode 100644 drivers/char/lrng/lrng_drng_chacha20.c + create mode 100644 drivers/char/lrng/lrng_drng_chacha20.h + create mode 100644 drivers/char/lrng/lrng_drng_drbg.c + create mode 100644 drivers/char/lrng/lrng_drng_drbg.h + create mode 100644 drivers/char/lrng/lrng_drng_kcapi.c + create mode 100644 drivers/char/lrng/lrng_drng_kcapi.h + create mode 100644 drivers/char/lrng/lrng_drng_mgr.c + create mode 100644 drivers/char/lrng/lrng_drng_mgr.h + create mode 100644 drivers/char/lrng/lrng_es_aux.c + create mode 100644 drivers/char/lrng/lrng_es_aux.h + create mode 100644 drivers/char/lrng/lrng_es_cpu.c + create mode 100644 drivers/char/lrng/lrng_es_cpu.h + create mode 100644 drivers/char/lrng/lrng_es_irq.c + create mode 100644 drivers/char/lrng/lrng_es_irq.h + create mode 100644 drivers/char/lrng/lrng_es_jent.c + create mode 100644 drivers/char/lrng/lrng_es_jent.h + create mode 100644 drivers/char/lrng/lrng_es_krng.c + create mode 100644 drivers/char/lrng/lrng_es_krng.h + create mode 100644 drivers/char/lrng/lrng_es_mgr.c + create mode 100644 drivers/char/lrng/lrng_es_mgr.h + create mode 100644 drivers/char/lrng/lrng_es_mgr_cb.h + create mode 100644 drivers/char/lrng/lrng_es_sched.c + create mode 100644 drivers/char/lrng/lrng_es_sched.h + create mode 100644 drivers/char/lrng/lrng_es_timer_common.c + create mode 100644 drivers/char/lrng/lrng_es_timer_common.h + create mode 100644 drivers/char/lrng/lrng_hash_kcapi.c + create mode 100644 drivers/char/lrng/lrng_health.c + create mode 100644 drivers/char/lrng/lrng_health.h + create mode 100644 drivers/char/lrng/lrng_interface_aux.c + create mode 100644 drivers/char/lrng/lrng_interface_dev.c + create mode 100644 drivers/char/lrng/lrng_interface_dev_common.c + create mode 100644 drivers/char/lrng/lrng_interface_dev_common.h + create mode 100644 drivers/char/lrng/lrng_interface_hwrand.c + create mode 100644 drivers/char/lrng/lrng_interface_kcapi.c + create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.c + create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.h + create mode 100644 drivers/char/lrng/lrng_interface_random_user.c + create mode 100644 drivers/char/lrng/lrng_numa.c + create mode 100644 drivers/char/lrng/lrng_numa.h + create mode 100644 drivers/char/lrng/lrng_proc.c + create mode 100644 drivers/char/lrng/lrng_proc.h + create mode 100644 drivers/char/lrng/lrng_selftest.c + create mode 100644 drivers/char/lrng/lrng_sha.h + create mode 100644 drivers/char/lrng/lrng_sha1.c + create mode 100644 drivers/char/lrng/lrng_sha256.c + create mode 100644 drivers/char/lrng/lrng_switch.c + create mode 100644 drivers/char/lrng/lrng_sysctl.c + create mode 100644 drivers/char/lrng/lrng_sysctl.h + create mode 100644 drivers/char/lrng/lrng_testing.c + create mode 100644 drivers/char/lrng/lrng_testing.h + rename {crypto => include/crypto/internal}/jitterentropy.h (100%) + create mode 100644 include/linux/lrng.h + +diff --git a/MAINTAINERS b/MAINTAINERS +index 9a5a422817af..14556e749fb6 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -11740,6 +11740,13 @@ F: Documentation/litmus-tests/ + F: Documentation/memory-barriers.txt + F: tools/memory-model/ + ++LINUX RANDOM NUMBER GENERATOR (LRNG) DRIVER ++M: Stephan Mueller ++S: Maintained ++W: https://www.chronox.de/lrng.html ++F: drivers/char/lrng/ ++F: include/linux/lrng.h ++ + LIS3LV02D ACCELEROMETER DRIVER + M: Eric Piel + S: Maintained +diff --git a/crypto/drbg.c b/crypto/drbg.c +index 177983b6ae38..ea79a31014a4 100644 +--- a/crypto/drbg.c ++++ b/crypto/drbg.c +@@ -115,7 +115,7 @@ + * the SHA256 / AES 256 over other ciphers. Thus, the favored + * DRBGs are the latest entries in this array. + */ +-static const struct drbg_core drbg_cores[] = { ++const struct drbg_core drbg_cores[] = { + #ifdef CONFIG_CRYPTO_DRBG_CTR + { + .flags = DRBG_CTR | DRBG_STRENGTH128, +@@ -192,6 +192,7 @@ static const struct drbg_core drbg_cores[] = { + }, + #endif /* CONFIG_CRYPTO_DRBG_HMAC */ + }; ++EXPORT_SYMBOL(drbg_cores); + + static int drbg_uninstantiate(struct drbg_state *drbg); + +@@ -207,7 +208,7 @@ static int drbg_uninstantiate(struct drbg_state *drbg); + * Return: normalized strength in *bytes* value or 32 as default + * to counter programming errors + */ +-static inline unsigned short drbg_sec_strength(drbg_flag_t flags) ++unsigned short drbg_sec_strength(drbg_flag_t flags) + { + switch (flags & DRBG_STRENGTH_MASK) { + case DRBG_STRENGTH128: +@@ -220,6 +221,7 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) + return 32; + } + } ++EXPORT_SYMBOL(drbg_sec_strength); + + /* + * FIPS 140-2 continuous self test for the noise source +@@ -1252,7 +1254,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, + } + + /* Free all substructures in a DRBG state without the DRBG state structure */ +-static inline void drbg_dealloc_state(struct drbg_state *drbg) ++void drbg_dealloc_state(struct drbg_state *drbg) + { + if (!drbg) + return; +@@ -1273,12 +1275,13 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) + drbg->fips_primed = false; + } + } ++EXPORT_SYMBOL(drbg_dealloc_state); + + /* + * Allocate all sub-structures for a DRBG state. + * The DRBG state structure must already be allocated. + */ +-static inline int drbg_alloc_state(struct drbg_state *drbg) ++int drbg_alloc_state(struct drbg_state *drbg) + { + int ret = -ENOMEM; + unsigned int sb_size = 0; +@@ -1359,6 +1362,7 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) + drbg_dealloc_state(drbg); + return ret; + } ++EXPORT_SYMBOL(drbg_alloc_state); + + /************************************************************************* + * DRBG interface functions +@@ -1895,8 +1899,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, + * + * return: flags + */ +-static inline void drbg_convert_tfm_core(const char *cra_driver_name, +- int *coreref, bool *pr) ++void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, bool *pr) + { + int i = 0; + size_t start = 0; +@@ -1923,6 +1926,7 @@ static inline void drbg_convert_tfm_core(const char *cra_driver_name, + } + } + } ++EXPORT_SYMBOL(drbg_convert_tfm_core); + + static int drbg_kcapi_init(struct crypto_tfm *tfm) + { +diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c +index 2d115bec15ae..e7dac734d237 100644 +--- a/crypto/jitterentropy-kcapi.c ++++ b/crypto/jitterentropy-kcapi.c +@@ -42,8 +42,7 @@ + #include + #include + #include +- +-#include "jitterentropy.h" ++#include + + /*************************************************************************** + * Helper function +diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c +index 93bff3213823..81b80a4d3d3a 100644 +--- a/crypto/jitterentropy.c ++++ b/crypto/jitterentropy.c +@@ -133,7 +133,7 @@ struct rand_data { + #define JENT_ENTROPY_SAFETY_FACTOR 64 + + #include +-#include "jitterentropy.h" ++#include + + /*************************************************************************** + * Adaptive Proportion Test +diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig +index 0f378d29dab0..39c91f09cbe0 100644 +--- a/drivers/char/Kconfig ++++ b/drivers/char/Kconfig +@@ -459,4 +459,6 @@ config RANDOM_TRUST_BOOTLOADER + believe its RNG facilities may be faulty. This may also be configured + at boot time with "random.trust_bootloader=on/off". + ++source "drivers/char/lrng/Kconfig" ++ + endmenu +diff --git a/drivers/char/Makefile b/drivers/char/Makefile +index 1b35d1724565..0d56f0a24fcd 100644 +--- a/drivers/char/Makefile ++++ b/drivers/char/Makefile +@@ -3,7 +3,8 @@ + # Makefile for the kernel character device drivers. + # + +-obj-y += mem.o random.o ++obj-y += mem.o ++obj-$(CONFIG_RANDOM_DEFAULT_IMPL) += random.o + obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o + obj-y += misc.o + obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o +@@ -45,3 +46,5 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o + obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/ + obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o + obj-$(CONFIG_ADI) += adi.o ++ ++obj-$(CONFIG_LRNG) += lrng/ +diff --git a/drivers/char/lrng/Kconfig b/drivers/char/lrng/Kconfig +new file mode 100644 +index 000000000000..bfd6b3ea4f1b +--- /dev/null ++++ b/drivers/char/lrng/Kconfig +@@ -0,0 +1,907 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# Linux Random Number Generator configuration ++# ++ ++config RANDOM_DEFAULT_IMPL ++ bool "Kernel RNG Default Implementation" ++ default y ++ help ++ The default random number generator as provided with ++ drivers/char/random.c is selected with this option. ++ ++config LRNG ++ bool "Linux Random Number Generator" ++ default n ++ select CRYPTO_LIB_SHA256 if CRYPTO ++ help ++ The Linux Random Number Generator (LRNG) generates entropy ++ from different entropy sources. Each entropy source can ++ be enabled and configured independently. The interrupt ++ entropy source can be configured to be SP800-90B compliant. ++ The entire LRNG can be configured to be SP800-90C compliant. ++ Runtime-switchable cryptographic support is available. ++ The LRNG delivers significant entropy during boot. ++ ++ The LRNG also provides compliance to SP800-90A/B/C. ++ ++menu "Linux Random Number Generator Configuration" ++ depends on LRNG ++ ++if LRNG ++ ++config LRNG_SHA256 ++ bool ++ default y if CRYPTO_LIB_SHA256 ++ ++config LRNG_SHA1 ++ bool ++ default y if !CRYPTO_LIB_SHA256 ++ ++config LRNG_COMMON_DEV_IF ++ bool ++ ++config LRNG_DRNG_ATOMIC ++ bool ++ select LRNG_DRNG_CHACHA20 ++ ++config LRNG_SYSCTL ++ bool ++ depends on SYSCTL ++ ++config LRNG_RANDOM_IF ++ bool ++ default n if RANDOM_DEFAULT_IMPL ++ default y if !RANDOM_DEFAULT_IMPL ++ select LRNG_COMMON_DEV_IF ++ select LRNG_DRNG_ATOMIC ++ select LRNG_SYSCTL ++ ++menu "LRNG Interfaces" ++ ++config LRNG_KCAPI_IF ++ tristate "Interface with Kernel Crypto API" ++ depends on CRYPTO_RNG ++ help ++ The LRNG can be registered with the kernel crypto API's ++ random number generator framework. This offers a random ++ number generator with the name "lrng" and a priority that ++ is intended to be higher than the existing RNG ++ implementations. ++ ++config LRNG_HWRAND_IF ++ tristate "Interface with Hardware Random Number Generator Framework" ++ depends on HW_RANDOM ++ select LRNG_DRNG_ATOMIC ++ help ++ The LRNG can be registered with the hardware random number ++ generator framework. This offers a random number generator ++ with the name "lrng" that is accessible via the framework. ++ For example it allows pulling data from the LRNG via the ++ /dev/hwrng file. ++ ++config LRNG_DEV_IF ++ bool "Character device file interface" ++ select LRNG_COMMON_DEV_IF ++ help ++ The LRNG can create a character device file that operates ++ identically to /dev/random including IOCTL, read and write ++ operations. ++ ++endmenu # "LRNG Interfaces" ++ ++menu "Entropy Source Configuration" ++ ++config LRNG_RUNTIME_ES_CONFIG ++ bool "Enable runtime configuration of entropy sources" ++ help ++ When enabling this option, the LRNG provides the mechanism ++ allowing to alter the entropy rate of each entropy source ++ during boot time and runtime. ++ ++ Each entropy source allows its entropy rate changed with ++ a kernel command line option. When not providing any ++ option, the default specified during kernel compilation ++ is applied. ++ ++comment "Common Timer-based Entropy Source Configuration" ++ ++config LRNG_IRQ_DFLT_TIMER_ES ++ bool ++ ++config LRNG_SCHED_DFLT_TIMER_ES ++ bool ++ ++config LRNG_TIMER_COMMON ++ bool ++ ++choice ++ prompt "Default Timer-based Entropy Source" ++ default LRNG_IRQ_DFLT_TIMER_ES ++ depends on LRNG_TIMER_COMMON ++ help ++ Select the timer-based entropy source that is credited ++ with entropy. The other timer-based entropy sources may ++ be operational and provide data, but are credited with no ++ entropy. ++ ++ config LRNG_IRQ_DFLT_TIMER_ES ++ bool "Interrupt Entropy Source" ++ depends on LRNG_IRQ ++ help ++ The interrupt entropy source is selected as a timer-based ++ entropy source to provide entropy. ++ ++ config LRNG_SCHED_DFLT_TIMER_ES ++ bool "Scheduler Entropy Source" ++ depends on LRNG_SCHED ++ help ++ The scheduler entropy source is selected as timer-based ++ entropy source to provide entropy. ++endchoice ++ ++choice ++ prompt "LRNG Entropy Collection Pool Size" ++ default LRNG_COLLECTION_SIZE_1024 ++ depends on LRNG_TIMER_COMMON ++ help ++ Select the size of the LRNG entropy collection pool ++ storing data for the interrupt as well as the scheduler ++ entropy sources without performing a compression ++ operation. The larger the collection size is, the faster ++ the average interrupt handling will be. The collection ++ size represents the number of bytes of the per-CPU memory ++ used to batch up entropy event data. ++ ++ The default value is good for regular operations. Choose ++ larger sizes for servers that have no memory limitations. ++ If runtime memory is precious, choose a smaller size. ++ ++ The collection size is unrelated to the entropy rate ++ or the amount of entropy the LRNG can process. ++ ++ config LRNG_COLLECTION_SIZE_32 ++ depends on LRNG_CONTINUOUS_COMPRESSION_ENABLED ++ depends on !LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION ++ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES ++ bool "32 interrupt events" ++ ++ config LRNG_COLLECTION_SIZE_256 ++ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES ++ bool "256 interrupt events" ++ ++ config LRNG_COLLECTION_SIZE_512 ++ bool "512 interrupt events" ++ ++ config LRNG_COLLECTION_SIZE_1024 ++ bool "1024 interrupt events (default)" ++ ++ config LRNG_COLLECTION_SIZE_2048 ++ bool "2048 interrupt events" ++ ++ config LRNG_COLLECTION_SIZE_4096 ++ bool "4096 interrupt events" ++ ++ config LRNG_COLLECTION_SIZE_8192 ++ bool "8192 interrupt events" ++ ++endchoice ++ ++config LRNG_COLLECTION_SIZE ++ int ++ default 32 if LRNG_COLLECTION_SIZE_32 ++ default 256 if LRNG_COLLECTION_SIZE_256 ++ default 512 if LRNG_COLLECTION_SIZE_512 ++ default 1024 if LRNG_COLLECTION_SIZE_1024 ++ default 2048 if LRNG_COLLECTION_SIZE_2048 ++ default 4096 if LRNG_COLLECTION_SIZE_4096 ++ default 8192 if LRNG_COLLECTION_SIZE_8192 ++ ++config LRNG_HEALTH_TESTS ++ bool "Enable internal entropy source online health tests" ++ depends on LRNG_TIMER_COMMON ++ help ++ The online health tests applied to the interrupt entropy ++ source and to the scheduler entropy source to validate ++ the noise source at runtime for fatal errors. These tests ++ include SP800-90B compliant tests which are invoked if ++ the system is booted with fips=1. In case of fatal errors ++ during active SP800-90B tests, the issue is logged and ++ the noise data is discarded. These tests are required for ++ full compliance of the interrupt entropy source with ++ SP800-90B. ++ ++ If both, the scheduler and the interrupt entropy sources, ++ are enabled, the health tests for both are applied ++ independent of each other. ++ ++ If unsure, say Y. ++ ++config LRNG_RCT_BROKEN ++ bool "SP800-90B RCT with dangerous low cutoff value" ++ depends on LRNG_HEALTH_TESTS ++ depends on BROKEN ++ default n ++ help ++ This option enables a dangerously low SP800-90B repetitive ++ count test (RCT) cutoff value which makes it very likely ++ that the RCT is triggered to raise a self test failure. ++ ++ This option is ONLY intended for developers wanting to ++ test the effectiveness of the SP800-90B RCT health test. ++ ++ If unsure, say N. ++ ++config LRNG_APT_BROKEN ++ bool "SP800-90B APT with dangerous low cutoff value" ++ depends on LRNG_HEALTH_TESTS ++ depends on BROKEN ++ default n ++ help ++ This option enables a dangerously low SP800-90B adaptive ++ proportion test (APT) cutoff value which makes it very ++ likely that the APT is triggered to raise a self test ++ failure. ++ ++ This option is ONLY intended for developers wanting to ++ test the effectiveness of the SP800-90B APT health test. ++ ++ If unsure, say N. ++ ++# Default taken from SP800-90B sec 4.4.1 - significance level 2^-30 ++config LRNG_RCT_CUTOFF ++ int ++ default 31 if !LRNG_RCT_BROKEN ++ default 1 if LRNG_RCT_BROKEN ++ ++# Default taken from SP800-90B sec 4.4.2 - significance level 2^-30 ++config LRNG_APT_CUTOFF ++ int ++ default 325 if !LRNG_APT_BROKEN ++ default 32 if LRNG_APT_BROKEN ++ ++comment "Interrupt Entropy Source" ++ ++config LRNG_IRQ ++ bool "Enable Interrupt Entropy Source as LRNG Seed Source" ++ default y ++ depends on !RANDOM_DEFAULT_IMPL ++ select LRNG_TIMER_COMMON ++ help ++ The LRNG models an entropy source based on the timing of the ++ occurrence of interrupts. Enable this option to enable this ++ IRQ entropy source. ++ ++ The IRQ entropy source is triggered every time an interrupt ++ arrives and thus causes the interrupt handler to execute ++ slightly longer. Disabling the IRQ entropy source implies ++ that the performance penalty on the interrupt handler added ++ by the LRNG is eliminated. Yet, this entropy source is ++ considered to be an internal entropy source of the LRNG. ++ Thus, only disable it if you ensured that other entropy ++ sources are available that supply the LRNG with entropy. ++ ++ If you disable the IRQ entropy source, you MUST ensure ++ one or more entropy sources collectively have the ++ capability to deliver sufficient entropy with one invocation ++ at a rate compliant to the security strength of the DRNG ++ (usually 256 bits of entropy). In addition, if those ++ entropy sources do not deliver sufficient entropy during ++ first request, the reseed must be triggered from user ++ space or kernel space when sufficient entropy is considered ++ to be present. ++ ++ If unsure, say Y. ++ ++choice ++ prompt "Continuous entropy compression boot time setting" ++ default LRNG_CONTINUOUS_COMPRESSION_ENABLED ++ depends on LRNG_IRQ ++ help ++ Select the default behavior of the interrupt entropy source ++ continuous compression operation. ++ ++ The LRNG IRQ ES collects entropy data during each interrupt. ++ For performance reasons, a amount of entropy data defined by ++ the LRNG entropy collection pool size is concatenated into ++ an array. When that array is filled up, a hash is calculated ++ to compress the entropy. That hash is calculated in ++ interrupt context. ++ ++ In case such hash calculation in interrupt context is deemed ++ too time-consuming, the continuous compression operation ++ can be disabled. If disabled, the collection of entropy will ++ not trigger a hash compression operation in interrupt context. ++ The compression happens only when the DRNG is reseeded which is ++ in process context. This implies that old entropy data ++ collected after the last DRNG-reseed is overwritten with newer ++ entropy data once the collection pool is full instead of ++ retaining its entropy with the compression operation. ++ ++ config LRNG_CONTINUOUS_COMPRESSION_ENABLED ++ bool "Enable continuous compression (default)" ++ ++ config LRNG_CONTINUOUS_COMPRESSION_DISABLED ++ bool "Disable continuous compression" ++ ++endchoice ++ ++config LRNG_ENABLE_CONTINUOUS_COMPRESSION ++ bool ++ default y if LRNG_CONTINUOUS_COMPRESSION_ENABLED ++ default n if LRNG_CONTINUOUS_COMPRESSION_DISABLED ++ ++config LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION ++ bool "Runtime-switchable continuous entropy compression" ++ depends on LRNG_IRQ ++ help ++ Per default, the interrupt entropy source continuous ++ compression operation behavior is hard-wired into the kernel. ++ Enable this option to allow it to be configurable at boot time. ++ ++ To modify the default behavior of the continuous ++ compression operation, use the kernel command line option ++ of lrng_sw_noise.lrng_pcpu_continuous_compression. ++ ++ If unsure, say N. ++ ++config LRNG_IRQ_ENTROPY_RATE ++ int "Interrupt Entropy Source Entropy Rate" ++ depends on LRNG_IRQ ++ range 256 4294967295 if LRNG_IRQ_DFLT_TIMER_ES ++ range 4294967295 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES ++ default 256 if LRNG_IRQ_DFLT_TIMER_ES ++ default 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES ++ help ++ The LRNG will collect the configured number of interrupts to ++ obtain 256 bits of entropy. This value can be set to any between ++ 256 and 4294967295. The LRNG guarantees that this value is not ++ lower than 256. This lower limit implies that one interrupt event ++ is credited with one bit of entropy. This value is subject to the ++ increase by the oversampling factor, if no high-resolution timer ++ is found. ++ ++ In order to effectively disable the interrupt entropy source, ++ the option has to be set to 4294967295. In this case, the ++ interrupt entropy source will still deliver data but without ++ being credited with entropy. ++ ++comment "Jitter RNG Entropy Source" ++ ++config LRNG_JENT ++ bool "Enable Jitter RNG as LRNG Seed Source" ++ depends on CRYPTO ++ select CRYPTO_JITTERENTROPY ++ help ++ The LRNG may use the Jitter RNG as entropy source. Enabling ++ this option enables the use of the Jitter RNG. Its default ++ entropy level is 16 bits of entropy per 256 data bits delivered ++ by the Jitter RNG. This entropy level can be changed at boot ++ time or at runtime with the lrng_base.jitterrng configuration ++ variable. ++ ++config LRNG_JENT_ENTROPY_RATE ++ int "Jitter RNG Entropy Source Entropy Rate" ++ depends on LRNG_JENT ++ range 0 256 ++ default 16 ++ help ++ The option defines the amount of entropy the LRNG applies to 256 ++ bits of data obtained from the Jitter RNG entropy source. The ++ LRNG enforces the limit that this value must be in the range ++ between 0 and 256. ++ ++ When configuring this value to 0, the Jitter RNG entropy source ++ will provide 256 bits of data without being credited to contain ++ entropy. ++ ++comment "CPU Entropy Source" ++ ++config LRNG_CPU ++ bool "Enable CPU Entropy Source as LRNG Seed Source" ++ default y ++ help ++ Current CPUs commonly contain entropy sources which can be ++ used to seed the LRNG. For example, the Intel RDSEED ++ instruction, or the POWER DARN instruction will be sourced ++ to seed the LRNG if this option is enabled. ++ ++ Note, if this option is enabled and the underlying CPU ++ does not offer such entropy source, the LRNG will automatically ++ detect this and ignore the hardware. ++ ++config LRNG_CPU_FULL_ENT_MULTIPLIER ++ int ++ default 1 if !LRNG_TEST_CPU_ES_COMPRESSION ++ default 123 if LRNG_TEST_CPU_ES_COMPRESSION ++ ++config LRNG_CPU_ENTROPY_RATE ++ int "CPU Entropy Source Entropy Rate" ++ depends on LRNG_CPU ++ range 0 256 ++ default 8 ++ help ++ The option defines the amount of entropy the LRNG applies to 256 ++ bits of data obtained from the CPU entropy source. The LRNG ++ enforces the limit that this value must be in the range between ++ 0 and 256. ++ ++ When configuring this value to 0, the CPU entropy source will ++ provide 256 bits of data without being credited to contain ++ entropy. ++ ++ Note, this option is overwritten when the option ++ CONFIG_RANDOM_TRUST_CPU is set. ++ ++comment "Scheduler Entropy Source" ++ ++config LRNG_SCHED ++ bool "Enable Scheduer Entropy Source as LRNG Seed Source" ++ select LRNG_TIMER_COMMON ++ help ++ The LRNG models an entropy source based on the timing of the ++ occurrence of scheduler-triggered context switches. Enable ++ this option to enable this scheduler entropy source. ++ ++ The scheduler entropy source is triggered every time a ++ context switch is triggered thus causes the scheduler to ++ execute slightly longer. Disabling the scheduler entropy ++ source implies that the performance penalty on the scheduler ++ added by the LRNG is eliminated. Yet, this entropy source is ++ considered to be an internal entropy source of the LRNG. ++ Thus, only disable it if you ensured that other entropy ++ sources are available that supply the LRNG with entropy. ++ ++ If you disable the scheduler entropy source, you MUST ++ ensure one or more entropy sources collectively have the ++ capability to deliver sufficient entropy with one invocation ++ at a rate compliant to the security strength of the DRNG ++ (usually 256 bits of entropy). In addition, if those ++ entropy sources do not deliver sufficient entropy during ++ first request, the reseed must be triggered from user ++ space or kernel space when sufficient entropy is considered ++ to be present. ++ ++ If unsure, say Y. ++ ++config LRNG_SCHED_ENTROPY_RATE ++ int "Scheduler Entropy Source Entropy Rate" ++ depends on LRNG_SCHED ++ range 256 4294967295 if LRNG_SCHED_DFLT_TIMER_ES ++ range 4294967295 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES ++ default 256 if LRNG_SCHED_DFLT_TIMER_ES ++ default 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES ++ help ++ The LRNG will collect the configured number of context switches ++ triggered by the scheduler to obtain 256 bits of entropy. This ++ value can be set to any between 256 and 4294967295. The LRNG ++ guarantees that this value is not lower than 256. This lower ++ limit implies that one interrupt event is credited with one bit ++ of entropy. This value is subject to the increase by the ++ oversampling factor, if no high-resolution timer is found. ++ ++ In order to effectively disable the scheduler entropy source, ++ the option has to be set to 4294967295. In this case, the ++ scheduler entropy source will still deliver data but without ++ being credited with entropy. ++ ++comment "Kernel RNG Entropy Source" ++ ++config LRNG_KERNEL_RNG ++ bool "Enable Kernel RNG as LRNG Seed Source" ++ depends on RANDOM_DEFAULT_IMPL ++ help ++ The LRNG may use the kernel RNG (random.c) as entropy ++ source. ++ ++config LRNG_KERNEL_RNG_ENTROPY_RATE ++ int "Kernel RNG Entropy Source Entropy Rate" ++ depends on LRNG_KERNEL_RNG ++ range 0 256 ++ default 256 ++ help ++ The option defines the amount of entropy the LRNG applies to 256 ++ bits of data obtained from the kernel RNG entropy source. The ++ LRNG enforces the limit that this value must be in the range ++ between 0 and 256. ++ ++ When configuring this value to 0, the kernel RNG entropy source ++ will provide 256 bits of data without being credited to contain ++ entropy. ++ ++ Note: This value is set to 0 automatically when booting the ++ kernel in FIPS mode (with fips=1 kernel command line option). ++ This is due to the fact that random.c is not SP800-90B ++ compliant. ++ ++endmenu # "Entropy Source Configuration" ++ ++config LRNG_DRNG_CHACHA20 ++ tristate ++ ++config LRNG_DRBG ++ tristate ++ depends on CRYPTO ++ select CRYPTO_DRBG_MENU ++ ++config LRNG_DRNG_KCAPI ++ tristate ++ depends on CRYPTO ++ select CRYPTO_RNG ++ ++config LRNG_SWITCH ++ bool ++ ++menuconfig LRNG_SWITCH_HASH ++ bool "Support conditioning hash runtime switching" ++ select LRNG_SWITCH ++ help ++ The LRNG uses a default message digest. With this ++ configuration option other message digests can be selected ++ and loaded at runtime. ++ ++if LRNG_SWITCH_HASH ++ ++config LRNG_HASH_KCAPI ++ tristate "Kernel crypto API hashing support for LRNG" ++ select CRYPTO_HASH ++ select CRYPTO_SHA512 ++ help ++ Enable the kernel crypto API support for entropy compression ++ and conditioning functions. ++ ++endif # LRNG_SWITCH_HASH ++ ++menuconfig LRNG_SWITCH_DRNG ++ bool "Support DRNG runtime switching" ++ select LRNG_SWITCH ++ help ++ The LRNG uses a default DRNG With this configuration ++ option other DRNGs or message digests can be selected and ++ loaded at runtime. ++ ++if LRNG_SWITCH_DRNG ++ ++config LRNG_SWITCH_DRNG_CHACHA20 ++ tristate "ChaCha20-based DRNG support for LRNG" ++ depends on !LRNG_DFLT_DRNG_CHACHA20 ++ select LRNG_DRNG_CHACHA20 ++ help ++ Enable the ChaCha20-based DRNG. This DRNG implementation ++ does not depend on the kernel crypto API presence. ++ ++config LRNG_SWITCH_DRBG ++ tristate "SP800-90A support for the LRNG" ++ depends on !LRNG_DFLT_DRNG_DRBG ++ select LRNG_DRBG ++ help ++ Enable the SP800-90A DRBG support for the LRNG. Once the ++ module is loaded, output from /dev/random, /dev/urandom, ++ getrandom(2), or get_random_bytes_full is provided by a DRBG. ++ ++config LRNG_SWITCH_DRNG_KCAPI ++ tristate "Kernel Crypto API support for the LRNG" ++ depends on !LRNG_DFLT_DRNG_KCAPI ++ depends on !LRNG_SWITCH_DRBG ++ select LRNG_DRNG_KCAPI ++ help ++ Enable the support for generic pseudo-random number ++ generators offered by the kernel crypto API with the ++ LRNG. Once the module is loaded, output from /dev/random, ++ /dev/urandom, getrandom(2), or get_random_bytes is ++ provided by the selected kernel crypto API RNG. ++ ++endif # LRNG_SWITCH_DRNG ++ ++choice ++ prompt "LRNG Default DRNG" ++ default LRNG_DFLT_DRNG_CHACHA20 ++ help ++ Select the default deterministic random number generator ++ that is used by the LRNG. When enabling the switchable ++ cryptographic mechanism support, this DRNG can be ++ replaced at runtime. ++ ++ config LRNG_DFLT_DRNG_CHACHA20 ++ bool "ChaCha20-based DRNG" ++ select LRNG_DRNG_CHACHA20 ++ ++ config LRNG_DFLT_DRNG_DRBG ++ bool "SP800-90A DRBG" ++ select LRNG_DRBG ++ ++ config LRNG_DFLT_DRNG_KCAPI ++ bool "Kernel Crypto API DRNG" ++ select LRNG_DRNG_KCAPI ++endchoice ++ ++menuconfig LRNG_TESTING_MENU ++ bool "LRNG testing interfaces" ++ depends on DEBUG_FS ++ help ++ Enable one or more of the following test interfaces. ++ ++ If unsure, say N. ++ ++if LRNG_TESTING_MENU ++ ++config LRNG_TESTING ++ bool ++ ++config LRNG_TESTING_RECORDING ++ bool ++ ++comment "Interrupt Entropy Source Test Interfaces" ++ ++config LRNG_RAW_HIRES_ENTROPY ++ bool "Interface to obtain raw unprocessed IRQ noise source data" ++ default y ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned high resolution time stamp noise that ++ is collected by the LRNG for statistical analysis. Extracted ++ noise data is not used to seed the LRNG. ++ ++ The raw noise data can be obtained using the lrng_raw_hires ++ debugfs file. Using the option lrng_testing.boot_raw_hires_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_RAW_JIFFIES_ENTROPY ++ bool "Entropy test interface to Jiffies of IRQ noise source" ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned Jiffies that is collected by ++ the LRNG for statistical analysis. This data is used for ++ seeding the LRNG if a high-resolution time stamp is not ++ available. If a high-resolution time stamp is detected, ++ the Jiffies value is not collected by the LRNG and no ++ data is provided via the test interface. Extracted noise ++ data is not used to seed the random number generator. ++ ++ The raw noise data can be obtained using the lrng_raw_jiffies ++ debugfs file. Using the option lrng_testing.boot_raw_jiffies_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_RAW_IRQ_ENTROPY ++ bool "Entropy test interface to IRQ number noise source" ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned interrupt number that is collected by ++ the LRNG for statistical analysis. Extracted noise data is ++ not used to seed the random number generator. ++ ++ The raw noise data can be obtained using the lrng_raw_irq ++ debugfs file. Using the option lrng_testing.boot_raw_irq_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_RAW_RETIP_ENTROPY ++ bool "Entropy test interface to RETIP value of IRQ noise source" ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned return instruction pointer value ++ that is collected by the LRNG for statistical analysis. ++ Extracted noise data is not used to seed the random number ++ generator. ++ ++ The raw noise data can be obtained using the lrng_raw_retip ++ debugfs file. Using the option lrng_testing.boot_raw_retip_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_RAW_REGS_ENTROPY ++ bool "Entropy test interface to IRQ register value noise source" ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned interrupt register value that is ++ collected by the LRNG for statistical analysis. Extracted noise ++ data is not used to seed the random number generator. ++ ++ The raw noise data can be obtained using the lrng_raw_regs ++ debugfs file. Using the option lrng_testing.boot_raw_regs_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_RAW_ARRAY ++ bool "Test interface to LRNG raw entropy IRQ storage array" ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw noise data that is collected by the LRNG ++ in the per-CPU array for statistical analysis. The purpose ++ of this interface is to verify that the array handling code ++ truly only concatenates data and provides the same entropy ++ rate as the raw unconditioned noise source when assessing ++ the collected data byte-wise. ++ ++ The data can be obtained using the lrng_raw_array debugfs ++ file. Using the option lrng_testing.boot_raw_array=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_IRQ_PERF ++ bool "LRNG interrupt entropy source performance monitor" ++ depends on LRNG_IRQ ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ With this option, the performance monitor of the LRNG ++ interrupt handling code is enabled. The file provides ++ the execution time of the interrupt handler in ++ cycles. ++ ++ The interrupt performance data can be obtained using ++ the lrng_irq_perf debugfs file. Using the option ++ lrng_testing.boot_irq_perf=1 the performance data of ++ the first 1000 entropy events since boot can be sampled. ++ ++comment "Scheduler Entropy Source Test Interfaces" ++ ++config LRNG_RAW_SCHED_HIRES_ENTROPY ++ bool "Interface to obtain raw unprocessed scheduler noise source data" ++ depends on LRNG_SCHED ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned high resolution time stamp noise that ++ is collected by the LRNG for the Scheduler-based noise source ++ for statistical analysis. Extracted noise data is not used to ++ seed the LRNG. ++ ++ The raw noise data can be obtained using the lrng_raw_sched_hires ++ debugfs file. Using the option ++ lrng_testing.boot_raw_sched_hires_test=1 the raw noise of the ++ first 1000 entropy events since boot can be sampled. ++ ++config LRNG_RAW_SCHED_PID_ENTROPY ++ bool "Entropy test interface to PID value" ++ depends on LRNG_SCHED ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned PID value that is collected by the ++ LRNG for statistical analysis. Extracted noise ++ data is not used to seed the random number generator. ++ ++ The raw noise data can be obtained using the ++ lrng_raw_sched_pid debugfs file. Using the option ++ lrng_testing.boot_raw_sched_pid_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_RAW_SCHED_START_TIME_ENTROPY ++ bool "Entropy test interface to task start time value" ++ depends on LRNG_SCHED ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned task start time value that is collected ++ by the LRNG for statistical analysis. Extracted noise ++ data is not used to seed the random number generator. ++ ++ The raw noise data can be obtained using the ++ lrng_raw_sched_starttime debugfs file. Using the option ++ lrng_testing.boot_raw_sched_starttime_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++ ++config LRNG_RAW_SCHED_NVCSW_ENTROPY ++ bool "Entropy test interface to task context switch numbers" ++ depends on LRNG_SCHED ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ The test interface allows a privileged process to capture ++ the raw unconditioned task numbers of context switches that ++ are collected by the LRNG for statistical analysis. Extracted ++ noise data is not used to seed the random number generator. ++ ++ The raw noise data can be obtained using the ++ lrng_raw_sched_nvcsw debugfs file. Using the option ++ lrng_testing.boot_raw_sched_nvcsw_test=1 ++ the raw noise of the first 1000 entropy events since boot ++ can be sampled. ++ ++config LRNG_SCHED_PERF ++ bool "LRNG scheduler entropy source performance monitor" ++ depends on LRNG_SCHED ++ select LRNG_TESTING ++ select LRNG_TESTING_RECORDING ++ help ++ With this option, the performance monitor of the LRNG ++ scheduler event handling code is enabled. The file provides ++ the execution time of the interrupt handler in cycles. ++ ++ The scheduler performance data can be obtained using ++ the lrng_sched_perf debugfs file. Using the option ++ lrng_testing.boot_sched_perf=1 the performance data of ++ the first 1000 entropy events since boot can be sampled. ++ ++comment "Auxiliary Test Interfaces" ++ ++config LRNG_ACVT_HASH ++ bool "Enable LRNG ACVT Hash interface" ++ select LRNG_TESTING ++ help ++ With this option, the LRNG built-in hash function used for ++ auxiliary pool management and prior to switching the ++ cryptographic backends is made available for ACVT. The ++ interface allows writing of the data to be hashed ++ into the interface. The read operation triggers the hash ++ operation to generate message digest. ++ ++ The ACVT interface is available with the lrng_acvt_hash ++ debugfs file. ++ ++config LRNG_RUNTIME_MAX_WO_RESEED_CONFIG ++ bool "Enable runtime configuration of max reseed threshold" ++ help ++ When enabling this option, the LRNG provides an interface ++ allowing the setting of the maximum number of DRNG generate ++ operations without a reseed that has full entropy. The ++ interface is lrng_drng.max_wo_reseed. ++ ++config LRNG_TEST_CPU_ES_COMPRESSION ++ bool "Force CPU ES compression operation" ++ help ++ When enabling this option, the CPU ES compression operation ++ is forced by setting an arbitrary value > 1 for the data ++ multiplier even when the CPU ES would deliver full entropy. ++ This allows testing of the compression operation. It ++ therefore forces to pull more data from the CPU ES ++ than what may be required. ++ ++endif #LRNG_TESTING_MENU ++ ++config LRNG_SELFTEST ++ bool "Enable power-on and on-demand self-tests" ++ help ++ The power-on self-tests are executed during boot time ++ covering the ChaCha20 DRNG, the hash operation used for ++ processing the entropy pools and the auxiliary pool, and ++ the time stamp management of the LRNG. ++ ++ The on-demand self-tests are triggered by writing any ++ value into the SysFS file selftest_status. At the same ++ time, when reading this file, the test status is ++ returned. A zero indicates that all tests were executed ++ successfully. ++ ++ If unsure, say Y. ++ ++if LRNG_SELFTEST ++ ++config LRNG_SELFTEST_PANIC ++ bool "Panic the kernel upon self-test failure" ++ help ++ If the option is enabled, the kernel is terminated if an ++ LRNG power-on self-test failure is detected. ++ ++endif # LRNG_SELFTEST ++ ++endif # LRNG ++ ++endmenu # LRNG +diff --git a/drivers/char/lrng/Makefile b/drivers/char/lrng/Makefile +new file mode 100644 +index 000000000000..f61fc40f4620 +--- /dev/null ++++ b/drivers/char/lrng/Makefile +@@ -0,0 +1,39 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# Makefile for the Entropy Source and DRNG Manager. ++# ++ ++obj-y += lrng_es_mgr.o lrng_drng_mgr.o \ ++ lrng_es_aux.o ++obj-$(CONFIG_LRNG_SHA256) += lrng_sha256.o ++obj-$(CONFIG_LRNG_SHA1) += lrng_sha1.o ++ ++obj-$(CONFIG_SYSCTL) += lrng_proc.o ++obj-$(CONFIG_LRNG_SYSCTL) += lrng_sysctl.o ++obj-$(CONFIG_NUMA) += lrng_numa.o ++ ++obj-$(CONFIG_LRNG_SWITCH) += lrng_switch.o ++obj-$(CONFIG_LRNG_HASH_KCAPI) += lrng_hash_kcapi.o ++obj-$(CONFIG_LRNG_DRNG_CHACHA20) += lrng_drng_chacha20.o ++obj-$(CONFIG_LRNG_DRBG) += lrng_drng_drbg.o ++obj-$(CONFIG_LRNG_DRNG_KCAPI) += lrng_drng_kcapi.o ++obj-$(CONFIG_LRNG_DRNG_ATOMIC) += lrng_drng_atomic.o ++ ++obj-$(CONFIG_LRNG_TIMER_COMMON) += lrng_es_timer_common.o ++obj-$(CONFIG_LRNG_IRQ) += lrng_es_irq.o ++obj-$(CONFIG_LRNG_KERNEL_RNG) += lrng_es_krng.o ++obj-$(CONFIG_LRNG_SCHED) += lrng_es_sched.o ++obj-$(CONFIG_LRNG_CPU) += lrng_es_cpu.o ++obj-$(CONFIG_LRNG_JENT) += lrng_es_jent.o ++ ++obj-$(CONFIG_LRNG_HEALTH_TESTS) += lrng_health.o ++obj-$(CONFIG_LRNG_TESTING) += lrng_testing.o ++obj-$(CONFIG_LRNG_SELFTEST) += lrng_selftest.o ++ ++obj-$(CONFIG_LRNG_COMMON_DEV_IF) += lrng_interface_dev_common.o ++obj-$(CONFIG_LRNG_RANDOM_IF) += lrng_interface_random_user.o \ ++ lrng_interface_random_kernel.o \ ++ lrng_interface_aux.o ++obj-$(CONFIG_LRNG_KCAPI_IF) += lrng_interface_kcapi.o ++obj-$(CONFIG_LRNG_DEV_IF) += lrng_interface_dev.o ++obj-$(CONFIG_LRNG_HWRAND_IF) += lrng_interface_hwrand.o +diff --git a/drivers/char/lrng/lrng_definitions.h b/drivers/char/lrng/lrng_definitions.h +new file mode 100644 +index 000000000000..50f87c40f701 +--- /dev/null ++++ b/drivers/char/lrng/lrng_definitions.h +@@ -0,0 +1,151 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_DEFINITIONS_H ++#define _LRNG_DEFINITIONS_H ++ ++#include ++#include ++#include ++ ++/*************************** General LRNG parameter ***************************/ ++ ++/* ++ * Specific settings for different use cases ++ */ ++#ifdef CONFIG_CRYPTO_FIPS ++# define LRNG_OVERSAMPLE_ES_BITS 64 ++# define LRNG_SEED_BUFFER_INIT_ADD_BITS 128 ++#else /* CONFIG_CRYPTO_FIPS */ ++# define LRNG_OVERSAMPLE_ES_BITS 0 ++# define LRNG_SEED_BUFFER_INIT_ADD_BITS 0 ++#endif /* CONFIG_CRYPTO_FIPS */ ++ ++/* Security strength of LRNG -- this must match DRNG security strength */ ++#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32 ++#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8) ++#define LRNG_DRNG_INIT_SEED_SIZE_BITS \ ++ (LRNG_DRNG_SECURITY_STRENGTH_BITS + LRNG_SEED_BUFFER_INIT_ADD_BITS) ++#define LRNG_DRNG_INIT_SEED_SIZE_BYTES (LRNG_DRNG_INIT_SEED_SIZE_BITS >> 3) ++ ++/* ++ * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is ++ * considered a safer margin. ++ * ++ * This value is allowed to be changed. ++ */ ++#define LRNG_DRNG_MAX_REQSIZE (1<<12) ++ ++/* ++ * SP800-90A defines a maximum number of requests between reseeds of 2^48. ++ * The given value is considered a much safer margin, balancing requests for ++ * frequent reseeds with the need to conserve entropy. This value MUST NOT be ++ * larger than INT_MAX because it is used in an atomic_t. ++ * ++ * This value is allowed to be changed. ++ */ ++#define LRNG_DRNG_RESEED_THRESH (1<<20) ++ ++/* ++ * Maximum DRNG generation operations without reseed having full entropy ++ * This value defines the absolute maximum value of DRNG generation operations ++ * without a reseed holding full entropy. LRNG_DRNG_RESEED_THRESH is the ++ * threshold when a new reseed is attempted. But it is possible that this fails ++ * to deliver full entropy. In this case the DRNG will continue to provide data ++ * even though it was not reseeded with full entropy. To avoid in the extreme ++ * case that no reseed is performed for too long, this threshold is enforced. ++ * If that absolute low value is reached, the LRNG is marked as not operational. ++ * ++ * This value is allowed to be changed. ++ */ ++#define LRNG_DRNG_MAX_WITHOUT_RESEED (1<<30) ++ ++/* ++ * Min required seed entropy is 128 bits covering the minimum entropy ++ * requirement of SP800-131A and the German BSI's TR02102. ++ * ++ * This value is allowed to be changed. ++ */ ++#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS ++#define LRNG_MIN_SEED_ENTROPY_BITS 128 ++#define LRNG_INIT_ENTROPY_BITS 32 ++ ++/* ++ * Wakeup value ++ * ++ * This value is allowed to be changed but must not be larger than the ++ * digest size of the hash operation used update the aux_pool. ++ */ ++#ifdef CONFIG_LRNG_SHA256 ++# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE ++#else ++# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE ++#endif ++#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE ++ ++/* ++ * If the switching support is configured, we must provide support up to ++ * the largest digest size. Without switching support, we know it is only ++ * the built-in digest size. ++ */ ++#ifdef CONFIG_LRNG_SWITCH ++# define LRNG_MAX_DIGESTSIZE 64 ++#else ++# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE ++#endif ++ ++/* ++ * Oversampling factor of timer-based events to obtain ++ * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a ++ * high-resolution time stamp is not available. In this case, jiffies and ++ * register contents are used to fill the entropy pool. These noise sources ++ * are much less entropic than the high-resolution timer. The entropy content ++ * is the entropy content assumed with LRNG_[IRQ|SCHED]_ENTROPY_BITS divided by ++ * LRNG_ES_OVERSAMPLING_FACTOR. ++ * ++ * This value is allowed to be changed. ++ */ ++#define LRNG_ES_OVERSAMPLING_FACTOR 10 ++ ++/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */ ++#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN ++ ++/* ++ * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK ++ * as it will be casted into a struct shash_desc. ++ */ ++#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE) ++ ++/****************************** Helper code ***********************************/ ++ ++static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits) ++{ ++ /* Obtain entropy statement */ ++ ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS; ++ /* Cap entropy to buffer size in bits */ ++ ent_bits = min_t(u32, ent_bits, requested_bits); ++ return ent_bits; ++} ++ ++/* Convert entropy in bits into nr. of events with the same entropy content. */ ++static inline u32 lrng_entropy_to_data(u32 entropy_bits, u32 entropy_rate) ++{ ++ return ((entropy_bits * entropy_rate) / ++ LRNG_DRNG_SECURITY_STRENGTH_BITS); ++} ++ ++/* Convert number of events into entropy value. */ ++static inline u32 lrng_data_to_entropy(u32 num, u32 entropy_rate) ++{ ++ return ((num * LRNG_DRNG_SECURITY_STRENGTH_BITS) / ++ entropy_rate); ++} ++ ++static inline u32 atomic_read_u32(atomic_t *v) ++{ ++ return (u32)atomic_read(v); ++} ++ ++#endif /* _LRNG_DEFINITIONS_H */ +diff --git a/drivers/char/lrng/lrng_drng_atomic.c b/drivers/char/lrng/lrng_drng_atomic.c +new file mode 100644 +index 000000000000..c112366677b6 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_atomic.c +@@ -0,0 +1,171 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG DRNG for atomic contexts ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++ ++#include "lrng_drng_atomic.h" ++#include "lrng_drng_chacha20.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_mgr.h" ++#include "lrng_sha.h" ++ ++static struct chacha20_state chacha20_atomic = { ++ LRNG_CC20_INIT_RFC7539(.block) ++}; ++ ++/* ++ * DRNG usable in atomic context. This DRNG will always use the ChaCha20 ++ * DRNG. It will never benefit from a DRNG switch like the "regular" DRNG. If ++ * there was no DRNG switch, the atomic DRNG is identical to the "regular" DRNG. ++ * ++ * The reason for having this is due to the fact that DRNGs other than ++ * the ChaCha20 DRNG may sleep. ++ */ ++static struct lrng_drng lrng_drng_atomic = { ++ LRNG_DRNG_STATE_INIT(lrng_drng_atomic, ++ &chacha20_atomic, NULL, ++ &lrng_cc20_drng_cb, &lrng_sha_hash_cb), ++ .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_atomic.spin_lock) ++}; ++ ++void lrng_drng_atomic_reset(void) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); ++ lrng_drng_reset(&lrng_drng_atomic); ++ spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); ++} ++ ++void lrng_drng_atomic_force_reseed(void) ++{ ++ lrng_drng_atomic.force_reseed = lrng_drng_atomic.fully_seeded; ++} ++ ++static bool lrng_drng_atomic_must_reseed(struct lrng_drng *drng) ++{ ++ return (!drng->fully_seeded || ++ atomic_read(&lrng_drng_atomic.requests) == 0 || ++ drng->force_reseed || ++ time_after(jiffies, ++ drng->last_seeded + lrng_drng_reseed_max_time * HZ)); ++} ++ ++void lrng_drng_atomic_seed_drng(struct lrng_drng *regular_drng) ++{ ++ u8 seedbuf[LRNG_DRNG_SECURITY_STRENGTH_BYTES] ++ __aligned(LRNG_KCAPI_ALIGN); ++ int ret; ++ ++ if (!lrng_drng_atomic_must_reseed(&lrng_drng_atomic)) ++ return; ++ ++ /* ++ * Reseed atomic DRNG another DRNG "regular" while this regular DRNG ++ * is reseeded. Therefore, this code operates in non-atomic context and ++ * thus can use the lrng_drng_get function to get random numbers from ++ * the just freshly seeded DRNG. ++ */ ++ ret = lrng_drng_get(regular_drng, seedbuf, sizeof(seedbuf)); ++ ++ if (ret < 0) { ++ pr_warn("Error generating random numbers for atomic DRNG: %d\n", ++ ret); ++ } else { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); ++ lrng_drng_inject(&lrng_drng_atomic, seedbuf, ret, ++ regular_drng->fully_seeded, "atomic"); ++ spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); ++ } ++ memzero_explicit(&seedbuf, sizeof(seedbuf)); ++} ++ ++void lrng_drng_atomic_seed_es(void) ++{ ++ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); ++ struct lrng_drng *drng = &lrng_drng_atomic; ++ unsigned long flags; ++ u32 requested_bits = LRNG_MIN_SEED_ENTROPY_BITS; ++ ++ if (lrng_drng_atomic.fully_seeded) ++ return; ++ ++ /* ++ * When the LRNG reached the minimally seeded level, leave 256 bits of ++ * entropy for the regular DRNG. In addition ensure that additional ++ * 256 bits are available for the atomic DRNG to get to the fully ++ * seeded stage of the LRNG. ++ */ ++ if (lrng_state_min_seeded()) { ++ u32 avail_bits = lrng_avail_entropy(); ++ ++ requested_bits = ++ (avail_bits >= 2 * LRNG_FULL_SEED_ENTROPY_BITS) ? ++ LRNG_FULL_SEED_ENTROPY_BITS : 0; ++ ++ if (!requested_bits) ++ return; ++ } ++ ++ pr_debug("atomic DRNG seeding attempt to pull %u bits of entropy directly from entropy sources\n", ++ requested_bits); ++ ++ lrng_fill_seed_buffer(&seedbuf, requested_bits); ++ spin_lock_irqsave(&drng->spin_lock, flags); ++ lrng_drng_inject(&lrng_drng_atomic, (u8 *)&seedbuf, sizeof(seedbuf), ++ lrng_fully_seeded(drng->fully_seeded, ++ lrng_entropy_rate_eb(&seedbuf)), ++ "atomic"); ++ spin_unlock_irqrestore(&drng->spin_lock, flags); ++ lrng_init_ops(&seedbuf); ++ ++ /* ++ * Do not call lrng_init_ops(seedbuf) here as the atomic DRNG does not ++ * serve common users. ++ */ ++ memzero_explicit(&seedbuf, sizeof(seedbuf)); ++} ++ ++static void lrng_drng_atomic_get(u8 *outbuf, u32 outbuflen) ++{ ++ struct lrng_drng *drng = &lrng_drng_atomic; ++ unsigned long flags; ++ ++ if (!outbuf || !outbuflen) ++ return; ++ ++ outbuflen = min_t(size_t, outbuflen, INT_MAX); ++ ++ while (outbuflen) { ++ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); ++ int ret; ++ ++ atomic_dec(&drng->requests); ++ ++ spin_lock_irqsave(&drng->spin_lock, flags); ++ ret = drng->drng_cb->drng_generate(drng->drng, outbuf, todo); ++ spin_unlock_irqrestore(&drng->spin_lock, flags); ++ if (ret <= 0) { ++ pr_warn("getting random data from DRNG failed (%d)\n", ++ ret); ++ return; ++ } ++ outbuflen -= ret; ++ outbuf += ret; ++ } ++} ++ ++void lrng_get_random_bytes(void *buf, int nbytes) ++{ ++ lrng_drng_atomic_get((u8 *)buf, (u32)nbytes); ++ lrng_debug_report_seedlevel("lrng_get_random_bytes"); ++} ++EXPORT_SYMBOL(lrng_get_random_bytes); +diff --git a/drivers/char/lrng/lrng_drng_atomic.h b/drivers/char/lrng/lrng_drng_atomic.h +new file mode 100644 +index 000000000000..d5d24fc94f2e +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_atomic.h +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_DRNG_ATOMIC_H ++#define _LRNG_DRNG_ATOMIC_H ++ ++#include "lrng_drng_mgr.h" ++ ++#ifdef CONFIG_LRNG_DRNG_ATOMIC ++void lrng_drng_atomic_reset(void); ++void lrng_drng_atomic_seed_drng(struct lrng_drng *drng); ++void lrng_drng_atomic_seed_es(void); ++void lrng_drng_atomic_force_reseed(void); ++#else /* CONFIG_LRNG_DRNG_ATOMIC */ ++static inline void lrng_drng_atomic_reset(void) { } ++static inline void lrng_drng_atomic_seed_drng(struct lrng_drng *drng) { } ++static inline void lrng_drng_atomic_seed_es(void) { } ++static inline void lrng_drng_atomic_force_reseed(void) { } ++#endif /* CONFIG_LRNG_DRNG_ATOMIC */ ++ ++#endif /* _LRNG_DRNG_ATOMIC_H */ +diff --git a/drivers/char/lrng/lrng_drng_chacha20.c b/drivers/char/lrng/lrng_drng_chacha20.c +new file mode 100644 +index 000000000000..31be102e3007 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_chacha20.c +@@ -0,0 +1,195 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Backend for the LRNG providing the cryptographic primitives using ++ * ChaCha20 cipher implementations. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_drng_chacha20.h" ++ ++/******************************* ChaCha20 DRNG *******************************/ ++ ++#define CHACHA_BLOCK_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) ++ ++/* ++ * Update of the ChaCha20 state by either using an unused buffer part or by ++ * generating one ChaCha20 block which is half of the state of the ChaCha20. ++ * The block is XORed into the key part of the state. This shall ensure ++ * backtracking resistance as well as a proper mix of the ChaCha20 state once ++ * the key is injected. ++ */ ++static void lrng_chacha20_update(struct chacha20_state *chacha20_state, ++ __le32 *buf, u32 used_words) ++{ ++ struct chacha20_block *chacha20 = &chacha20_state->block; ++ u32 i; ++ __le32 tmp[CHACHA_BLOCK_WORDS]; ++ ++ BUILD_BUG_ON(sizeof(struct chacha20_block) != CHACHA_BLOCK_SIZE); ++ BUILD_BUG_ON(CHACHA_BLOCK_SIZE != 2 * CHACHA_KEY_SIZE); ++ ++ if (used_words > CHACHA_KEY_SIZE_WORDS) { ++ chacha20_block(&chacha20->constants[0], (u8 *)tmp); ++ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) ++ chacha20->key.u[i] ^= le32_to_cpu(tmp[i]); ++ memzero_explicit(tmp, sizeof(tmp)); ++ } else { ++ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) ++ chacha20->key.u[i] ^= le32_to_cpu(buf[i + used_words]); ++ } ++ ++ /* Deterministic increment of nonce as required in RFC 7539 chapter 4 */ ++ chacha20->nonce[0]++; ++ if (chacha20->nonce[0] == 0) { ++ chacha20->nonce[1]++; ++ if (chacha20->nonce[1] == 0) ++ chacha20->nonce[2]++; ++ } ++ ++ /* Leave counter untouched as it is start value is undefined in RFC */ ++} ++ ++/* ++ * Seed the ChaCha20 DRNG by injecting the input data into the key part of ++ * the ChaCha20 state. If the input data is longer than the ChaCha20 key size, ++ * perform a ChaCha20 operation after processing of key size input data. ++ * This operation shall spread out the entropy into the ChaCha20 state before ++ * new entropy is injected into the key part. ++ */ ++static int lrng_cc20_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) ++{ ++ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; ++ struct chacha20_block *chacha20 = &chacha20_state->block; ++ ++ while (inbuflen) { ++ u32 i, todo = min_t(u32, inbuflen, CHACHA_KEY_SIZE); ++ ++ for (i = 0; i < todo; i++) ++ chacha20->key.b[i] ^= inbuf[i]; ++ ++ /* Break potential dependencies between the inbuf key blocks */ ++ lrng_chacha20_update(chacha20_state, NULL, ++ CHACHA_BLOCK_WORDS); ++ inbuf += todo; ++ inbuflen -= todo; ++ } ++ ++ return 0; ++} ++ ++/* ++ * Chacha20 DRNG generation of random numbers: the stream output of ChaCha20 ++ * is the random number. After the completion of the generation of the ++ * stream, the entire ChaCha20 state is updated. ++ * ++ * Note, as the ChaCha20 implements a 32 bit counter, we must ensure ++ * that this function is only invoked for at most 2^32 - 1 ChaCha20 blocks ++ * before a reseed or an update happens. This is ensured by the variable ++ * outbuflen which is a 32 bit integer defining the number of bytes to be ++ * generated by the ChaCha20 DRNG. At the end of this function, an update ++ * operation is invoked which implies that the 32 bit counter will never be ++ * overflown in this implementation. ++ */ ++static int lrng_cc20_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) ++{ ++ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; ++ struct chacha20_block *chacha20 = &chacha20_state->block; ++ __le32 aligned_buf[CHACHA_BLOCK_WORDS]; ++ u32 ret = outbuflen, used = CHACHA_BLOCK_WORDS; ++ int zeroize_buf = 0; ++ ++ while (outbuflen >= CHACHA_BLOCK_SIZE) { ++ chacha20_block(&chacha20->constants[0], outbuf); ++ outbuf += CHACHA_BLOCK_SIZE; ++ outbuflen -= CHACHA_BLOCK_SIZE; ++ } ++ ++ if (outbuflen) { ++ chacha20_block(&chacha20->constants[0], (u8 *)aligned_buf); ++ memcpy(outbuf, aligned_buf, outbuflen); ++ used = ((outbuflen + sizeof(aligned_buf[0]) - 1) / ++ sizeof(aligned_buf[0])); ++ zeroize_buf = 1; ++ } ++ ++ lrng_chacha20_update(chacha20_state, aligned_buf, used); ++ ++ if (zeroize_buf) ++ memzero_explicit(aligned_buf, sizeof(aligned_buf)); ++ ++ return ret; ++} ++ ++/* ++ * Allocation of the DRNG state ++ */ ++static void *lrng_cc20_drng_alloc(u32 sec_strength) ++{ ++ struct chacha20_state *state = NULL; ++ ++ if (sec_strength > CHACHA_KEY_SIZE) { ++ pr_err("Security strength of ChaCha20 DRNG (%u bits) lower than requested by LRNG (%u bits)\n", ++ CHACHA_KEY_SIZE * 8, sec_strength * 8); ++ return ERR_PTR(-EINVAL); ++ } ++ if (sec_strength < CHACHA_KEY_SIZE) ++ pr_warn("Security strength of ChaCha20 DRNG (%u bits) higher than requested by LRNG (%u bits)\n", ++ CHACHA_KEY_SIZE * 8, sec_strength * 8); ++ ++ state = kmalloc(sizeof(struct chacha20_state), GFP_KERNEL); ++ if (!state) ++ return ERR_PTR(-ENOMEM); ++ pr_debug("memory for ChaCha20 core allocated\n"); ++ ++ lrng_cc20_init_rfc7539(&state->block); ++ ++ return state; ++} ++ ++static void lrng_cc20_drng_dealloc(void *drng) ++{ ++ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; ++ ++ pr_debug("ChaCha20 core zeroized and freed\n"); ++ kfree_sensitive(chacha20_state); ++} ++ ++static const char *lrng_cc20_drng_name(void) ++{ ++ return "ChaCha20 DRNG"; ++} ++ ++const struct lrng_drng_cb lrng_cc20_drng_cb = { ++ .drng_name = lrng_cc20_drng_name, ++ .drng_alloc = lrng_cc20_drng_alloc, ++ .drng_dealloc = lrng_cc20_drng_dealloc, ++ .drng_seed = lrng_cc20_drng_seed_helper, ++ .drng_generate = lrng_cc20_drng_generate_helper, ++}; ++ ++#if !defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) && \ ++ !defined(CONFIG_LRNG_DRNG_ATOMIC) ++static int __init lrng_cc20_drng_init(void) ++{ ++ return lrng_set_drng_cb(&lrng_cc20_drng_cb); ++} ++ ++static void __exit lrng_cc20_drng_exit(void) ++{ ++ lrng_set_drng_cb(NULL); ++} ++ ++late_initcall(lrng_cc20_drng_init); ++module_exit(lrng_cc20_drng_exit); ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_AUTHOR("Stephan Mueller "); ++MODULE_DESCRIPTION("Entropy Source and DRNG Manager - ChaCha20-based DRNG backend"); ++#endif /* CONFIG_LRNG_DFLT_DRNG_CHACHA20 */ +diff --git a/drivers/char/lrng/lrng_drng_chacha20.h b/drivers/char/lrng/lrng_drng_chacha20.h +new file mode 100644 +index 000000000000..fee6571281b6 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_chacha20.h +@@ -0,0 +1,42 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * LRNG ChaCha20 definitions ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_CHACHA20_H ++#define _LRNG_CHACHA20_H ++ ++#include ++ ++/* State according to RFC 7539 section 2.3 */ ++struct chacha20_block { ++ u32 constants[4]; ++ union { ++#define CHACHA_KEY_SIZE_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) ++ u32 u[CHACHA_KEY_SIZE_WORDS]; ++ u8 b[CHACHA_KEY_SIZE]; ++ } key; ++ u32 counter; ++ u32 nonce[3]; ++}; ++ ++struct chacha20_state { ++ struct chacha20_block block; ++}; ++ ++static inline void lrng_cc20_init_rfc7539(struct chacha20_block *chacha20) ++{ ++ chacha_init_consts(chacha20->constants); ++} ++ ++#define LRNG_CC20_INIT_RFC7539(x) \ ++ x.constants[0] = 0x61707865, \ ++ x.constants[1] = 0x3320646e, \ ++ x.constants[2] = 0x79622d32, \ ++ x.constants[3] = 0x6b206574, ++ ++extern const struct lrng_drng_cb lrng_cc20_drng_cb; ++ ++#endif /* _LRNG_CHACHA20_H */ +diff --git a/drivers/char/lrng/lrng_drng_drbg.c b/drivers/char/lrng/lrng_drng_drbg.c +new file mode 100644 +index 000000000000..1feec2c095a5 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_drbg.c +@@ -0,0 +1,179 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Backend for the LRNG providing the cryptographic primitives using the ++ * kernel crypto API and its DRBG. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_drng_drbg.h" ++ ++/* ++ * Define a DRBG plus a hash / MAC used to extract data from the entropy pool. ++ * For LRNG_HASH_NAME you can use a hash or a MAC (HMAC or CMAC) of your choice ++ * (Note, you should use the suggested selections below -- using SHA-1 or MD5 ++ * is not wise). The idea is that the used cipher primitive can be selected to ++ * be the same as used for the DRBG. I.e. the LRNG only uses one cipher ++ * primitive using the same cipher implementation with the options offered in ++ * the following. This means, if the CTR DRBG is selected and AES-NI is present, ++ * both the CTR DRBG and the selected cmac(aes) use AES-NI. ++ * ++ * The security strengths of the DRBGs are all 256 bits according to ++ * SP800-57 section 5.6.1. ++ * ++ * This definition is allowed to be changed. ++ */ ++#ifdef CONFIG_CRYPTO_DRBG_CTR ++static unsigned int lrng_drbg_type = 0; ++#elif defined CONFIG_CRYPTO_DRBG_HMAC ++static unsigned int lrng_drbg_type = 1; ++#elif defined CONFIG_CRYPTO_DRBG_HASH ++static unsigned int lrng_drbg_type = 2; ++#else ++#error "Unknown DRBG in use" ++#endif ++ ++/* The parameter must be r/o in sysfs as otherwise races appear. */ ++module_param(lrng_drbg_type, uint, 0444); ++MODULE_PARM_DESC(lrng_drbg_type, "DRBG type used for LRNG (0->CTR_DRBG, 1->HMAC_DRBG, 2->Hash_DRBG)"); ++ ++struct lrng_drbg { ++ const char *hash_name; ++ const char *drbg_core; ++}; ++ ++static const struct lrng_drbg lrng_drbg_types[] = { ++ { /* CTR_DRBG with AES-256 using derivation function */ ++ .drbg_core = "drbg_nopr_ctr_aes256", ++ }, { /* HMAC_DRBG with SHA-512 */ ++ .drbg_core = "drbg_nopr_hmac_sha512", ++ }, { /* Hash_DRBG with SHA-512 using derivation function */ ++ .drbg_core = "drbg_nopr_sha512" ++ } ++}; ++ ++static int lrng_drbg_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) ++{ ++ struct drbg_state *drbg = (struct drbg_state *)drng; ++ LIST_HEAD(seedlist); ++ struct drbg_string data; ++ int ret; ++ ++ drbg_string_fill(&data, inbuf, inbuflen); ++ list_add_tail(&data.list, &seedlist); ++ ret = drbg->d_ops->update(drbg, &seedlist, drbg->seeded); ++ ++ if (ret >= 0) ++ drbg->seeded = true; ++ ++ return ret; ++} ++ ++static int lrng_drbg_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) ++{ ++ struct drbg_state *drbg = (struct drbg_state *)drng; ++ ++ return drbg->d_ops->generate(drbg, outbuf, outbuflen, NULL); ++} ++ ++static void *lrng_drbg_drng_alloc(u32 sec_strength) ++{ ++ struct drbg_state *drbg; ++ int coreref = -1; ++ bool pr = false; ++ int ret; ++ ++ drbg_convert_tfm_core(lrng_drbg_types[lrng_drbg_type].drbg_core, ++ &coreref, &pr); ++ if (coreref < 0) ++ return ERR_PTR(-EFAULT); ++ ++ drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); ++ if (!drbg) ++ return ERR_PTR(-ENOMEM); ++ ++ drbg->core = &drbg_cores[coreref]; ++ drbg->seeded = false; ++ ret = drbg_alloc_state(drbg); ++ if (ret) ++ goto err; ++ ++ if (sec_strength > drbg_sec_strength(drbg->core->flags)) { ++ pr_err("Security strength of DRBG (%u bits) lower than requested by LRNG (%u bits)\n", ++ drbg_sec_strength(drbg->core->flags) * 8, ++ sec_strength * 8); ++ goto dealloc; ++ } ++ ++ if (sec_strength < drbg_sec_strength(drbg->core->flags)) ++ pr_warn("Security strength of DRBG (%u bits) higher than requested by LRNG (%u bits)\n", ++ drbg_sec_strength(drbg->core->flags) * 8, ++ sec_strength * 8); ++ ++ pr_info("DRBG with %s core allocated\n", drbg->core->backend_cra_name); ++ ++ return drbg; ++ ++dealloc: ++ if (drbg->d_ops) ++ drbg->d_ops->crypto_fini(drbg); ++ drbg_dealloc_state(drbg); ++err: ++ kfree(drbg); ++ return ERR_PTR(-EINVAL); ++} ++ ++static void lrng_drbg_drng_dealloc(void *drng) ++{ ++ struct drbg_state *drbg = (struct drbg_state *)drng; ++ ++ if (drbg && drbg->d_ops) ++ drbg->d_ops->crypto_fini(drbg); ++ drbg_dealloc_state(drbg); ++ kfree_sensitive(drbg); ++ pr_info("DRBG deallocated\n"); ++} ++ ++static const char *lrng_drbg_name(void) ++{ ++ return lrng_drbg_types[lrng_drbg_type].drbg_core; ++} ++ ++const struct lrng_drng_cb lrng_drbg_cb = { ++ .drng_name = lrng_drbg_name, ++ .drng_alloc = lrng_drbg_drng_alloc, ++ .drng_dealloc = lrng_drbg_drng_dealloc, ++ .drng_seed = lrng_drbg_drng_seed_helper, ++ .drng_generate = lrng_drbg_drng_generate_helper, ++}; ++ ++#ifndef CONFIG_LRNG_DFLT_DRNG_DRBG ++static int __init lrng_drbg_init(void) ++{ ++ if (lrng_drbg_type >= ARRAY_SIZE(lrng_drbg_types)) { ++ pr_err("lrng_drbg_type parameter too large (given %u - max: %lu)", ++ lrng_drbg_type, ++ (unsigned long)ARRAY_SIZE(lrng_drbg_types) - 1); ++ return -EAGAIN; ++ } ++ return lrng_set_drng_cb(&lrng_drbg_cb); ++} ++ ++static void __exit lrng_drbg_exit(void) ++{ ++ lrng_set_drng_cb(NULL); ++} ++ ++late_initcall(lrng_drbg_init); ++module_exit(lrng_drbg_exit); ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_AUTHOR("Stephan Mueller "); ++MODULE_DESCRIPTION("Entropy Source and DRNG Manager - SP800-90A DRBG backend"); ++#endif /* CONFIG_LRNG_DFLT_DRNG_DRBG */ +diff --git a/drivers/char/lrng/lrng_drng_drbg.h b/drivers/char/lrng/lrng_drng_drbg.h +new file mode 100644 +index 000000000000..b3d556caf294 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_drbg.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * LRNG SP800-90A definitions ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_DRBG_H ++#define _LRNG_DRBG_H ++ ++extern const struct lrng_drng_cb lrng_drbg_cb; ++ ++#endif /* _LRNG_DRBG_H */ +diff --git a/drivers/char/lrng/lrng_drng_kcapi.c b/drivers/char/lrng/lrng_drng_kcapi.c +new file mode 100644 +index 000000000000..a204bcf52a9a +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_kcapi.c +@@ -0,0 +1,208 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Backend for the LRNG providing the cryptographic primitives using the ++ * kernel crypto API. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "lrng_drng_kcapi.h" ++ ++static char *drng_name = NULL; ++module_param(drng_name, charp, 0444); ++MODULE_PARM_DESC(drng_name, "Kernel crypto API name of DRNG"); ++ ++static char *seed_hash = NULL; ++module_param(seed_hash, charp, 0444); ++MODULE_PARM_DESC(seed_hash, ++ "Kernel crypto API name of hash with output size equal to seedsize of DRNG to bring seed string to the size required by the DRNG"); ++ ++struct lrng_drng_info { ++ struct crypto_rng *kcapi_rng; ++ struct crypto_shash *hash_tfm; ++}; ++ ++static int lrng_kcapi_drng_seed_helper(void *drng, const u8 *inbuf, ++ u32 inbuflen) ++{ ++ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; ++ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; ++ struct crypto_shash *hash_tfm = lrng_drng_info->hash_tfm; ++ SHASH_DESC_ON_STACK(shash, hash_tfm); ++ u32 digestsize; ++ u8 digest[HASH_MAX_DIGESTSIZE] __aligned(8); ++ int ret; ++ ++ if (!hash_tfm) ++ return crypto_rng_reset(kcapi_rng, inbuf, inbuflen); ++ ++ shash->tfm = hash_tfm; ++ digestsize = crypto_shash_digestsize(hash_tfm); ++ ++ ret = crypto_shash_digest(shash, inbuf, inbuflen, digest); ++ shash_desc_zero(shash); ++ if (ret) ++ return ret; ++ ++ ret = crypto_rng_reset(kcapi_rng, digest, digestsize); ++ if (ret) ++ return ret; ++ ++ memzero_explicit(digest, digestsize); ++ return 0; ++} ++ ++static int lrng_kcapi_drng_generate_helper(void *drng, u8 *outbuf, ++ u32 outbuflen) ++{ ++ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; ++ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; ++ int ret = crypto_rng_get_bytes(kcapi_rng, outbuf, outbuflen); ++ ++ if (ret < 0) ++ return ret; ++ ++ return outbuflen; ++} ++ ++static void *lrng_kcapi_drng_alloc(u32 sec_strength) ++{ ++ struct lrng_drng_info *lrng_drng_info; ++ struct crypto_rng *kcapi_rng; ++ u32 time = random_get_entropy(); ++ int seedsize, rv; ++ void *ret = ERR_PTR(-ENOMEM); ++ ++ if (!drng_name) { ++ pr_err("DRNG name missing\n"); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ if (!memcmp(drng_name, "stdrng", 6) || ++ !memcmp(drng_name, "lrng", 4) || ++ !memcmp(drng_name, "drbg", 4) || ++ !memcmp(drng_name, "jitterentropy_rng", 17)) { ++ pr_err("Refusing to load the requested random number generator\n"); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ lrng_drng_info = kzalloc(sizeof(*lrng_drng_info), GFP_KERNEL); ++ if (!lrng_drng_info) ++ return ERR_PTR(-ENOMEM); ++ ++ kcapi_rng = crypto_alloc_rng(drng_name, 0, 0); ++ if (IS_ERR(kcapi_rng)) { ++ pr_err("DRNG %s cannot be allocated\n", drng_name); ++ ret = ERR_CAST(kcapi_rng); ++ goto free; ++ } ++ ++ lrng_drng_info->kcapi_rng = kcapi_rng; ++ ++ seedsize = crypto_rng_seedsize(kcapi_rng); ++ if (seedsize) { ++ struct crypto_shash *hash_tfm; ++ ++ if (!seed_hash) { ++ switch (seedsize) { ++ case 32: ++ seed_hash = "sha256"; ++ break; ++ case 48: ++ seed_hash = "sha384"; ++ break; ++ case 64: ++ seed_hash = "sha512"; ++ break; ++ default: ++ pr_err("Seed size %d cannot be processed\n", ++ seedsize); ++ goto dealloc; ++ } ++ } ++ ++ hash_tfm = crypto_alloc_shash(seed_hash, 0, 0); ++ if (IS_ERR(hash_tfm)) { ++ ret = ERR_CAST(hash_tfm); ++ goto dealloc; ++ } ++ ++ if (seedsize != crypto_shash_digestsize(hash_tfm)) { ++ pr_err("Seed hash output size not equal to DRNG seed size\n"); ++ crypto_free_shash(hash_tfm); ++ ret = ERR_PTR(-EINVAL); ++ goto dealloc; ++ } ++ ++ lrng_drng_info->hash_tfm = hash_tfm; ++ ++ pr_info("Seed hash %s allocated\n", seed_hash); ++ } ++ ++ rv = lrng_kcapi_drng_seed_helper(lrng_drng_info, (u8 *)(&time), ++ sizeof(time)); ++ if (rv) { ++ ret = ERR_PTR(rv); ++ goto dealloc; ++ } ++ ++ pr_info("Kernel crypto API DRNG %s allocated\n", drng_name); ++ ++ return lrng_drng_info; ++ ++dealloc: ++ crypto_free_rng(kcapi_rng); ++free: ++ kfree(lrng_drng_info); ++ return ret; ++} ++ ++static void lrng_kcapi_drng_dealloc(void *drng) ++{ ++ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; ++ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; ++ ++ crypto_free_rng(kcapi_rng); ++ if (lrng_drng_info->hash_tfm) ++ crypto_free_shash(lrng_drng_info->hash_tfm); ++ kfree(lrng_drng_info); ++ pr_info("DRNG %s deallocated\n", drng_name); ++} ++ ++static const char *lrng_kcapi_drng_name(void) ++{ ++ return drng_name; ++} ++ ++const struct lrng_drng_cb lrng_kcapi_drng_cb = { ++ .drng_name = lrng_kcapi_drng_name, ++ .drng_alloc = lrng_kcapi_drng_alloc, ++ .drng_dealloc = lrng_kcapi_drng_dealloc, ++ .drng_seed = lrng_kcapi_drng_seed_helper, ++ .drng_generate = lrng_kcapi_drng_generate_helper, ++}; ++ ++#ifndef CONFIG_LRNG_DFLT_DRNG_KCAPI ++static int __init lrng_kcapi_init(void) ++{ ++ return lrng_set_drng_cb(&lrng_kcapi_drng_cb); ++} ++static void __exit lrng_kcapi_exit(void) ++{ ++ lrng_set_drng_cb(NULL); ++} ++ ++late_initcall(lrng_kcapi_init); ++module_exit(lrng_kcapi_exit); ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_AUTHOR("Stephan Mueller "); ++MODULE_DESCRIPTION("Entropy Source and DRNG Manager - kernel crypto API DRNG backend"); ++#endif /* CONFIG_LRNG_DFLT_DRNG_KCAPI */ +diff --git a/drivers/char/lrng/lrng_drng_kcapi.h b/drivers/char/lrng/lrng_drng_kcapi.h +new file mode 100644 +index 000000000000..5db25aaf830c +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_kcapi.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * LRNG kernel crypto API DRNG definition ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_KCAPI_DRNG_H ++#define _LRNG_KCAPI_DRNG_H ++ ++extern const struct lrng_drng_cb lrng_kcapi_drng_cb; ++ ++#endif /* _LRNG_KCAPI_DRNG_H */ +diff --git a/drivers/char/lrng/lrng_drng_mgr.c b/drivers/char/lrng/lrng_drng_mgr.c +new file mode 100644 +index 000000000000..e9e08c186530 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_mgr.c +@@ -0,0 +1,657 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG DRNG management ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "lrng_drng_atomic.h" ++#include "lrng_drng_chacha20.h" ++#include "lrng_drng_drbg.h" ++#include "lrng_drng_kcapi.h" ++#include "lrng_drng_mgr.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_mgr.h" ++#include "lrng_interface_random_kernel.h" ++#include "lrng_numa.h" ++#include "lrng_sha.h" ++ ++/* ++ * Maximum number of seconds between DRNG reseed intervals of the DRNG. Note, ++ * this is enforced with the next request of random numbers from the ++ * DRNG. Setting this value to zero implies a reseeding attempt before every ++ * generated random number. ++ */ ++int lrng_drng_reseed_max_time = 600; ++ ++/* ++ * Is LRNG for general-purpose use (i.e. is at least the lrng_drng_init ++ * fully allocated)? ++ */ ++static atomic_t lrng_avail = ATOMIC_INIT(0); ++ ++/* Guard protecting all crypto callback update operation of all DRNGs. */ ++DEFINE_MUTEX(lrng_crypto_cb_update); ++ ++/* ++ * Default hash callback that provides the crypto primitive right from the ++ * kernel start. It must not perform any memory allocation operation, but ++ * simply perform the hash calculation. ++ */ ++const struct lrng_hash_cb *lrng_default_hash_cb = &lrng_sha_hash_cb; ++ ++/* ++ * Default DRNG callback that provides the crypto primitive which is ++ * allocated either during late kernel boot stage. So, it is permissible for ++ * the callback to perform memory allocation operations. ++ */ ++const struct lrng_drng_cb *lrng_default_drng_cb = ++#if defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) ++ &lrng_cc20_drng_cb; ++#elif defined(CONFIG_LRNG_DFLT_DRNG_DRBG) ++ &lrng_drbg_cb; ++#elif defined(CONFIG_LRNG_DFLT_DRNG_KCAPI) ++ &lrng_kcapi_drng_cb; ++#else ++#error "Unknown default DRNG selected" ++#endif ++ ++/* DRNG for non-atomic use cases */ ++static struct lrng_drng lrng_drng_init = { ++ LRNG_DRNG_STATE_INIT(lrng_drng_init, NULL, NULL, NULL, ++ &lrng_sha_hash_cb), ++ .lock = __MUTEX_INITIALIZER(lrng_drng_init.lock), ++}; ++ ++/* Prediction-resistance DRNG: only deliver as much data as received entropy */ ++static struct lrng_drng lrng_drng_pr = { ++ LRNG_DRNG_STATE_INIT(lrng_drng_pr, NULL, NULL, NULL, ++ &lrng_sha_hash_cb), ++ .lock = __MUTEX_INITIALIZER(lrng_drng_pr.lock), ++}; ++ ++static u32 max_wo_reseed = LRNG_DRNG_MAX_WITHOUT_RESEED; ++#ifdef CONFIG_LRNG_RUNTIME_MAX_WO_RESEED_CONFIG ++module_param(max_wo_reseed, uint, 0444); ++MODULE_PARM_DESC(max_wo_reseed, ++ "Maximum number of DRNG generate operation without full reseed\n"); ++#endif ++ ++/* Wait queue to wait until the LRNG is initialized - can freely be used */ ++DECLARE_WAIT_QUEUE_HEAD(lrng_init_wait); ++ ++/********************************** Helper ************************************/ ++ ++bool lrng_get_available(void) ++{ ++ return likely(atomic_read(&lrng_avail)); ++} ++ ++struct lrng_drng *lrng_drng_init_instance(void) ++{ ++ return &lrng_drng_init; ++} ++ ++struct lrng_drng *lrng_drng_pr_instance(void) ++{ ++ return &lrng_drng_pr; ++} ++ ++struct lrng_drng *lrng_drng_node_instance(void) ++{ ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ int node = numa_node_id(); ++ ++ if (lrng_drng && lrng_drng[node]) ++ return lrng_drng[node]; ++ ++ return lrng_drng_init_instance(); ++} ++ ++void lrng_drng_reset(struct lrng_drng *drng) ++{ ++ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); ++ atomic_set(&drng->requests_since_fully_seeded, 0); ++ drng->last_seeded = jiffies; ++ drng->fully_seeded = false; ++ drng->force_reseed = true; ++ pr_debug("reset DRNG\n"); ++} ++ ++/* Initialize the DRNG, except the mutex lock */ ++int lrng_drng_alloc_common(struct lrng_drng *drng, ++ const struct lrng_drng_cb *drng_cb) ++{ ++ if (!drng || !drng_cb) ++ return -EINVAL; ++ if (!IS_ERR_OR_NULL(drng->drng)) ++ return 0; ++ ++ drng->drng_cb = drng_cb; ++ drng->drng = drng_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); ++ if (IS_ERR(drng->drng)) ++ return PTR_ERR(drng->drng); ++ ++ lrng_drng_reset(drng); ++ return 0; ++} ++ ++/* Initialize the default DRNG during boot and perform its seeding */ ++int lrng_drng_initalize(void) ++{ ++ int ret; ++ ++ if (lrng_get_available()) ++ return 0; ++ ++ /* Catch programming error */ ++ WARN_ON(lrng_drng_init.hash_cb != lrng_default_hash_cb); ++ ++ mutex_lock(&lrng_drng_init.lock); ++ if (lrng_get_available()) { ++ mutex_unlock(&lrng_drng_init.lock); ++ return 0; ++ } ++ ++ /* Initialize the PR DRNG inside init lock as it guards lrng_avail. */ ++ mutex_lock(&lrng_drng_pr.lock); ++ ret = lrng_drng_alloc_common(&lrng_drng_pr, lrng_default_drng_cb); ++ mutex_unlock(&lrng_drng_pr.lock); ++ ++ if (!ret) { ++ ret = lrng_drng_alloc_common(&lrng_drng_init, ++ lrng_default_drng_cb); ++ if (!ret) ++ atomic_set(&lrng_avail, 1); ++ } ++ mutex_unlock(&lrng_drng_init.lock); ++ if (ret) ++ return ret; ++ ++ pr_debug("LRNG for general use is available\n"); ++ ++ /* Seed the DRNG with any entropy available */ ++ if (lrng_pool_trylock()) { ++ pr_info("Initial DRNG initialized triggering first seeding\n"); ++ lrng_drng_seed_work(NULL); ++ } else { ++ pr_info("Initial DRNG initialized without seeding\n"); ++ } ++ ++ return 0; ++} ++ ++static int __init lrng_drng_make_available(void) ++{ ++ return lrng_drng_initalize(); ++} ++late_initcall(lrng_drng_make_available); ++ ++bool lrng_sp80090c_compliant(void) ++{ ++ /* SP800-90C compliant oversampling is only requested in FIPS mode */ ++ return fips_enabled; ++} ++ ++bool lrng_ntg1_compliant(void) ++{ ++ /* Implies using of /dev/random with O_SYNC */ ++ return true; ++} ++ ++/************************* Random Number Generation ***************************/ ++ ++/* Inject a data buffer into the DRNG - caller must hold its lock */ ++void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, ++ bool fully_seeded, const char *drng_type) ++{ ++ BUILD_BUG_ON(LRNG_DRNG_RESEED_THRESH > INT_MAX); ++ pr_debug("seeding %s DRNG with %u bytes\n", drng_type, inbuflen); ++ if (drng->drng_cb->drng_seed(drng->drng, inbuf, inbuflen) < 0) { ++ pr_warn("seeding of %s DRNG failed\n", drng_type); ++ drng->force_reseed = true; ++ } else { ++ int gc = LRNG_DRNG_RESEED_THRESH - atomic_read(&drng->requests); ++ ++ pr_debug("%s DRNG stats since last seeding: %lu secs; generate calls: %d\n", ++ drng_type, ++ (time_after(jiffies, drng->last_seeded) ? ++ (jiffies - drng->last_seeded) : 0) / HZ, gc); ++ ++ /* Count the numbers of generate ops since last fully seeded */ ++ if (fully_seeded) ++ atomic_set(&drng->requests_since_fully_seeded, 0); ++ else ++ atomic_add(gc, &drng->requests_since_fully_seeded); ++ ++ drng->last_seeded = jiffies; ++ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); ++ drng->force_reseed = false; ++ ++ if (!drng->fully_seeded) { ++ drng->fully_seeded = fully_seeded; ++ if (drng->fully_seeded) ++ pr_debug("%s DRNG fully seeded\n", drng_type); ++ } ++ } ++} ++ ++/* ++ * Perform the seeding of the DRNG with data from entropy source. ++ * The function returns the entropy injected into the DRNG in bits. ++ */ ++static u32 lrng_drng_seed_es_nolock(struct lrng_drng *drng) ++{ ++ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); ++ u32 collected_entropy; ++ ++ lrng_fill_seed_buffer(&seedbuf, ++ lrng_get_seed_entropy_osr(drng->fully_seeded)); ++ ++ collected_entropy = lrng_entropy_rate_eb(&seedbuf); ++ lrng_drng_inject(drng, (u8 *)&seedbuf, sizeof(seedbuf), ++ lrng_fully_seeded(drng->fully_seeded, ++ collected_entropy), ++ "regular"); ++ ++ /* Set the seeding state of the LRNG */ ++ lrng_init_ops(&seedbuf); ++ ++ memzero_explicit(&seedbuf, sizeof(seedbuf)); ++ ++ return collected_entropy; ++} ++ ++static void lrng_drng_seed_es(struct lrng_drng *drng) ++{ ++ mutex_lock(&drng->lock); ++ lrng_drng_seed_es_nolock(drng); ++ mutex_unlock(&drng->lock); ++} ++ ++static void lrng_drng_seed(struct lrng_drng *drng) ++{ ++ BUILD_BUG_ON(LRNG_MIN_SEED_ENTROPY_BITS > ++ LRNG_DRNG_SECURITY_STRENGTH_BITS); ++ ++ if (lrng_get_available()) { ++ /* (Re-)Seed DRNG */ ++ lrng_drng_seed_es(drng); ++ /* (Re-)Seed atomic DRNG from regular DRNG */ ++ lrng_drng_atomic_seed_drng(drng); ++ } else { ++ /* ++ * If no-one is waiting for the DRNG, seed the atomic DRNG ++ * directly from the entropy sources. ++ */ ++ if (!wq_has_sleeper(&lrng_init_wait)) ++ lrng_drng_atomic_seed_es(); ++ else ++ lrng_init_ops(NULL); ++ } ++} ++ ++static void _lrng_drng_seed_work(struct lrng_drng *drng, u32 node) ++{ ++ pr_debug("reseed triggered by system events for DRNG on NUMA node %d\n", ++ node); ++ lrng_drng_seed(drng); ++ if (drng->fully_seeded) { ++ /* Prevent reseed storm */ ++ drng->last_seeded += node * 100 * HZ; ++ } ++} ++ ++/* ++ * DRNG reseed trigger: Kernel thread handler triggered by the schedule_work() ++ */ ++void lrng_drng_seed_work(struct work_struct *dummy) ++{ ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ u32 node; ++ ++ if (lrng_drng) { ++ for_each_online_node(node) { ++ struct lrng_drng *drng = lrng_drng[node]; ++ ++ if (drng && !drng->fully_seeded) { ++ _lrng_drng_seed_work(drng, node); ++ goto out; ++ } ++ } ++ } else { ++ if (!lrng_drng_init.fully_seeded) { ++ _lrng_drng_seed_work(&lrng_drng_init, 0); ++ goto out; ++ } ++ } ++ ++ if (!lrng_drng_pr.fully_seeded) { ++ _lrng_drng_seed_work(&lrng_drng_pr, 0); ++ goto out; ++ } ++ ++ lrng_pool_all_numa_nodes_seeded(true); ++ ++out: ++ /* Allow the seeding operation to be called again */ ++ lrng_pool_unlock(); ++} ++ ++/* Force all DRNGs to reseed before next generation */ ++void lrng_drng_force_reseed(void) ++{ ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ u32 node; ++ ++ /* ++ * If the initial DRNG is over the reseed threshold, allow a forced ++ * reseed only for the initial DRNG as this is the fallback for all. It ++ * must be kept seeded before all others to keep the LRNG operational. ++ */ ++ if (!lrng_drng || ++ (atomic_read_u32(&lrng_drng_init.requests_since_fully_seeded) > ++ LRNG_DRNG_RESEED_THRESH)) { ++ lrng_drng_init.force_reseed = lrng_drng_init.fully_seeded; ++ pr_debug("force reseed of initial DRNG\n"); ++ return; ++ } ++ for_each_online_node(node) { ++ struct lrng_drng *drng = lrng_drng[node]; ++ ++ if (!drng) ++ continue; ++ ++ drng->force_reseed = drng->fully_seeded; ++ pr_debug("force reseed of DRNG on node %u\n", node); ++ } ++ lrng_drng_atomic_force_reseed(); ++} ++EXPORT_SYMBOL(lrng_drng_force_reseed); ++ ++static bool lrng_drng_must_reseed(struct lrng_drng *drng) ++{ ++ return (atomic_dec_and_test(&drng->requests) || ++ drng->force_reseed || ++ time_after(jiffies, ++ drng->last_seeded + lrng_drng_reseed_max_time * HZ)); ++} ++ ++/* ++ * lrng_drng_get() - Get random data out of the DRNG which is reseeded ++ * frequently. ++ * ++ * @drng: DRNG instance ++ * @outbuf: buffer for storing random data ++ * @outbuflen: length of outbuf ++ * ++ * Return: ++ * * < 0 in error case (DRNG generation or update failed) ++ * * >=0 returning the returned number of bytes ++ */ ++int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen) ++{ ++ u32 processed = 0; ++ bool pr = (drng == &lrng_drng_pr) ? true : false; ++ ++ if (!outbuf || !outbuflen) ++ return 0; ++ ++ if (!lrng_get_available()) ++ return -EOPNOTSUPP; ++ ++ outbuflen = min_t(size_t, outbuflen, INT_MAX); ++ ++ /* If DRNG operated without proper reseed for too long, block LRNG */ ++ BUILD_BUG_ON(LRNG_DRNG_MAX_WITHOUT_RESEED < LRNG_DRNG_RESEED_THRESH); ++ if (atomic_read_u32(&drng->requests_since_fully_seeded) > max_wo_reseed) ++ lrng_unset_fully_seeded(drng); ++ ++ while (outbuflen) { ++ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); ++ int ret; ++ ++ /* In normal operation, check whether to reseed */ ++ if (!pr && lrng_drng_must_reseed(drng)) { ++ if (!lrng_pool_trylock()) { ++ drng->force_reseed = true; ++ } else { ++ lrng_drng_seed(drng); ++ lrng_pool_unlock(); ++ } ++ } ++ ++ mutex_lock(&drng->lock); ++ ++ if (pr) { ++ /* If async reseed did not deliver entropy, try now */ ++ if (!drng->fully_seeded) { ++ u32 coll_ent_bits; ++ ++ /* If we cannot get the pool lock, try again. */ ++ if (!lrng_pool_trylock()) { ++ mutex_unlock(&drng->lock); ++ continue; ++ } ++ ++ coll_ent_bits = lrng_drng_seed_es_nolock(drng); ++ ++ lrng_pool_unlock(); ++ ++ /* If no new entropy was received, stop now. */ ++ if (!coll_ent_bits) { ++ mutex_unlock(&drng->lock); ++ goto out; ++ } ++ ++ /* Produce no more data than received entropy */ ++ todo = min_t(u32, todo, coll_ent_bits >> 3); ++ } ++ ++ /* Do not produce more than DRNG security strength */ ++ todo = min_t(u32, todo, lrng_security_strength() >> 3); ++ } ++ ret = drng->drng_cb->drng_generate(drng->drng, ++ outbuf + processed, todo); ++ ++ mutex_unlock(&drng->lock); ++ if (ret <= 0) { ++ pr_warn("getting random data from DRNG failed (%d)\n", ++ ret); ++ return -EFAULT; ++ } ++ processed += ret; ++ outbuflen -= ret; ++ ++ if (pr) { ++ /* Force the async reseed for PR DRNG */ ++ lrng_unset_fully_seeded(drng); ++ if (outbuflen) ++ cond_resched(); ++ } ++ } ++ ++out: ++ return processed; ++} ++ ++int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr) ++{ ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ struct lrng_drng *drng = &lrng_drng_init; ++ int ret, node = numa_node_id(); ++ ++ might_sleep(); ++ ++ if (pr) ++ drng = &lrng_drng_pr; ++ else if (lrng_drng && lrng_drng[node] && lrng_drng[node]->fully_seeded) ++ drng = lrng_drng[node]; ++ ++ ret = lrng_drng_initalize(); ++ if (ret) ++ return ret; ++ ++ return lrng_drng_get(drng, outbuf, outbuflen); ++} ++ ++/* Reset LRNG such that all existing entropy is gone */ ++static void _lrng_reset(struct work_struct *work) ++{ ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ ++ if (!lrng_drng) { ++ mutex_lock(&lrng_drng_init.lock); ++ lrng_drng_reset(&lrng_drng_init); ++ mutex_unlock(&lrng_drng_init.lock); ++ } else { ++ u32 node; ++ ++ for_each_online_node(node) { ++ struct lrng_drng *drng = lrng_drng[node]; ++ ++ if (!drng) ++ continue; ++ mutex_lock(&drng->lock); ++ lrng_drng_reset(drng); ++ mutex_unlock(&drng->lock); ++ } ++ } ++ ++ mutex_lock(&lrng_drng_pr.lock); ++ lrng_drng_reset(&lrng_drng_pr); ++ mutex_unlock(&lrng_drng_pr.lock); ++ ++ lrng_drng_atomic_reset(); ++ lrng_set_entropy_thresh(LRNG_INIT_ENTROPY_BITS); ++ ++ lrng_reset_state(); ++} ++ ++static DECLARE_WORK(lrng_reset_work, _lrng_reset); ++ ++void lrng_reset(void) ++{ ++ schedule_work(&lrng_reset_work); ++} ++ ++/******************* Generic LRNG kernel output interfaces ********************/ ++ ++static int lrng_drng_sleep_while_not_all_nodes_seeded(unsigned int nonblock) ++{ ++ if (lrng_pool_all_numa_nodes_seeded_get()) ++ return 0; ++ if (nonblock) ++ return -EAGAIN; ++ wait_event_interruptible(lrng_init_wait, ++ lrng_pool_all_numa_nodes_seeded_get()); ++ return 0; ++} ++ ++int lrng_drng_sleep_while_nonoperational(int nonblock) ++{ ++ if (likely(lrng_state_operational())) ++ return 0; ++ if (nonblock) ++ return -EAGAIN; ++ return wait_event_interruptible(lrng_init_wait, ++ lrng_state_operational()); ++} ++ ++int lrng_drng_sleep_while_non_min_seeded(void) ++{ ++ if (likely(lrng_state_min_seeded())) ++ return 0; ++ return wait_event_interruptible(lrng_init_wait, ++ lrng_state_min_seeded()); ++} ++ ++ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags) ++{ ++ struct entropy_buf *eb = (struct entropy_buf *)(buf + 2); ++ u64 buflen = sizeof(struct entropy_buf) + 2 * sizeof(u64); ++ u64 collected_bits = 0; ++ int ret; ++ ++ /* Ensure buffer is aligned as required */ ++ BUILD_BUG_ON(sizeof(buflen) < LRNG_KCAPI_ALIGN); ++ if (nbytes < sizeof(buflen)) ++ return -EINVAL; ++ ++ /* Write buffer size into first word */ ++ buf[0] = buflen; ++ if (nbytes < buflen) ++ return -EMSGSIZE; ++ ++ ret = lrng_drng_sleep_while_not_all_nodes_seeded( ++ flags & LRNG_GET_SEED_NONBLOCK); ++ if (ret) ++ return ret; ++ ++ /* Try to get the pool lock and sleep on it to get it. */ ++ lrng_pool_lock(); ++ ++ /* If an LRNG DRNG becomes unseeded, give this DRNG precedence. */ ++ if (!lrng_pool_all_numa_nodes_seeded_get()) { ++ lrng_pool_unlock(); ++ return 0; ++ } ++ ++ /* ++ * Try to get seed data - a rarely used busyloop is cheaper than a wait ++ * queue that is constantly woken up by the hot code path of ++ * lrng_init_ops. ++ */ ++ for (;;) { ++ lrng_fill_seed_buffer(eb, ++ lrng_get_seed_entropy_osr(flags & ++ LRNG_GET_SEED_FULLY_SEEDED)); ++ collected_bits = lrng_entropy_rate_eb(eb); ++ ++ /* Break the collection loop if we got entropy, ... */ ++ if (collected_bits || ++ /* ... a DRNG becomes unseeded, give DRNG precedence, ... */ ++ !lrng_pool_all_numa_nodes_seeded_get() || ++ /* ... if the caller does not want a blocking behavior. */ ++ (flags & LRNG_GET_SEED_NONBLOCK)) ++ break; ++ ++ schedule(); ++ } ++ ++ lrng_pool_unlock(); ++ ++ /* Write collected entropy size into second word */ ++ buf[1] = collected_bits; ++ ++ return (ssize_t)buflen; ++} ++ ++void lrng_get_random_bytes_full(void *buf, int nbytes) ++{ ++ lrng_drng_sleep_while_nonoperational(0); ++ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); ++} ++EXPORT_SYMBOL(lrng_get_random_bytes_full); ++ ++void lrng_get_random_bytes_min(void *buf, int nbytes) ++{ ++ lrng_drng_sleep_while_non_min_seeded(); ++ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); ++} ++EXPORT_SYMBOL(lrng_get_random_bytes_min); ++ ++int lrng_get_random_bytes_pr(void *buf, int nbytes) ++{ ++ lrng_drng_sleep_while_nonoperational(0); ++ return lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, true); ++} ++EXPORT_SYMBOL(lrng_get_random_bytes_pr); +diff --git a/drivers/char/lrng/lrng_drng_mgr.h b/drivers/char/lrng/lrng_drng_mgr.h +new file mode 100644 +index 000000000000..ec120e4980b5 +--- /dev/null ++++ b/drivers/char/lrng/lrng_drng_mgr.h +@@ -0,0 +1,86 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_DRNG_H ++#define _LRNG_DRNG_H ++ ++#include ++#include ++#include ++ ++#include "lrng_definitions.h" ++ ++extern struct wait_queue_head lrng_init_wait; ++extern int lrng_drng_reseed_max_time; ++extern struct mutex lrng_crypto_cb_update; ++extern const struct lrng_drng_cb *lrng_default_drng_cb; ++extern const struct lrng_hash_cb *lrng_default_hash_cb; ++ ++/* DRNG state handle */ ++struct lrng_drng { ++ void *drng; /* DRNG handle */ ++ void *hash; /* Hash handle */ ++ const struct lrng_drng_cb *drng_cb; /* DRNG callbacks */ ++ const struct lrng_hash_cb *hash_cb; /* Hash callbacks */ ++ atomic_t requests; /* Number of DRNG requests */ ++ atomic_t requests_since_fully_seeded; /* Number DRNG requests since ++ * last fully seeded ++ */ ++ unsigned long last_seeded; /* Last time it was seeded */ ++ bool fully_seeded; /* Is DRNG fully seeded? */ ++ bool force_reseed; /* Force a reseed */ ++ ++ rwlock_t hash_lock; /* Lock hash_cb replacement */ ++ /* Lock write operations on DRNG state, DRNG replacement of drng_cb */ ++ struct mutex lock; /* Non-atomic DRNG operation */ ++ spinlock_t spin_lock; /* Atomic DRNG operation */ ++}; ++ ++#define LRNG_DRNG_STATE_INIT(x, d, h, d_cb, h_cb) \ ++ .drng = d, \ ++ .hash = h, \ ++ .drng_cb = d_cb, \ ++ .hash_cb = h_cb, \ ++ .requests = ATOMIC_INIT(LRNG_DRNG_RESEED_THRESH),\ ++ .requests_since_fully_seeded = ATOMIC_INIT(0), \ ++ .last_seeded = 0, \ ++ .fully_seeded = false, \ ++ .force_reseed = true, \ ++ .hash_lock = __RW_LOCK_UNLOCKED(x.hash_lock) ++ ++struct lrng_drng *lrng_drng_init_instance(void); ++struct lrng_drng *lrng_drng_pr_instance(void); ++struct lrng_drng *lrng_drng_node_instance(void); ++ ++void lrng_reset(void); ++int lrng_drng_alloc_common(struct lrng_drng *drng, ++ const struct lrng_drng_cb *crypto_cb); ++int lrng_drng_initalize(void); ++bool lrng_sp80090c_compliant(void); ++bool lrng_ntg1_compliant(void); ++bool lrng_get_available(void); ++void lrng_drng_reset(struct lrng_drng *drng); ++void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, ++ bool fully_seeded, const char *drng_type); ++int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen); ++int lrng_drng_sleep_while_nonoperational(int nonblock); ++int lrng_drng_sleep_while_non_min_seeded(void); ++int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr); ++void lrng_drng_seed_work(struct work_struct *dummy); ++void lrng_drng_force_reseed(void); ++ ++static inline u32 lrng_compress_osr(void) ++{ ++ return lrng_sp80090c_compliant() ? LRNG_OVERSAMPLE_ES_BITS : 0; ++} ++ ++static inline u32 lrng_reduce_by_osr(u32 entropy_bits) ++{ ++ u32 osr_bits = lrng_compress_osr(); ++ ++ return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0; ++} ++ ++#endif /* _LRNG_DRNG_H */ +diff --git a/drivers/char/lrng/lrng_es_aux.c b/drivers/char/lrng/lrng_es_aux.c +new file mode 100644 +index 000000000000..245bc829998b +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_aux.c +@@ -0,0 +1,335 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Slow Entropy Source: Auxiliary entropy pool ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++ ++#include "lrng_es_aux.h" ++#include "lrng_es_mgr.h" ++#include "lrng_sysctl.h" ++ ++/* ++ * This is the auxiliary pool ++ * ++ * The aux pool array is aligned to 8 bytes to comfort the kernel crypto API ++ * cipher implementations of the hash functions used to read the pool: for some ++ * accelerated implementations, we need an alignment to avoid a realignment ++ * which involves memcpy(). The alignment to 8 bytes should satisfy all crypto ++ * implementations. ++ */ ++struct lrng_pool { ++ u8 aux_pool[LRNG_POOL_SIZE]; /* Aux pool: digest state */ ++ atomic_t aux_entropy_bits; ++ atomic_t digestsize; /* Digest size of used hash */ ++ bool initialized; /* Aux pool initialized? */ ++ ++ /* Serialize read of entropy pool and update of aux pool */ ++ spinlock_t lock; ++}; ++ ++static struct lrng_pool lrng_pool __aligned(LRNG_KCAPI_ALIGN) = { ++ .aux_entropy_bits = ATOMIC_INIT(0), ++ .digestsize = ATOMIC_INIT(LRNG_ATOMIC_DIGEST_SIZE), ++ .initialized = false, ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_pool.lock) ++}; ++ ++/********************************** Helper ***********************************/ ++ ++/* Entropy in bits present in aux pool */ ++static u32 lrng_aux_avail_entropy(u32 __unused) ++{ ++ /* Cap available entropy with max entropy */ ++ u32 avail_bits = min_t(u32, lrng_get_digestsize(), ++ atomic_read_u32(&lrng_pool.aux_entropy_bits)); ++ ++ /* Consider oversampling rate due to aux pool conditioning */ ++ return lrng_reduce_by_osr(avail_bits); ++} ++ ++/* Set the digest size of the used hash in bytes */ ++static void lrng_set_digestsize(u32 digestsize) ++{ ++ struct lrng_pool *pool = &lrng_pool; ++ u32 ent_bits = atomic_xchg_relaxed(&pool->aux_entropy_bits, 0), ++ old_digestsize = lrng_get_digestsize(); ++ ++ atomic_set(&lrng_pool.digestsize, digestsize); ++ ++ /* ++ * Update the write wakeup threshold which must not be larger ++ * than the digest size of the current conditioning hash. ++ */ ++ digestsize = lrng_reduce_by_osr(digestsize << 3); ++ lrng_sysctl_update_max_write_thresh(digestsize); ++ lrng_write_wakeup_bits = digestsize; ++ ++ /* ++ * In case the new digest is larger than the old one, cap the available ++ * entropy to the old message digest used to process the existing data. ++ */ ++ ent_bits = min_t(u32, ent_bits, old_digestsize); ++ atomic_add(ent_bits, &pool->aux_entropy_bits); ++} ++ ++static int __init lrng_init_wakeup_bits(void) ++{ ++ u32 digestsize = lrng_reduce_by_osr(lrng_get_digestsize()); ++ ++ lrng_sysctl_update_max_write_thresh(digestsize); ++ lrng_write_wakeup_bits = digestsize; ++ return 0; ++} ++core_initcall(lrng_init_wakeup_bits); ++ ++/* Obtain the digest size provided by the used hash in bits */ ++u32 lrng_get_digestsize(void) ++{ ++ return atomic_read_u32(&lrng_pool.digestsize) << 3; ++} ++ ++/* Set entropy content in user-space controllable aux pool */ ++void lrng_pool_set_entropy(u32 entropy_bits) ++{ ++ atomic_set(&lrng_pool.aux_entropy_bits, entropy_bits); ++} ++ ++static void lrng_aux_reset(void) ++{ ++ lrng_pool_set_entropy(0); ++} ++ ++/* ++ * Replace old with new hash for auxiliary pool handling ++ * ++ * Assumption: the caller must guarantee that the new_cb is available during the ++ * entire operation (e.g. it must hold the write lock against pointer updating). ++ */ ++static int ++lrng_aux_switch_hash(struct lrng_drng *drng, int __unused, ++ const struct lrng_hash_cb *new_cb, void *new_hash, ++ const struct lrng_hash_cb *old_cb) ++{ ++ struct lrng_drng *init_drng = lrng_drng_init_instance(); ++ struct lrng_pool *pool = &lrng_pool; ++ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; ++ u8 digest[LRNG_MAX_DIGESTSIZE]; ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) ++ return -EOPNOTSUPP; ++ ++ if (unlikely(!pool->initialized)) ++ return 0; ++ ++ /* We only switch if the processed DRNG is the initial DRNG. */ ++ if (init_drng != drng) ++ return 0; ++ ++ /* Get the aux pool hash with old digest ... */ ++ ret = old_cb->hash_final(shash, digest) ?: ++ /* ... re-initialize the hash with the new digest ... */ ++ new_cb->hash_init(shash, new_hash) ?: ++ /* ++ * ... feed the old hash into the new state. We may feed ++ * uninitialized memory into the new state, but this is ++ * considered no issue and even good as we have some more ++ * uncertainty here. ++ */ ++ new_cb->hash_update(shash, digest, sizeof(digest)); ++ if (!ret) { ++ lrng_set_digestsize(new_cb->hash_digestsize(new_hash)); ++ pr_debug("Re-initialize aux entropy pool with hash %s\n", ++ new_cb->hash_name()); ++ } ++ ++ memzero_explicit(digest, sizeof(digest)); ++ return ret; ++} ++ ++/* Insert data into auxiliary pool by using the hash update function. */ ++static int ++lrng_aux_pool_insert_locked(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) ++{ ++ struct lrng_pool *pool = &lrng_pool; ++ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; ++ struct lrng_drng *drng = lrng_drng_init_instance(); ++ const struct lrng_hash_cb *hash_cb; ++ unsigned long flags; ++ void *hash; ++ int ret; ++ ++ entropy_bits = min_t(u32, entropy_bits, inbuflen << 3); ++ ++ read_lock_irqsave(&drng->hash_lock, flags); ++ hash_cb = drng->hash_cb; ++ hash = drng->hash; ++ ++ if (unlikely(!pool->initialized)) { ++ ret = hash_cb->hash_init(shash, hash); ++ if (ret) ++ goto out; ++ pool->initialized = true; ++ } ++ ++ ret = hash_cb->hash_update(shash, inbuf, inbuflen); ++ if (ret) ++ goto out; ++ ++ /* ++ * Cap the available entropy to the hash output size compliant to ++ * SP800-90B section 3.1.5.1 table 1. ++ */ ++ entropy_bits += atomic_read_u32(&pool->aux_entropy_bits); ++ atomic_set(&pool->aux_entropy_bits, ++ min_t(u32, entropy_bits, ++ hash_cb->hash_digestsize(hash) << 3)); ++ ++out: ++ read_unlock_irqrestore(&drng->hash_lock, flags); ++ return ret; ++} ++ ++int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) ++{ ++ struct lrng_pool *pool = &lrng_pool; ++ unsigned long flags; ++ int ret; ++ ++ spin_lock_irqsave(&pool->lock, flags); ++ ret = lrng_aux_pool_insert_locked(inbuf, inbuflen, entropy_bits); ++ spin_unlock_irqrestore(&pool->lock, flags); ++ ++ lrng_es_add_entropy(); ++ ++ return ret; ++} ++EXPORT_SYMBOL(lrng_pool_insert_aux); ++ ++/************************* Get data from entropy pool *************************/ ++ ++/* ++ * Get auxiliary entropy pool and its entropy content for seed buffer. ++ * Caller must hold lrng_pool.pool->lock. ++ * @outbuf: buffer to store data in with size requested_bits ++ * @requested_bits: Requested amount of entropy ++ * @return: amount of entropy in outbuf in bits. ++ */ ++static u32 lrng_aux_get_pool(u8 *outbuf, u32 requested_bits) ++{ ++ struct lrng_pool *pool = &lrng_pool; ++ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; ++ struct lrng_drng *drng = lrng_drng_init_instance(); ++ const struct lrng_hash_cb *hash_cb; ++ unsigned long flags; ++ void *hash; ++ u32 collected_ent_bits, returned_ent_bits, unused_bits = 0, ++ digestsize, digestsize_bits, requested_bits_osr; ++ u8 aux_output[LRNG_MAX_DIGESTSIZE]; ++ ++ if (unlikely(!pool->initialized)) ++ return 0; ++ ++ read_lock_irqsave(&drng->hash_lock, flags); ++ ++ hash_cb = drng->hash_cb; ++ hash = drng->hash; ++ digestsize = hash_cb->hash_digestsize(hash); ++ digestsize_bits = digestsize << 3; ++ ++ /* Cap to maximum entropy that can ever be generated with given hash */ ++ lrng_cap_requested(digestsize_bits, requested_bits); ++ ++ /* Ensure that no more than the size of aux_pool can be requested */ ++ requested_bits = min_t(u32, requested_bits, (LRNG_MAX_DIGESTSIZE << 3)); ++ requested_bits_osr = requested_bits + lrng_compress_osr(); ++ ++ /* Cap entropy with entropy counter from aux pool and the used digest */ ++ collected_ent_bits = min_t(u32, digestsize_bits, ++ atomic_xchg_relaxed(&pool->aux_entropy_bits, 0)); ++ ++ /* We collected too much entropy and put the overflow back */ ++ if (collected_ent_bits > requested_bits_osr) { ++ /* Amount of bits we collected too much */ ++ unused_bits = collected_ent_bits - requested_bits_osr; ++ /* Put entropy back */ ++ atomic_add(unused_bits, &pool->aux_entropy_bits); ++ /* Fix collected entropy */ ++ collected_ent_bits = requested_bits_osr; ++ } ++ ++ /* Apply oversampling: discount requested oversampling rate */ ++ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); ++ ++ pr_debug("obtained %u bits by collecting %u bits of entropy from aux pool, %u bits of entropy remaining\n", ++ returned_ent_bits, collected_ent_bits, unused_bits); ++ ++ /* Get the digest for the aux pool to be returned to the caller ... */ ++ if (hash_cb->hash_final(shash, aux_output) || ++ /* ++ * ... and re-initialize the aux state. Do not add the aux pool ++ * digest for backward secrecy as it will be added with the ++ * insertion of the complete seed buffer after it has been filled. ++ */ ++ hash_cb->hash_init(shash, hash)) { ++ returned_ent_bits = 0; ++ } else { ++ /* ++ * Do not truncate the output size exactly to collected_ent_bits ++ * as the aux pool may contain data that is not credited with ++ * entropy, but we want to use them to stir the DRNG state. ++ */ ++ memcpy(outbuf, aux_output, requested_bits >> 3); ++ } ++ ++ read_unlock_irqrestore(&drng->hash_lock, flags); ++ memzero_explicit(aux_output, digestsize); ++ return returned_ent_bits; ++} ++ ++static void lrng_aux_get_backtrack(struct entropy_buf *eb, u32 requested_bits, ++ bool __unused) ++{ ++ struct lrng_pool *pool = &lrng_pool; ++ unsigned long flags; ++ ++ /* Ensure aux pool extraction and backtracking op are atomic */ ++ spin_lock_irqsave(&pool->lock, flags); ++ ++ eb->e_bits[lrng_ext_es_aux] = lrng_aux_get_pool(eb->e[lrng_ext_es_aux], ++ requested_bits); ++ ++ /* Mix the extracted data back into pool for backtracking resistance */ ++ if (lrng_aux_pool_insert_locked((u8 *)eb, ++ sizeof(struct entropy_buf), 0)) ++ pr_warn("Backtracking resistance operation failed\n"); ++ ++ spin_unlock_irqrestore(&pool->lock, flags); ++} ++ ++static void lrng_aux_es_state(unsigned char *buf, size_t buflen) ++{ ++ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ ++ /* Assume the lrng_drng_init lock is taken by caller */ ++ snprintf(buf, buflen, ++ " Hash for operating entropy pool: %s\n" ++ " Available entropy: %u\n", ++ lrng_drng_init->hash_cb->hash_name(), ++ lrng_aux_avail_entropy(0)); ++} ++ ++struct lrng_es_cb lrng_es_aux = { ++ .name = "Auxiliary", ++ .get_ent = lrng_aux_get_backtrack, ++ .curr_entropy = lrng_aux_avail_entropy, ++ .max_entropy = lrng_get_digestsize, ++ .state = lrng_aux_es_state, ++ .reset = lrng_aux_reset, ++ .switch_hash = lrng_aux_switch_hash, ++}; +diff --git a/drivers/char/lrng/lrng_es_aux.h b/drivers/char/lrng/lrng_es_aux.h +new file mode 100644 +index 000000000000..bc41e6474aad +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_aux.h +@@ -0,0 +1,44 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_AUX_H ++#define _LRNG_ES_AUX_H ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_mgr_cb.h" ++ ++u32 lrng_get_digestsize(void); ++void lrng_pool_set_entropy(u32 entropy_bits); ++int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits); ++ ++extern struct lrng_es_cb lrng_es_aux; ++ ++/****************************** Helper code ***********************************/ ++ ++/* Obtain the security strength of the LRNG in bits */ ++static inline u32 lrng_security_strength(void) ++{ ++ /* ++ * We use a hash to read the entropy in the entropy pool. According to ++ * SP800-90B table 1, the entropy can be at most the digest size. ++ * Considering this together with the last sentence in section 3.1.5.1.2 ++ * the security strength of a (approved) hash is equal to its output ++ * size. On the other hand the entropy cannot be larger than the ++ * security strength of the used DRBG. ++ */ ++ return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize()); ++} ++ ++static inline u32 lrng_get_seed_entropy_osr(bool fully_seeded) ++{ ++ u32 requested_bits = lrng_security_strength(); ++ ++ /* Apply oversampling during initialization according to SP800-90C */ ++ if (lrng_sp80090c_compliant() && !fully_seeded) ++ requested_bits += LRNG_SEED_BUFFER_INIT_ADD_BITS; ++ return requested_bits; ++} ++ ++#endif /* _LRNG_ES_AUX_H */ +diff --git a/drivers/char/lrng/lrng_es_cpu.c b/drivers/char/lrng/lrng_es_cpu.c +new file mode 100644 +index 000000000000..f982cc31df4e +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_cpu.c +@@ -0,0 +1,280 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Fast Entropy Source: CPU-based entropy source ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_definitions.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_cpu.h" ++ ++/* ++ * Estimated entropy of data is a 32th of LRNG_DRNG_SECURITY_STRENGTH_BITS. ++ * As we have no ability to review the implementation of those noise sources, ++ * it is prudent to have a conservative estimate here. ++ */ ++#define LRNG_ARCHRANDOM_DEFAULT_STRENGTH CONFIG_LRNG_CPU_ENTROPY_RATE ++#define LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH LRNG_DRNG_SECURITY_STRENGTH_BITS ++#ifdef CONFIG_RANDOM_TRUST_CPU ++static u32 cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; ++#else ++static u32 cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; ++#endif ++#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG ++module_param(cpu_entropy, uint, 0644); ++MODULE_PARM_DESC(cpu_entropy, "Entropy in bits of 256 data bits from CPU noise source (e.g. RDSEED)"); ++#endif ++ ++static int __init lrng_parse_trust_cpu(char *arg) ++{ ++ int ret; ++ bool trust_cpu = false; ++ ++ ret = kstrtobool(arg, &trust_cpu); ++ if (ret) ++ return ret; ++ ++ if (trust_cpu) { ++ cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; ++ lrng_es_add_entropy(); ++ } else { ++ cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; ++ } ++ ++ return 0; ++} ++early_param("random.trust_cpu", lrng_parse_trust_cpu); ++ ++static u32 lrng_cpu_entropylevel(u32 requested_bits) ++{ ++ return lrng_fast_noise_entropylevel(cpu_entropy, requested_bits); ++} ++ ++static u32 lrng_cpu_poolsize(void) ++{ ++ return lrng_cpu_entropylevel(lrng_security_strength()); ++} ++ ++static u32 lrng_get_cpu_data(u8 *outbuf, u32 requested_bits) ++{ ++ size_t longs = 0; ++ u32 i, req = requested_bits >> 3; ++ ++ /* operate on full blocks */ ++ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BYTES % sizeof(unsigned long)); ++ BUILD_BUG_ON(LRNG_SEED_BUFFER_INIT_ADD_BITS % sizeof(unsigned long)); ++ /* ensure we have aligned buffers */ ++ BUILD_BUG_ON(LRNG_KCAPI_ALIGN % sizeof(unsigned long)); ++ ++ for (i = 0; i < req; i += longs) { ++ longs = arch_get_random_seed_longs( ++ (unsigned long *)(outbuf + i), req - i); ++ if (longs) ++ continue; ++ longs = arch_get_random_longs((unsigned long *)(outbuf + i), ++ req - i); ++ if (!longs) { ++ cpu_entropy = 0; ++ return 0; ++ } ++ } ++ ++ return requested_bits; ++} ++ ++static u32 lrng_get_cpu_data_compress(u8 *outbuf, u32 requested_bits, ++ u32 data_multiplier) ++{ ++ SHASH_DESC_ON_STACK(shash, NULL); ++ const struct lrng_hash_cb *hash_cb; ++ struct lrng_drng *drng = lrng_drng_node_instance(); ++ unsigned long flags; ++ u32 ent_bits = 0, i, partial_bits = 0, digestsize, digestsize_bits, ++ full_bits; ++ void *hash; ++ ++ read_lock_irqsave(&drng->hash_lock, flags); ++ hash_cb = drng->hash_cb; ++ hash = drng->hash; ++ ++ digestsize = hash_cb->hash_digestsize(hash); ++ digestsize_bits = digestsize << 3; ++ /* Cap to maximum entropy that can ever be generated with given hash */ ++ lrng_cap_requested(digestsize_bits, requested_bits); ++ full_bits = requested_bits * data_multiplier; ++ ++ /* Calculate oversampling for SP800-90C */ ++ if (lrng_sp80090c_compliant()) { ++ /* Complete amount of bits to be pulled */ ++ full_bits += LRNG_OVERSAMPLE_ES_BITS * data_multiplier; ++ /* Full blocks that will be pulled */ ++ data_multiplier = full_bits / requested_bits; ++ /* Partial block in bits to be pulled */ ++ partial_bits = full_bits - (data_multiplier * requested_bits); ++ } ++ ++ if (hash_cb->hash_init(shash, hash)) ++ goto out; ++ ++ /* Hash all data from the CPU entropy source */ ++ for (i = 0; i < data_multiplier; i++) { ++ ent_bits = lrng_get_cpu_data(outbuf, requested_bits); ++ if (!ent_bits) ++ goto out; ++ ++ if (hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) ++ goto err; ++ } ++ ++ /* Hash partial block, if applicable */ ++ ent_bits = lrng_get_cpu_data(outbuf, partial_bits); ++ if (ent_bits && ++ hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) ++ goto err; ++ ++ pr_debug("pulled %u bits from CPU RNG entropy source\n", full_bits); ++ ent_bits = requested_bits; ++ ++ /* Generate the compressed data to be returned to the caller */ ++ if (requested_bits < digestsize_bits) { ++ u8 digest[LRNG_MAX_DIGESTSIZE]; ++ ++ if (hash_cb->hash_final(shash, digest)) ++ goto err; ++ ++ /* Truncate output data to requested size */ ++ memcpy(outbuf, digest, requested_bits >> 3); ++ memzero_explicit(digest, digestsize); ++ } else { ++ if (hash_cb->hash_final(shash, outbuf)) ++ goto err; ++ } ++ ++out: ++ hash_cb->hash_desc_zero(shash); ++ read_unlock_irqrestore(&drng->hash_lock, flags); ++ return ent_bits; ++ ++err: ++ ent_bits = 0; ++ goto out; ++} ++ ++/* ++ * If CPU entropy source requires does not return full entropy, return the ++ * multiplier of how much data shall be sampled from it. ++ */ ++static u32 lrng_cpu_multiplier(void) ++{ ++ static u32 data_multiplier = 0; ++ unsigned long v; ++ ++ if (data_multiplier > 0) ++ return data_multiplier; ++ ++ if (IS_ENABLED(CONFIG_X86) && !arch_get_random_seed_longs(&v, 1)) { ++ /* ++ * Intel SPEC: pulling 512 blocks from RDRAND ensures ++ * one reseed making it logically equivalent to RDSEED. ++ */ ++ data_multiplier = 512; ++ } else if (IS_ENABLED(CONFIG_PPC)) { ++ /* ++ * PowerISA defines DARN to deliver at least 0.5 bits of ++ * entropy per data bit. ++ */ ++ data_multiplier = 2; ++ } else if (IS_ENABLED(CONFIG_RISCV)) { ++ /* ++ * riscv-crypto-spec-scalar-1.0.0-rc6.pdf section 4.2 defines ++ * this requirement. ++ */ ++ data_multiplier = 2; ++ } else { ++ /* CPU provides full entropy */ ++ data_multiplier = CONFIG_LRNG_CPU_FULL_ENT_MULTIPLIER; ++ } ++ return data_multiplier; ++} ++ ++static int ++lrng_cpu_switch_hash(struct lrng_drng *drng, int node, ++ const struct lrng_hash_cb *new_cb, void *new_hash, ++ const struct lrng_hash_cb *old_cb) ++{ ++ u32 digestsize, multiplier; ++ ++ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) ++ return -EOPNOTSUPP; ++ ++ digestsize = lrng_get_digestsize(); ++ multiplier = lrng_cpu_multiplier(); ++ ++ /* ++ * It would be security violation if the new digestsize is smaller than ++ * the set CPU entropy rate. ++ */ ++ WARN_ON(multiplier > 1 && digestsize < cpu_entropy); ++ cpu_entropy = min_t(u32, digestsize, cpu_entropy); ++ return 0; ++} ++ ++/* ++ * lrng_get_arch() - Get CPU entropy source entropy ++ * ++ * @eb: entropy buffer to store entropy ++ * @requested_bits: requested entropy in bits ++ */ ++static void lrng_cpu_get(struct entropy_buf *eb, u32 requested_bits, ++ bool __unused) ++{ ++ u32 ent_bits, data_multiplier = lrng_cpu_multiplier(); ++ ++ if (data_multiplier <= 1) { ++ ent_bits = lrng_get_cpu_data(eb->e[lrng_ext_es_cpu], ++ requested_bits); ++ } else { ++ ent_bits = lrng_get_cpu_data_compress(eb->e[lrng_ext_es_cpu], ++ requested_bits, ++ data_multiplier); ++ } ++ ++ ent_bits = lrng_cpu_entropylevel(ent_bits); ++ pr_debug("obtained %u bits of entropy from CPU RNG entropy source\n", ++ ent_bits); ++ eb->e_bits[lrng_ext_es_cpu] = ent_bits; ++} ++ ++static void lrng_cpu_es_state(unsigned char *buf, size_t buflen) ++{ ++ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ u32 data_multiplier = lrng_cpu_multiplier(); ++ ++ /* Assume the lrng_drng_init lock is taken by caller */ ++ snprintf(buf, buflen, ++ " Hash for compressing data: %s\n" ++ " Available entropy: %u\n" ++ " Data multiplier: %u\n", ++ (data_multiplier <= 1) ? ++ "N/A" : lrng_drng_init->hash_cb->hash_name(), ++ lrng_cpu_poolsize(), ++ data_multiplier); ++} ++ ++struct lrng_es_cb lrng_es_cpu = { ++ .name = "CPU", ++ .get_ent = lrng_cpu_get, ++ .curr_entropy = lrng_cpu_entropylevel, ++ .max_entropy = lrng_cpu_poolsize, ++ .state = lrng_cpu_es_state, ++ .reset = NULL, ++ .switch_hash = lrng_cpu_switch_hash, ++}; +diff --git a/drivers/char/lrng/lrng_es_cpu.h b/drivers/char/lrng/lrng_es_cpu.h +new file mode 100644 +index 000000000000..8dbb4d9a2926 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_cpu.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_CPU_H ++#define _LRNG_ES_CPU_H ++ ++#include "lrng_es_mgr_cb.h" ++ ++#ifdef CONFIG_LRNG_CPU ++ ++extern struct lrng_es_cb lrng_es_cpu; ++ ++#endif /* CONFIG_LRNG_CPU */ ++ ++#endif /* _LRNG_ES_CPU_H */ +diff --git a/drivers/char/lrng/lrng_es_irq.c b/drivers/char/lrng/lrng_es_irq.c +new file mode 100644 +index 000000000000..97c5c1d26447 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_irq.c +@@ -0,0 +1,727 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Slow Entropy Source: Interrupt data collection ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "lrng_es_aux.h" ++#include "lrng_es_irq.h" ++#include "lrng_es_timer_common.h" ++#include "lrng_health.h" ++#include "lrng_numa.h" ++#include "lrng_testing.h" ++ ++/* ++ * Number of interrupts to be recorded to assume that DRNG security strength ++ * bits of entropy are received. ++ * Note: a value below the DRNG security strength should not be defined as this ++ * may imply the DRNG can never be fully seeded in case other noise ++ * sources are unavailable. ++ */ ++#define LRNG_IRQ_ENTROPY_BITS LRNG_UINT32_C(CONFIG_LRNG_IRQ_ENTROPY_RATE) ++ ++ ++/* Number of interrupts required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ ++static u32 lrng_irq_entropy_bits = LRNG_IRQ_ENTROPY_BITS; ++ ++static u32 irq_entropy __read_mostly = LRNG_IRQ_ENTROPY_BITS; ++#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG ++module_param(irq_entropy, uint, 0444); ++MODULE_PARM_DESC(irq_entropy, ++ "How many interrupts must be collected for obtaining 256 bits of entropy\n"); ++#endif ++ ++/* Per-CPU array holding concatenated IRQ entropy events */ ++static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_irq_array) ++ __aligned(LRNG_KCAPI_ALIGN); ++static DEFINE_PER_CPU(u32, lrng_irq_array_ptr) = 0; ++static DEFINE_PER_CPU(atomic_t, lrng_irq_array_irqs) = ATOMIC_INIT(0); ++ ++/* ++ * The entropy collection is performed by executing the following steps: ++ * 1. fill up the per-CPU array holding the time stamps ++ * 2. once the per-CPU array is full, a compression of the data into ++ * the entropy pool is performed - this happens in interrupt context ++ * ++ * If step 2 is not desired in interrupt context, the following boolean ++ * needs to be set to false. This implies that old entropy data in the ++ * per-CPU array collected since the last DRNG reseed is overwritten with ++ * new entropy data instead of retaining the entropy with the compression ++ * operation. ++ * ++ * Impact on entropy: ++ * ++ * If continuous compression is enabled, the maximum entropy that is collected ++ * per CPU between DRNG reseeds is equal to the digest size of the used hash. ++ * ++ * If continuous compression is disabled, the maximum number of entropy events ++ * that can be collected per CPU is equal to LRNG_DATA_ARRAY_SIZE. This amount ++ * of events is converted into an entropy statement which then represents the ++ * maximum amount of entropy collectible per CPU between DRNG reseeds. ++ */ ++static bool lrng_irq_continuous_compression __read_mostly = ++ IS_ENABLED(CONFIG_LRNG_ENABLE_CONTINUOUS_COMPRESSION); ++ ++#ifdef CONFIG_LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION ++module_param(lrng_irq_continuous_compression, bool, 0444); ++MODULE_PARM_DESC(lrng_irq_continuous_compression, ++ "Perform entropy compression if per-CPU entropy data array is full\n"); ++#endif ++ ++/* ++ * Per-CPU entropy pool with compressed entropy event ++ * ++ * The per-CPU entropy pool is defined as the hash state. New data is simply ++ * inserted into the entropy pool by performing a hash update operation. ++ * To read the entropy pool, a hash final must be invoked. However, before ++ * the entropy pool is released again after a hash final, the hash init must ++ * be performed. ++ */ ++static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_irq_pool) ++ __aligned(LRNG_KCAPI_ALIGN); ++/* ++ * Lock to allow other CPUs to read the pool - as this is only done during ++ * reseed which is infrequent, this lock is hardly contended. ++ */ ++static DEFINE_PER_CPU(spinlock_t, lrng_irq_lock); ++static DEFINE_PER_CPU(bool, lrng_irq_lock_init) = false; ++ ++static bool lrng_irq_pool_online(int cpu) ++{ ++ return per_cpu(lrng_irq_lock_init, cpu); ++} ++ ++static void __init lrng_irq_check_compression_state(void) ++{ ++ /* One pool must hold sufficient entropy for disabled compression */ ++ if (!lrng_irq_continuous_compression) { ++ u32 max_ent = min_t(u32, lrng_get_digestsize(), ++ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, ++ lrng_irq_entropy_bits)); ++ if (max_ent < lrng_security_strength()) { ++ pr_warn("Force continuous compression operation to ensure LRNG can hold enough entropy\n"); ++ lrng_irq_continuous_compression = true; ++ } ++ } ++} ++ ++void __init lrng_irq_es_init(bool highres_timer) ++{ ++ /* Set a minimum number of interrupts that must be collected */ ++ irq_entropy = max_t(u32, LRNG_IRQ_ENTROPY_BITS, irq_entropy); ++ ++ if (highres_timer) { ++ lrng_irq_entropy_bits = irq_entropy; ++ } else { ++ u32 new_entropy = irq_entropy * LRNG_ES_OVERSAMPLING_FACTOR; ++ ++ lrng_irq_entropy_bits = (irq_entropy < new_entropy) ? ++ new_entropy : irq_entropy; ++ pr_warn("operating without high-resolution timer and applying IRQ oversampling factor %u\n", ++ LRNG_ES_OVERSAMPLING_FACTOR); ++ } ++ ++ lrng_irq_check_compression_state(); ++} ++ ++/* ++ * Reset all per-CPU pools - reset entropy estimator but leave the pool data ++ * that may or may not have entropy unchanged. ++ */ ++static void lrng_irq_reset(void) ++{ ++ int cpu; ++ ++ /* Trigger GCD calculation anew. */ ++ lrng_gcd_set(0); ++ ++ for_each_online_cpu(cpu) ++ atomic_set(per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); ++} ++ ++static u32 lrng_irq_avail_pool_size(void) ++{ ++ u32 max_size = 0, max_pool = lrng_get_digestsize(); ++ int cpu; ++ ++ if (!lrng_irq_continuous_compression) ++ max_pool = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); ++ ++ for_each_online_cpu(cpu) { ++ if (lrng_irq_pool_online(cpu)) ++ max_size += max_pool; ++ } ++ ++ return max_size; ++} ++ ++/* Return entropy of unused IRQs present in all per-CPU pools. */ ++static u32 lrng_irq_avail_entropy(u32 __unused) ++{ ++ u32 digestsize_irqs, irq = 0; ++ int cpu; ++ ++ /* Only deliver entropy when SP800-90B self test is completed */ ++ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) ++ return 0; ++ ++ /* Obtain the cap of maximum numbers of IRQs we count */ ++ digestsize_irqs = lrng_entropy_to_data(lrng_get_digestsize(), ++ lrng_irq_entropy_bits); ++ if (!lrng_irq_continuous_compression) { ++ /* Cap to max. number of IRQs the array can hold */ ++ digestsize_irqs = min_t(u32, digestsize_irqs, ++ LRNG_DATA_NUM_VALUES); ++ } ++ ++ for_each_online_cpu(cpu) { ++ if (!lrng_irq_pool_online(cpu)) ++ continue; ++ irq += min_t(u32, digestsize_irqs, ++ atomic_read_u32(per_cpu_ptr(&lrng_irq_array_irqs, ++ cpu))); ++ } ++ ++ /* Consider oversampling rate */ ++ return lrng_reduce_by_osr(lrng_data_to_entropy(irq, ++ lrng_irq_entropy_bits)); ++} ++ ++/* ++ * Trigger a switch of the hash implementation for the per-CPU pool. ++ * ++ * For each per-CPU pool, obtain the message digest with the old hash ++ * implementation, initialize the per-CPU pool again with the new hash ++ * implementation and inject the message digest into the new state. ++ * ++ * Assumption: the caller must guarantee that the new_cb is available during the ++ * entire operation (e.g. it must hold the lock against pointer updating). ++ */ ++static int ++lrng_irq_switch_hash(struct lrng_drng *drng, int node, ++ const struct lrng_hash_cb *new_cb, void *new_hash, ++ const struct lrng_hash_cb *old_cb) ++{ ++ u8 digest[LRNG_MAX_DIGESTSIZE]; ++ u32 digestsize_irqs, found_irqs; ++ int ret = 0, cpu; ++ ++ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) ++ return -EOPNOTSUPP; ++ ++ for_each_online_cpu(cpu) { ++ struct shash_desc *pcpu_shash; ++ ++ /* ++ * Only switch the per-CPU pools for the current node because ++ * the hash_cb only applies NUMA-node-wide. ++ */ ++ if (cpu_to_node(cpu) != node || !lrng_irq_pool_online(cpu)) ++ continue; ++ ++ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, ++ cpu); ++ ++ digestsize_irqs = old_cb->hash_digestsize(pcpu_shash); ++ digestsize_irqs = lrng_entropy_to_data(digestsize_irqs << 3, ++ lrng_irq_entropy_bits); ++ ++ if (pcpu_shash->tfm == new_hash) ++ continue; ++ ++ /* Get the per-CPU pool hash with old digest ... */ ++ ret = old_cb->hash_final(pcpu_shash, digest) ?: ++ /* ... re-initialize the hash with the new digest ... */ ++ new_cb->hash_init(pcpu_shash, new_hash) ?: ++ /* ++ * ... feed the old hash into the new state. We may feed ++ * uninitialized memory into the new state, but this is ++ * considered no issue and even good as we have some more ++ * uncertainty here. ++ */ ++ new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); ++ if (ret) ++ goto out; ++ ++ /* ++ * In case the new digest is larger than the old one, cap ++ * the available entropy to the old message digest used to ++ * process the existing data. ++ */ ++ found_irqs = atomic_xchg_relaxed( ++ per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); ++ found_irqs = min_t(u32, found_irqs, digestsize_irqs); ++ atomic_add_return_relaxed(found_irqs, ++ per_cpu_ptr(&lrng_irq_array_irqs, cpu)); ++ ++ pr_debug("Re-initialize per-CPU interrupt entropy pool for CPU %d on NUMA node %d with hash %s\n", ++ cpu, node, new_cb->hash_name()); ++ } ++ ++out: ++ memzero_explicit(digest, sizeof(digest)); ++ return ret; ++} ++ ++/* ++ * When reading the per-CPU message digest, make sure we use the crypto ++ * callbacks defined for the NUMA node the per-CPU pool is defined for because ++ * the LRNG crypto switch support is only atomic per NUMA node. ++ */ ++static u32 ++lrng_irq_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, ++ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) ++{ ++ struct shash_desc *pcpu_shash = ++ (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, cpu); ++ spinlock_t *lock = per_cpu_ptr(&lrng_irq_lock, cpu); ++ unsigned long flags; ++ u32 digestsize_irqs, found_irqs; ++ ++ /* Lock guarding against reading / writing to per-CPU pool */ ++ spin_lock_irqsave(lock, flags); ++ ++ *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); ++ digestsize_irqs = lrng_entropy_to_data(*digestsize << 3, ++ lrng_irq_entropy_bits); ++ ++ /* Obtain entropy statement like for the entropy pool */ ++ found_irqs = atomic_xchg_relaxed( ++ per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); ++ /* Cap to maximum amount of data we can hold in hash */ ++ found_irqs = min_t(u32, found_irqs, digestsize_irqs); ++ ++ /* Cap to maximum amount of data we can hold in array */ ++ if (!lrng_irq_continuous_compression) ++ found_irqs = min_t(u32, found_irqs, LRNG_DATA_NUM_VALUES); ++ ++ /* Store all not-yet compressed data in data array into hash, ... */ ++ if (pcpu_hash_cb->hash_update(pcpu_shash, ++ (u8 *)per_cpu_ptr(lrng_irq_array, cpu), ++ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: ++ /* ... get the per-CPU pool digest, ... */ ++ pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: ++ /* ... re-initialize the hash, ... */ ++ pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: ++ /* ... feed the old hash into the new state. */ ++ pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) ++ found_irqs = 0; ++ ++ spin_unlock_irqrestore(lock, flags); ++ return found_irqs; ++} ++ ++/* ++ * Hash all per-CPU pools and return the digest to be used as seed data for ++ * seeding a DRNG. The caller must guarantee backtracking resistance. ++ * The function will only copy as much data as entropy is available into the ++ * caller-provided output buffer. ++ * ++ * This function handles the translation from the number of received interrupts ++ * into an entropy statement. The conversion depends on LRNG_IRQ_ENTROPY_BITS ++ * which defines how many interrupts must be received to obtain 256 bits of ++ * entropy. With this value, the function lrng_data_to_entropy converts a given ++ * data size (received interrupts, requested amount of data, etc.) into an ++ * entropy statement. lrng_entropy_to_data does the reverse. ++ * ++ * @eb: entropy buffer to store entropy ++ * @requested_bits: Requested amount of entropy ++ * @fully_seeded: indicator whether LRNG is fully seeded ++ */ ++static void lrng_irq_pool_hash(struct entropy_buf *eb, u32 requested_bits, ++ bool fully_seeded) ++{ ++ SHASH_DESC_ON_STACK(shash, NULL); ++ const struct lrng_hash_cb *hash_cb; ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ struct lrng_drng *drng = lrng_drng_init_instance(); ++ u8 digest[LRNG_MAX_DIGESTSIZE]; ++ unsigned long flags, flags2; ++ u32 found_irqs, collected_irqs = 0, collected_ent_bits, requested_irqs, ++ returned_ent_bits; ++ int ret, cpu; ++ void *hash; ++ ++ /* Only deliver entropy when SP800-90B self test is completed */ ++ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) { ++ eb->e_bits[lrng_int_es_irq] = 0; ++ return; ++ } ++ ++ /* Lock guarding replacement of per-NUMA hash */ ++ read_lock_irqsave(&drng->hash_lock, flags); ++ ++ hash_cb = drng->hash_cb; ++ hash = drng->hash; ++ ++ /* The hash state of filled with all per-CPU pool hashes. */ ++ ret = hash_cb->hash_init(shash, hash); ++ if (ret) ++ goto err; ++ ++ /* Cap to maximum entropy that can ever be generated with given hash */ ++ lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); ++ requested_irqs = lrng_entropy_to_data(requested_bits + ++ lrng_compress_osr(), ++ lrng_irq_entropy_bits); ++ ++ /* ++ * Harvest entropy from each per-CPU hash state - even though we may ++ * have collected sufficient entropy, we will hash all per-CPU pools. ++ */ ++ for_each_online_cpu(cpu) { ++ struct lrng_drng *pcpu_drng = drng; ++ u32 digestsize, pcpu_unused_irqs = 0; ++ int node = cpu_to_node(cpu); ++ ++ /* If pool is not online, then no entropy is present. */ ++ if (!lrng_irq_pool_online(cpu)) ++ continue; ++ ++ if (lrng_drng && lrng_drng[node]) ++ pcpu_drng = lrng_drng[node]; ++ ++ if (pcpu_drng == drng) { ++ found_irqs = lrng_irq_pool_hash_one(hash_cb, hash, ++ cpu, digest, ++ &digestsize); ++ } else { ++ read_lock_irqsave(&pcpu_drng->hash_lock, flags2); ++ found_irqs = ++ lrng_irq_pool_hash_one(pcpu_drng->hash_cb, ++ pcpu_drng->hash, cpu, ++ digest, &digestsize); ++ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); ++ } ++ ++ /* Inject the digest into the state of all per-CPU pools */ ++ ret = hash_cb->hash_update(shash, digest, digestsize); ++ if (ret) ++ goto err; ++ ++ collected_irqs += found_irqs; ++ if (collected_irqs > requested_irqs) { ++ pcpu_unused_irqs = collected_irqs - requested_irqs; ++ atomic_add_return_relaxed(pcpu_unused_irqs, ++ per_cpu_ptr(&lrng_irq_array_irqs, cpu)); ++ collected_irqs = requested_irqs; ++ } ++ pr_debug("%u interrupts used from entropy pool of CPU %d, %u interrupts remain unused\n", ++ found_irqs - pcpu_unused_irqs, cpu, pcpu_unused_irqs); ++ } ++ ++ ret = hash_cb->hash_final(shash, digest); ++ if (ret) ++ goto err; ++ ++ collected_ent_bits = lrng_data_to_entropy(collected_irqs, ++ lrng_irq_entropy_bits); ++ /* Apply oversampling: discount requested oversampling rate */ ++ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); ++ ++ pr_debug("obtained %u bits by collecting %u bits of entropy from entropy pool noise source\n", ++ returned_ent_bits, collected_ent_bits); ++ ++ /* ++ * Truncate to available entropy as implicitly allowed by SP800-90B ++ * section 3.1.5.1.1 table 1 which awards truncated hashes full ++ * entropy. ++ * ++ * During boot time, we read requested_bits data with ++ * returned_ent_bits entropy. In case our conservative entropy ++ * estimate underestimates the available entropy we can transport as ++ * much available entropy as possible. ++ */ ++ memcpy(eb->e[lrng_int_es_irq], digest, ++ fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); ++ eb->e_bits[lrng_int_es_irq] = returned_ent_bits; ++ ++out: ++ hash_cb->hash_desc_zero(shash); ++ read_unlock_irqrestore(&drng->hash_lock, flags); ++ memzero_explicit(digest, sizeof(digest)); ++ return; ++ ++err: ++ eb->e_bits[lrng_int_es_irq] = 0; ++ goto out; ++} ++ ++/* Compress the lrng_irq_array array into lrng_irq_pool */ ++static void lrng_irq_array_compress(void) ++{ ++ struct shash_desc *shash = ++ (struct shash_desc *)this_cpu_ptr(lrng_irq_pool); ++ struct lrng_drng *drng = lrng_drng_node_instance(); ++ const struct lrng_hash_cb *hash_cb; ++ spinlock_t *lock = this_cpu_ptr(&lrng_irq_lock); ++ unsigned long flags, flags2; ++ void *hash; ++ bool init = false; ++ ++ read_lock_irqsave(&drng->hash_lock, flags); ++ hash_cb = drng->hash_cb; ++ hash = drng->hash; ++ ++ if (unlikely(!this_cpu_read(lrng_irq_lock_init))) { ++ init = true; ++ spin_lock_init(lock); ++ this_cpu_write(lrng_irq_lock_init, true); ++ pr_debug("Initializing per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n", ++ raw_smp_processor_id(), numa_node_id(), ++ hash_cb->hash_name()); ++ } ++ ++ spin_lock_irqsave(lock, flags2); ++ ++ if (unlikely(init) && hash_cb->hash_init(shash, hash)) { ++ this_cpu_write(lrng_irq_lock_init, false); ++ pr_warn("Initialization of hash failed\n"); ++ } else if (lrng_irq_continuous_compression) { ++ /* Add entire per-CPU data array content into entropy pool. */ ++ if (hash_cb->hash_update(shash, ++ (u8 *)this_cpu_ptr(lrng_irq_array), ++ LRNG_DATA_ARRAY_SIZE * sizeof(u32))) ++ pr_warn_ratelimited("Hashing of entropy data failed\n"); ++ } ++ ++ spin_unlock_irqrestore(lock, flags2); ++ read_unlock_irqrestore(&drng->hash_lock, flags); ++} ++ ++/* Compress data array into hash */ ++static void lrng_irq_array_to_hash(u32 ptr) ++{ ++ u32 *array = this_cpu_ptr(lrng_irq_array); ++ ++ /* ++ * During boot time the hash operation is triggered more often than ++ * during regular operation. ++ */ ++ if (unlikely(!lrng_state_fully_seeded())) { ++ if ((ptr & 31) && (ptr < LRNG_DATA_WORD_MASK)) ++ return; ++ } else if (ptr < LRNG_DATA_WORD_MASK) { ++ return; ++ } ++ ++ if (lrng_raw_array_entropy_store(*array)) { ++ u32 i; ++ ++ /* ++ * If we fed even a part of the array to external analysis, we ++ * mark that the entire array and the per-CPU pool to have no ++ * entropy. This is due to the non-IID property of the data as ++ * we do not fully know whether the existing dependencies ++ * diminish the entropy beyond to what we expect it has. ++ */ ++ atomic_set(this_cpu_ptr(&lrng_irq_array_irqs), 0); ++ ++ for (i = 1; i < LRNG_DATA_ARRAY_SIZE; i++) ++ lrng_raw_array_entropy_store(*(array + i)); ++ } else { ++ lrng_irq_array_compress(); ++ /* Ping pool handler about received entropy */ ++ if (lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) ++ lrng_es_add_entropy(); ++ } ++} ++ ++/* ++ * Concatenate full 32 bit word at the end of time array even when current ++ * ptr is not aligned to sizeof(data). ++ */ ++static void _lrng_irq_array_add_u32(u32 data) ++{ ++ /* Increment pointer by number of slots taken for input value */ ++ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_irq_array_ptr, ++ LRNG_DATA_SLOTS_PER_UINT); ++ unsigned int pre_array; ++ ++ /* ++ * This function injects a unit into the array - guarantee that ++ * array unit size is equal to data type of input data. ++ */ ++ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != (sizeof(data) << 3)); ++ ++ /* ++ * The following logic requires at least two units holding ++ * the data as otherwise the pointer would immediately wrap when ++ * injection an u32 word. ++ */ ++ BUILD_BUG_ON(LRNG_DATA_NUM_VALUES <= LRNG_DATA_SLOTS_PER_UINT); ++ ++ lrng_data_split_u32(&ptr, &pre_ptr, &mask); ++ ++ /* MSB of data go into previous unit */ ++ pre_array = lrng_data_idx2array(pre_ptr); ++ /* zeroization of slot to ensure the following OR adds the data */ ++ this_cpu_and(lrng_irq_array[pre_array], ~(0xffffffff & ~mask)); ++ this_cpu_or(lrng_irq_array[pre_array], data & ~mask); ++ ++ /* Invoke compression as we just filled data array completely */ ++ if (unlikely(pre_ptr > ptr)) ++ lrng_irq_array_to_hash(LRNG_DATA_WORD_MASK); ++ ++ /* LSB of data go into current unit */ ++ this_cpu_write(lrng_irq_array[lrng_data_idx2array(ptr)], ++ data & mask); ++ ++ if (likely(pre_ptr <= ptr)) ++ lrng_irq_array_to_hash(ptr); ++} ++ ++/* Concatenate a 32-bit word at the end of the per-CPU array */ ++void lrng_irq_array_add_u32(u32 data) ++{ ++ /* ++ * Disregard entropy-less data without continuous compression to ++ * avoid it overwriting data with entropy when array ptr wraps. ++ */ ++ if (lrng_irq_continuous_compression) ++ _lrng_irq_array_add_u32(data); ++} ++ ++/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ ++static void lrng_irq_array_add_slot(u32 data) ++{ ++ /* Get slot */ ++ u32 ptr = this_cpu_inc_return(lrng_irq_array_ptr) & ++ LRNG_DATA_WORD_MASK; ++ unsigned int array = lrng_data_idx2array(ptr); ++ unsigned int slot = lrng_data_idx2slot(ptr); ++ ++ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS % LRNG_DATA_SLOTSIZE_BITS); ++ /* Ensure consistency of values */ ++ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != ++ sizeof(lrng_irq_array[0]) << 3); ++ ++ /* zeroization of slot to ensure the following OR adds the data */ ++ this_cpu_and(lrng_irq_array[array], ++ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, ++ slot))); ++ /* Store data into slot */ ++ this_cpu_or(lrng_irq_array[array], lrng_data_slot_val(data, slot)); ++ ++ lrng_irq_array_to_hash(ptr); ++} ++ ++static void ++lrng_time_process_common(u32 time, void(*add_time)(u32 data)) ++{ ++ enum lrng_health_res health_test; ++ ++ if (lrng_raw_hires_entropy_store(time)) ++ return; ++ ++ health_test = lrng_health_test(time, lrng_int_es_irq); ++ if (health_test > lrng_health_fail_use) ++ return; ++ ++ if (health_test == lrng_health_pass) ++ atomic_inc_return(this_cpu_ptr(&lrng_irq_array_irqs)); ++ ++ add_time(time); ++} ++ ++/* ++ * Batching up of entropy in per-CPU array before injecting into entropy pool. ++ */ ++static void lrng_time_process(void) ++{ ++ u32 now_time = random_get_entropy(); ++ ++ if (unlikely(!lrng_gcd_tested())) { ++ /* When GCD is unknown, we process the full time stamp */ ++ lrng_time_process_common(now_time, _lrng_irq_array_add_u32); ++ lrng_gcd_add_value(now_time); ++ } else { ++ /* GCD is known and applied */ ++ lrng_time_process_common((now_time / lrng_gcd_get()) & ++ LRNG_DATA_SLOTSIZE_MASK, ++ lrng_irq_array_add_slot); ++ } ++ ++ lrng_perf_time(now_time); ++} ++ ++/* Hot code path - Callback for interrupt handler */ ++void add_interrupt_randomness(int irq) ++{ ++ if (lrng_highres_timer()) { ++ lrng_time_process(); ++ } else { ++ struct pt_regs *regs = get_irq_regs(); ++ static atomic_t reg_idx = ATOMIC_INIT(0); ++ u64 ip; ++ u32 tmp; ++ ++ if (regs) { ++ u32 *ptr = (u32 *)regs; ++ int reg_ptr = atomic_add_return_relaxed(1, ®_idx); ++ size_t n = (sizeof(struct pt_regs) / sizeof(u32)); ++ ++ ip = instruction_pointer(regs); ++ tmp = *(ptr + (reg_ptr % n)); ++ tmp = lrng_raw_regs_entropy_store(tmp) ? 0 : tmp; ++ _lrng_irq_array_add_u32(tmp); ++ } else { ++ ip = _RET_IP_; ++ } ++ ++ lrng_time_process(); ++ ++ /* ++ * The XOR operation combining the different values is not ++ * considered to destroy entropy since the entirety of all ++ * processed values delivers the entropy (and not each ++ * value separately of the other values). ++ */ ++ tmp = lrng_raw_jiffies_entropy_store(jiffies) ? 0 : jiffies; ++ tmp ^= lrng_raw_irq_entropy_store(irq) ? 0 : irq; ++ tmp ^= lrng_raw_retip_entropy_store(ip) ? 0 : ip; ++ tmp ^= ip >> 32; ++ _lrng_irq_array_add_u32(tmp); ++ } ++} ++EXPORT_SYMBOL(add_interrupt_randomness); ++ ++static void lrng_irq_es_state(unsigned char *buf, size_t buflen) ++{ ++ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ ++ /* Assume the lrng_drng_init lock is taken by caller */ ++ snprintf(buf, buflen, ++ " Hash for operating entropy pool: %s\n" ++ " Available entropy: %u\n" ++ " per-CPU interrupt collection size: %u\n" ++ " Standards compliance: %s\n" ++ " High-resolution timer: %s\n" ++ " Continuous compression: %s\n", ++ lrng_drng_init->hash_cb->hash_name(), ++ lrng_irq_avail_entropy(0), ++ LRNG_DATA_NUM_VALUES, ++ lrng_sp80090b_compliant(lrng_int_es_irq) ? "SP800-90B " : "", ++ lrng_highres_timer() ? "true" : "false", ++ lrng_irq_continuous_compression ? "true" : "false"); ++} ++ ++struct lrng_es_cb lrng_es_irq = { ++ .name = "IRQ", ++ .get_ent = lrng_irq_pool_hash, ++ .curr_entropy = lrng_irq_avail_entropy, ++ .max_entropy = lrng_irq_avail_pool_size, ++ .state = lrng_irq_es_state, ++ .reset = lrng_irq_reset, ++ .switch_hash = lrng_irq_switch_hash, ++}; +diff --git a/drivers/char/lrng/lrng_es_irq.h b/drivers/char/lrng/lrng_es_irq.h +new file mode 100644 +index 000000000000..2cd746611cf0 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_irq.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_IRQ_H ++#define _LRNG_ES_IRQ_H ++ ++#include ++ ++#include "lrng_es_mgr_cb.h" ++ ++#ifdef CONFIG_LRNG_IRQ ++void lrng_irq_es_init(bool highres_timer); ++void lrng_irq_array_add_u32(u32 data); ++ ++extern struct lrng_es_cb lrng_es_irq; ++ ++#else /* CONFIG_LRNG_IRQ */ ++static inline void lrng_irq_es_init(bool highres_timer) { } ++static inline void lrng_irq_array_add_u32(u32 data) { } ++#endif /* CONFIG_LRNG_IRQ */ ++ ++#endif /* _LRNG_ES_IRQ_H */ +diff --git a/drivers/char/lrng/lrng_es_jent.c b/drivers/char/lrng/lrng_es_jent.c +new file mode 100644 +index 000000000000..dbb2fce591f0 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_jent.c +@@ -0,0 +1,136 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Fast Entropy Source: Jitter RNG ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_definitions.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_jent.h" ++ ++/* ++ * Estimated entropy of data is a 16th of LRNG_DRNG_SECURITY_STRENGTH_BITS. ++ * Albeit a full entropy assessment is provided for the noise source indicating ++ * that it provides high entropy rates and considering that it deactivates ++ * when it detects insufficient hardware, the chosen under estimation of ++ * entropy is considered to be acceptable to all reviewers. ++ */ ++static u32 jent_entropy = CONFIG_LRNG_JENT_ENTROPY_RATE; ++#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG ++module_param(jent_entropy, uint, 0644); ++MODULE_PARM_DESC(jent_entropy, "Entropy in bits of 256 data bits from Jitter RNG noise source"); ++#endif ++ ++static bool lrng_jent_initialized = false; ++static struct rand_data *lrng_jent_state; ++ ++static int __init lrng_jent_initialize(void) ++{ ++ /* Initialize the Jitter RNG after the clocksources are initialized. */ ++ if (jent_entropy_init() || ++ (lrng_jent_state = jent_entropy_collector_alloc(1, 0)) == NULL) { ++ jent_entropy = 0; ++ pr_info("Jitter RNG unusable on current system\n"); ++ return 0; ++ } ++ lrng_jent_initialized = true; ++ pr_debug("Jitter RNG working on current system\n"); ++ ++ /* ++ * In FIPS mode, the Jitter RNG is defined to have full of entropy ++ * unless a different value has been specified at the command line ++ * (i.e. the user overrides the default), and the default value is ++ * larger than zero (if it is zero, it is assumed that an RBG2(P) or ++ * RBG2(NP) construction is attempted that intends to exclude the ++ * Jitter RNG). ++ */ ++ if (fips_enabled && ++ CONFIG_LRNG_JENT_ENTROPY_RATE > 0 && ++ jent_entropy == CONFIG_LRNG_JENT_ENTROPY_RATE) ++ jent_entropy = LRNG_DRNG_SECURITY_STRENGTH_BITS; ++ ++ lrng_drng_force_reseed(); ++ if (jent_entropy) ++ lrng_es_add_entropy(); ++ ++ return 0; ++} ++device_initcall(lrng_jent_initialize); ++ ++static u32 lrng_jent_entropylevel(u32 requested_bits) ++{ ++ return lrng_fast_noise_entropylevel(lrng_jent_initialized ? ++ jent_entropy : 0, requested_bits); ++} ++ ++static u32 lrng_jent_poolsize(void) ++{ ++ return lrng_jent_entropylevel(lrng_security_strength()); ++} ++ ++/* ++ * lrng_get_jent() - Get Jitter RNG entropy ++ * ++ * @eb: entropy buffer to store entropy ++ * @requested_bits: requested entropy in bits ++ */ ++static void lrng_jent_get(struct entropy_buf *eb, u32 requested_bits, ++ bool __unused) ++{ ++ int ret; ++ u32 ent_bits = lrng_jent_entropylevel(requested_bits); ++ unsigned long flags; ++ static DEFINE_SPINLOCK(lrng_jent_lock); ++ ++ spin_lock_irqsave(&lrng_jent_lock, flags); ++ ++ if (!lrng_jent_initialized) { ++ spin_unlock_irqrestore(&lrng_jent_lock, flags); ++ goto err; ++ } ++ ++ ret = jent_read_entropy(lrng_jent_state, eb->e[lrng_ext_es_jitter], ++ requested_bits >> 3); ++ spin_unlock_irqrestore(&lrng_jent_lock, flags); ++ ++ if (ret) { ++ pr_debug("Jitter RNG failed with %d\n", ret); ++ goto err; ++ } ++ ++ pr_debug("obtained %u bits of entropy from Jitter RNG noise source\n", ++ ent_bits); ++ ++ eb->e_bits[lrng_ext_es_jitter] = ent_bits; ++ return; ++ ++err: ++ eb->e_bits[lrng_ext_es_jitter] = 0; ++} ++ ++static void lrng_jent_es_state(unsigned char *buf, size_t buflen) ++{ ++ snprintf(buf, buflen, ++ " Available entropy: %u\n" ++ " Enabled: %s\n", ++ lrng_jent_poolsize(), ++ lrng_jent_initialized ? "true" : "false"); ++} ++ ++struct lrng_es_cb lrng_es_jent = { ++ .name = "JitterRNG", ++ .get_ent = lrng_jent_get, ++ .curr_entropy = lrng_jent_entropylevel, ++ .max_entropy = lrng_jent_poolsize, ++ .state = lrng_jent_es_state, ++ .reset = NULL, ++ .switch_hash = NULL, ++}; +diff --git a/drivers/char/lrng/lrng_es_jent.h b/drivers/char/lrng/lrng_es_jent.h +new file mode 100644 +index 000000000000..32882d4bdf99 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_jent.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_JENT_H ++#define _LRNG_ES_JENT_H ++ ++#include "lrng_es_mgr_cb.h" ++ ++#ifdef CONFIG_LRNG_JENT ++ ++extern struct lrng_es_cb lrng_es_jent; ++ ++#endif /* CONFIG_LRNG_JENT */ ++ ++#endif /* _LRNG_ES_JENT_H */ +diff --git a/drivers/char/lrng/lrng_es_krng.c b/drivers/char/lrng/lrng_es_krng.c +new file mode 100644 +index 000000000000..519ba640cc75 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_krng.c +@@ -0,0 +1,100 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Fast Entropy Source: Linux kernel RNG (random.c) ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_es_aux.h" ++#include "lrng_es_krng.h" ++ ++static u32 krng_entropy = CONFIG_LRNG_KERNEL_RNG_ENTROPY_RATE; ++#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG ++module_param(krng_entropy, uint, 0644); ++MODULE_PARM_DESC(krng_entropy, "Entropy in bits of 256 data bits from the kernel RNG noise source"); ++#endif ++ ++static atomic_t lrng_krng_initial_rate = ATOMIC_INIT(0); ++ ++static u32 lrng_krng_fips_entropylevel(u32 entropylevel) ++{ ++ return fips_enabled ? 0 : entropylevel; ++} ++ ++static int lrng_krng_adjust_entropy(void) ++{ ++ u32 entropylevel; ++ ++ krng_entropy = atomic_read_u32(&lrng_krng_initial_rate); ++ ++ entropylevel = lrng_krng_fips_entropylevel(krng_entropy); ++ pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", ++ entropylevel); ++ lrng_drng_force_reseed(); ++ if (entropylevel) ++ lrng_es_add_entropy(); ++ return 0; ++} ++ ++static u32 lrng_krng_entropylevel(u32 requested_bits) ++{ ++ static bool init = false; ++ ++ if (unlikely(!init) && rng_is_initialized()) { ++ init = true; ++ lrng_krng_adjust_entropy(); ++ } ++ ++ return lrng_fast_noise_entropylevel( ++ lrng_krng_fips_entropylevel(krng_entropy), requested_bits); ++} ++ ++static u32 lrng_krng_poolsize(void) ++{ ++ return lrng_krng_entropylevel(lrng_security_strength()); ++} ++ ++/* ++ * lrng_krng_get() - Get kernel RNG entropy ++ * ++ * @eb: entropy buffer to store entropy ++ * @requested_bits: requested entropy in bits ++ */ ++static void lrng_krng_get(struct entropy_buf *eb, u32 requested_bits, ++ bool __unused) ++{ ++ u32 ent_bits = lrng_krng_entropylevel(requested_bits); ++ ++ get_random_bytes(eb->e[lrng_ext_es_krng], requested_bits >> 3); ++ ++ pr_debug("obtained %u bits of entropy from kernel RNG noise source\n", ++ ent_bits); ++ ++ eb->e_bits[lrng_ext_es_krng] = ent_bits; ++} ++ ++static void lrng_krng_es_state(unsigned char *buf, size_t buflen) ++{ ++ snprintf(buf, buflen, ++ " Available entropy: %u\n" ++ " Entropy Rate per 256 data bits: %u\n", ++ lrng_krng_poolsize(), ++ lrng_krng_entropylevel(256)); ++} ++ ++struct lrng_es_cb lrng_es_krng = { ++ .name = "KernelRNG", ++ .get_ent = lrng_krng_get, ++ .curr_entropy = lrng_krng_entropylevel, ++ .max_entropy = lrng_krng_poolsize, ++ .state = lrng_krng_es_state, ++ .reset = NULL, ++ .switch_hash = NULL, ++}; +diff --git a/drivers/char/lrng/lrng_es_krng.h b/drivers/char/lrng/lrng_es_krng.h +new file mode 100644 +index 000000000000..cf982b9eea05 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_krng.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_RANDOM_H ++#define _LRNG_ES_RANDOM_H ++ ++#include "lrng_es_mgr_cb.h" ++ ++#ifdef CONFIG_LRNG_KERNEL_RNG ++ ++extern struct lrng_es_cb lrng_es_krng; ++ ++#endif /* CONFIG_LRNG_KERNEL_RNG */ ++ ++#endif /* _LRNG_ES_RANDOM_H */ +diff --git a/drivers/char/lrng/lrng_es_mgr.c b/drivers/char/lrng/lrng_es_mgr.c +new file mode 100644 +index 000000000000..23eba26b9f45 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_mgr.c +@@ -0,0 +1,460 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Entropy sources management ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_cpu.h" ++#include "lrng_es_irq.h" ++#include "lrng_es_jent.h" ++#include "lrng_es_krng.h" ++#include "lrng_es_mgr.h" ++#include "lrng_es_sched.h" ++#include "lrng_interface_dev_common.h" ++#include "lrng_interface_random_kernel.h" ++ ++struct lrng_state { ++ bool can_invalidate; /* Can invalidate batched entropy? */ ++ bool perform_seedwork; /* Can seed work be performed? */ ++ bool lrng_operational; /* Is DRNG operational? */ ++ bool lrng_fully_seeded; /* Is DRNG fully seeded? */ ++ bool lrng_min_seeded; /* Is DRNG minimally seeded? */ ++ bool all_online_numa_node_seeded;/* All NUMA DRNGs seeded? */ ++ ++ /* ++ * To ensure that external entropy providers cannot dominate the ++ * internal noise sources but yet cannot be dominated by internal ++ * noise sources, the following booleans are intended to allow ++ * external to provide seed once when a DRNG reseed occurs. This ++ * triggering of external noise source is performed even when the ++ * entropy pool has sufficient entropy. ++ */ ++ ++ atomic_t boot_entropy_thresh; /* Reseed threshold */ ++ struct mutex reseed_in_progress; /* Flag for on executing reseed */ ++ struct work_struct lrng_seed_work; /* (re)seed work queue */ ++}; ++ ++static struct lrng_state lrng_state = { ++ false, false, false, false, false, false, ++ .boot_entropy_thresh = ATOMIC_INIT(LRNG_INIT_ENTROPY_BITS), ++ .reseed_in_progress = ++ __MUTEX_INITIALIZER(lrng_state.reseed_in_progress), ++}; ++ ++/* ++ * If the entropy count falls under this number of bits, then we ++ * should wake up processes which are selecting or polling on write ++ * access to /dev/random. ++ */ ++u32 lrng_write_wakeup_bits = (LRNG_WRITE_WAKEUP_ENTROPY << 3); ++ ++/* ++ * The entries must be in the same order as defined by enum lrng_internal_es and ++ * enum lrng_external_es ++ */ ++struct lrng_es_cb *lrng_es[] = { ++#ifdef CONFIG_LRNG_IRQ ++ &lrng_es_irq, ++#endif ++#ifdef CONFIG_LRNG_SCHED ++ &lrng_es_sched, ++#endif ++#ifdef CONFIG_LRNG_JENT ++ &lrng_es_jent, ++#endif ++#ifdef CONFIG_LRNG_CPU ++ &lrng_es_cpu, ++#endif ++#ifdef CONFIG_LRNG_KERNEL_RNG ++ &lrng_es_krng, ++#endif ++ &lrng_es_aux ++}; ++ ++/********************************** Helper ***********************************/ ++ ++void lrng_debug_report_seedlevel(const char *name) ++{ ++#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM ++ static void *previous = NULL; ++ void *caller = (void *) _RET_IP_; ++ ++ if (READ_ONCE(previous) == caller) ++ return; ++ ++ if (!lrng_state_min_seeded()) ++ pr_notice("%pS %s called without reaching minimally seeded level (available entropy %u)\n", ++ caller, name, lrng_avail_entropy()); ++ ++ WRITE_ONCE(previous, caller); ++#endif ++} ++ ++/* ++ * Reading of the LRNG pool is only allowed by one caller. The reading is ++ * only performed to (re)seed DRNGs. Thus, if this "lock" is already taken, ++ * the reseeding operation is in progress. The caller is not intended to wait ++ * but continue with its other operation. ++ */ ++int lrng_pool_trylock(void) ++{ ++ return mutex_trylock(&lrng_state.reseed_in_progress); ++} ++ ++void lrng_pool_lock(void) ++{ ++ mutex_lock(&lrng_state.reseed_in_progress); ++} ++ ++void lrng_pool_unlock(void) ++{ ++ mutex_unlock(&lrng_state.reseed_in_progress); ++} ++ ++/* Set new entropy threshold for reseeding during boot */ ++void lrng_set_entropy_thresh(u32 new_entropy_bits) ++{ ++ atomic_set(&lrng_state.boot_entropy_thresh, new_entropy_bits); ++} ++ ++/* ++ * Reset LRNG state - the entropy counters are reset, but the data that may ++ * or may not have entropy remains in the pools as this data will not hurt. ++ */ ++void lrng_reset_state(void) ++{ ++ u32 i; ++ ++ for_each_lrng_es(i) { ++ if (lrng_es[i]->reset) ++ lrng_es[i]->reset(); ++ } ++ lrng_state.lrng_operational = false; ++ lrng_state.lrng_fully_seeded = false; ++ lrng_state.lrng_min_seeded = false; ++ lrng_state.all_online_numa_node_seeded = false; ++ pr_debug("reset LRNG\n"); ++} ++ ++/* Set flag that all DRNGs are fully seeded */ ++void lrng_pool_all_numa_nodes_seeded(bool set) ++{ ++ lrng_state.all_online_numa_node_seeded = set; ++ if (set) ++ wake_up_all(&lrng_init_wait); ++} ++ ++bool lrng_pool_all_numa_nodes_seeded_get(void) ++{ ++ return lrng_state.all_online_numa_node_seeded; ++} ++ ++/* Return boolean whether LRNG reached minimally seed level */ ++bool lrng_state_min_seeded(void) ++{ ++ return lrng_state.lrng_min_seeded; ++} ++ ++/* Return boolean whether LRNG reached fully seed level */ ++bool lrng_state_fully_seeded(void) ++{ ++ return lrng_state.lrng_fully_seeded; ++} ++ ++/* Return boolean whether LRNG is considered fully operational */ ++bool lrng_state_operational(void) ++{ ++ return lrng_state.lrng_operational; ++} ++ ++static void lrng_init_wakeup(void) ++{ ++ wake_up_all(&lrng_init_wait); ++ lrng_init_wakeup_dev(); ++} ++ ++bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy) ++{ ++ return (collected_entropy >= lrng_get_seed_entropy_osr(fully_seeded)); ++} ++ ++u32 lrng_entropy_rate_eb(struct entropy_buf *eb) ++{ ++ u32 i, collected_entropy = 0; ++ ++ for_each_lrng_es(i) ++ collected_entropy += eb->e_bits[i]; ++ ++ return collected_entropy; ++} ++ ++/* Mark one DRNG as not fully seeded */ ++void lrng_unset_fully_seeded(struct lrng_drng *drng) ++{ ++ drng->fully_seeded = false; ++ lrng_pool_all_numa_nodes_seeded(false); ++ ++ /* ++ * The init DRNG instance must always be fully seeded as this instance ++ * is the fall-back if any of the per-NUMA node DRNG instances is ++ * insufficiently seeded. Thus, we mark the entire LRNG as ++ * non-operational if the initial DRNG becomes not fully seeded. ++ */ ++ if (drng == lrng_drng_init_instance() && lrng_state_operational()) { ++ pr_debug("LRNG set to non-operational\n"); ++ lrng_state.lrng_operational = false; ++ lrng_state.lrng_fully_seeded = false; ++ ++ /* If sufficient entropy is available, reseed now. */ ++ lrng_es_add_entropy(); ++ } ++} ++ ++/* Policy to enable LRNG operational mode */ ++static void lrng_set_operational(void) ++{ ++ /* ++ * LRNG is operational if the initial DRNG is fully seeded. This state ++ * can only occur if either the external entropy sources provided ++ * sufficient entropy, or the SP800-90B startup test completed for ++ * the internal ES to supply also entropy data. ++ */ ++ if (lrng_state.lrng_fully_seeded) { ++ lrng_state.lrng_operational = true; ++ lrng_init_wakeup(); ++ pr_info("LRNG fully operational\n"); ++ } ++} ++ ++static u32 lrng_avail_entropy_thresh(void) ++{ ++ u32 ent_thresh = lrng_security_strength(); ++ ++ /* ++ * Apply oversampling during initialization according to SP800-90C as ++ * we request a larger buffer from the ES. ++ */ ++ if (lrng_sp80090c_compliant() && ++ !lrng_state.all_online_numa_node_seeded) ++ ent_thresh += LRNG_SEED_BUFFER_INIT_ADD_BITS; ++ ++ return ent_thresh; ++} ++ ++/* Available entropy in the entire LRNG considering all entropy sources */ ++u32 lrng_avail_entropy(void) ++{ ++ u32 i, ent = 0, ent_thresh = lrng_avail_entropy_thresh(); ++ ++ BUILD_BUG_ON(ARRAY_SIZE(lrng_es) != lrng_ext_es_last); ++ for_each_lrng_es(i) ++ ent += lrng_es[i]->curr_entropy(ent_thresh); ++ return ent; ++} ++ ++u32 lrng_avail_entropy_aux(void) ++{ ++ u32 ent_thresh = lrng_avail_entropy_thresh(); ++ ++ return lrng_es[lrng_ext_es_aux]->curr_entropy(ent_thresh); ++} ++ ++/* ++ * lrng_init_ops() - Set seed stages of LRNG ++ * ++ * Set the slow noise source reseed trigger threshold. The initial threshold ++ * is set to the minimum data size that can be read from the pool: a word. Upon ++ * reaching this value, the next seed threshold of 128 bits is set followed ++ * by 256 bits. ++ * ++ * @eb: buffer containing the size of entropy currently injected into DRNG - if ++ * NULL, the function obtains the available entropy from the ES. ++ */ ++void lrng_init_ops(struct entropy_buf *eb) ++{ ++ struct lrng_state *state = &lrng_state; ++ u32 i, requested_bits, seed_bits = 0; ++ ++ if (state->lrng_operational) ++ return; ++ ++ requested_bits = lrng_get_seed_entropy_osr( ++ state->all_online_numa_node_seeded); ++ ++ if (eb) { ++ for_each_lrng_es(i) ++ seed_bits += eb->e_bits[i]; ++ } else { ++ u32 ent_thresh = lrng_avail_entropy_thresh(); ++ ++ for_each_lrng_es(i) ++ seed_bits += lrng_es[i]->curr_entropy(ent_thresh); ++ } ++ ++ /* DRNG is seeded with full security strength */ ++ if (state->lrng_fully_seeded) { ++ lrng_set_operational(); ++ lrng_set_entropy_thresh(requested_bits); ++ } else if (lrng_fully_seeded(state->all_online_numa_node_seeded, ++ seed_bits)) { ++ if (state->can_invalidate) ++ invalidate_batched_entropy(); ++ ++ state->lrng_fully_seeded = true; ++ lrng_set_operational(); ++ state->lrng_min_seeded = true; ++ pr_info("LRNG fully seeded with %u bits of entropy\n", ++ seed_bits); ++ lrng_set_entropy_thresh(requested_bits); ++ } else if (!state->lrng_min_seeded) { ++ ++ /* DRNG is seeded with at least 128 bits of entropy */ ++ if (seed_bits >= LRNG_MIN_SEED_ENTROPY_BITS) { ++ if (state->can_invalidate) ++ invalidate_batched_entropy(); ++ ++ state->lrng_min_seeded = true; ++ pr_info("LRNG minimally seeded with %u bits of entropy\n", ++ seed_bits); ++ lrng_set_entropy_thresh(requested_bits); ++ lrng_init_wakeup(); ++ ++ /* DRNG is seeded with at least LRNG_INIT_ENTROPY_BITS bits */ ++ } else if (seed_bits >= LRNG_INIT_ENTROPY_BITS) { ++ pr_info("LRNG initial entropy level %u bits of entropy\n", ++ seed_bits); ++ lrng_set_entropy_thresh(LRNG_MIN_SEED_ENTROPY_BITS); ++ } ++ } ++} ++ ++int __init lrng_rand_initialize(void) ++{ ++ struct seed { ++ ktime_t time; ++ unsigned long data[((LRNG_MAX_DIGESTSIZE + ++ sizeof(unsigned long) - 1) / ++ sizeof(unsigned long))]; ++ struct new_utsname utsname; ++ } seed __aligned(LRNG_KCAPI_ALIGN); ++ size_t longs = 0; ++ unsigned int i; ++ ++ seed.time = ktime_get_real(); ++ ++ for (i = 0; i < ARRAY_SIZE(seed.data); i += longs) { ++#ifdef CONFIG_LRNG_RANDOM_IF ++ longs = arch_get_random_seed_longs_early( ++ seed.data + i, ARRAY_SIZE(seed.data) - i); ++ if (longs) ++ continue; ++ longs = arch_get_random_longs_early(seed.data + i, ++ ARRAY_SIZE(seed.data) - i); ++ if (longs) ++ continue; ++#else ++ longs = arch_get_random_seed_longs(seed.data + i, ++ ARRAY_SIZE(seed.data) - i); ++ if (longs) ++ continue; ++ longs = arch_get_random_longs(seed.data + i, ++ ARRAY_SIZE(seed.data) - i); ++ if (longs) ++ continue; ++#endif ++ seed.data[i] = random_get_entropy(); ++ longs = 1; ++ } ++ memcpy(&seed.utsname, utsname(), sizeof(*(utsname()))); ++ ++ lrng_pool_insert_aux((u8 *)&seed, sizeof(seed), 0); ++ memzero_explicit(&seed, sizeof(seed)); ++ ++ /* Initialize the seed work queue */ ++ INIT_WORK(&lrng_state.lrng_seed_work, lrng_drng_seed_work); ++ lrng_state.perform_seedwork = true; ++ ++ invalidate_batched_entropy(); ++ ++ lrng_state.can_invalidate = true; ++ ++ return 0; ++} ++ ++#ifndef CONFIG_LRNG_RANDOM_IF ++early_initcall(lrng_rand_initialize); ++#endif ++ ++/* Interface requesting a reseed of the DRNG */ ++void lrng_es_add_entropy(void) ++{ ++ /* ++ * Once all DRNGs are fully seeded, the system-triggered arrival of ++ * entropy will not cause any reseeding any more. ++ */ ++ if (likely(lrng_state.all_online_numa_node_seeded)) ++ return; ++ ++ /* Only trigger the DRNG reseed if we have collected entropy. */ ++ if (lrng_avail_entropy() < ++ atomic_read_u32(&lrng_state.boot_entropy_thresh)) ++ return; ++ ++ /* Ensure that the seeding only occurs once at any given time. */ ++ if (!lrng_pool_trylock()) ++ return; ++ ++ /* Seed the DRNG with any available noise. */ ++ if (lrng_state.perform_seedwork) ++ schedule_work(&lrng_state.lrng_seed_work); ++ else ++ lrng_drng_seed_work(NULL); ++} ++ ++/* Fill the seed buffer with data from the noise sources */ ++void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits) ++{ ++ struct lrng_state *state = &lrng_state; ++ u32 i, req_ent = lrng_sp80090c_compliant() ? ++ lrng_security_strength() : LRNG_MIN_SEED_ENTROPY_BITS; ++ ++ /* Guarantee that requested bits is a multiple of bytes */ ++ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BITS % 8); ++ ++ /* always reseed the DRNG with the current time stamp */ ++ eb->now = random_get_entropy(); ++ ++ /* ++ * Require at least 128 bits of entropy for any reseed. If the LRNG is ++ * operated SP800-90C compliant we want to comply with SP800-90A section ++ * 9.2 mandating that DRNG is reseeded with the security strength. ++ */ ++ if (state->lrng_fully_seeded && (lrng_avail_entropy() < req_ent)) { ++ for_each_lrng_es(i) ++ eb->e_bits[i] = 0; ++ ++ goto wakeup; ++ } ++ ++ /* Concatenate the output of the entropy sources. */ ++ for_each_lrng_es(i) { ++ lrng_es[i]->get_ent(eb, requested_bits, ++ state->lrng_fully_seeded); ++ } ++ ++ /* allow external entropy provider to provide seed */ ++ lrng_state_exseed_allow_all(); ++ ++wakeup: ++ lrng_writer_wakeup(); ++} +diff --git a/drivers/char/lrng/lrng_es_mgr.h b/drivers/char/lrng/lrng_es_mgr.h +new file mode 100644 +index 000000000000..f9bbd4863120 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_mgr.h +@@ -0,0 +1,51 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_MGR_H ++#define _LRNG_ES_MGR_H ++ ++#include "lrng_es_mgr_cb.h" ++ ++/*************************** General LRNG parameter ***************************/ ++ ++#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */ ++ ++/* Helper to concatenate a macro with an integer type */ ++#define LRNG_PASTER(x, y) x ## y ++#define LRNG_UINT32_C(x) LRNG_PASTER(x, U) ++ ++/************************* Entropy sources management *************************/ ++ ++extern struct lrng_es_cb *lrng_es[]; ++ ++#define for_each_lrng_es(ctr) \ ++ for ((ctr) = 0; (ctr) < lrng_ext_es_last; (ctr)++) ++ ++bool lrng_pool_all_numa_nodes_seeded_get(void); ++bool lrng_state_min_seeded(void); ++void lrng_debug_report_seedlevel(const char *name); ++int lrng_rand_initialize(void); ++bool lrng_state_operational(void); ++ ++extern u32 lrng_write_wakeup_bits; ++void lrng_set_entropy_thresh(u32 new); ++u32 lrng_avail_entropy(void); ++u32 lrng_avail_entropy_aux(void); ++void lrng_reset_state(void); ++ ++bool lrng_state_fully_seeded(void); ++ ++int lrng_pool_trylock(void); ++void lrng_pool_lock(void); ++void lrng_pool_unlock(void); ++void lrng_pool_all_numa_nodes_seeded(bool set); ++ ++bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy); ++u32 lrng_entropy_rate_eb(struct entropy_buf *eb); ++void lrng_unset_fully_seeded(struct lrng_drng *drng); ++void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits); ++void lrng_init_ops(struct entropy_buf *eb); ++ ++#endif /* _LRNG_ES_MGR_H */ +diff --git a/drivers/char/lrng/lrng_es_mgr_cb.h b/drivers/char/lrng/lrng_es_mgr_cb.h +new file mode 100644 +index 000000000000..08b24e1b7766 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_mgr_cb.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ * ++ * Definition of an entropy source. ++ */ ++ ++#ifndef _LRNG_ES_MGR_CB_H ++#define _LRNG_ES_MGR_CB_H ++ ++#include ++ ++#include "lrng_definitions.h" ++#include "lrng_drng_mgr.h" ++ ++enum lrng_internal_es { ++#ifdef CONFIG_LRNG_IRQ ++ lrng_int_es_irq, /* IRQ-based entropy source */ ++#endif ++#ifdef CONFIG_LRNG_SCHED ++ lrng_int_es_sched, /* Scheduler entropy source */ ++#endif ++ lrng_int_es_last, /* MUST be the last entry */ ++}; ++ ++enum lrng_external_es { ++ lrng_ext_link = lrng_int_es_last - 1, /* Link entry */ ++#ifdef CONFIG_LRNG_JENT ++ lrng_ext_es_jitter, /* Jitter RNG */ ++#endif ++#ifdef CONFIG_LRNG_CPU ++ lrng_ext_es_cpu, /* CPU-based, e.g. RDSEED */ ++#endif ++#ifdef CONFIG_LRNG_KERNEL_RNG ++ lrng_ext_es_krng, /* random.c */ ++#endif ++ lrng_ext_es_aux, /* MUST BE LAST ES! */ ++ lrng_ext_es_last /* MUST be the last entry */ ++}; ++ ++struct entropy_buf { ++ u8 e[lrng_ext_es_last][LRNG_DRNG_INIT_SEED_SIZE_BYTES]; ++ u32 now, e_bits[lrng_ext_es_last]; ++}; ++ ++/* ++ * struct lrng_es_cb - callback defining an entropy source ++ * @name: Name of the entropy source. ++ * @get_ent: Fetch entropy into the entropy_buf. The ES shall only deliver ++ * data if its internal initialization is complete, including any ++ * SP800-90B startup testing or similar. ++ * @curr_entropy: Return amount of currently available entropy. ++ * @max_entropy: Maximum amount of entropy the entropy source is able to ++ * maintain. ++ * @state: Buffer with human-readable ES state. ++ * @reset: Reset entropy source (drop all entropy and reinitialize). ++ * This callback may be NULL. ++ * @switch_hash: callback to switch from an old hash callback definition to ++ * a new one. This callback may be NULL. ++ */ ++struct lrng_es_cb { ++ const char *name; ++ void (*get_ent)(struct entropy_buf *eb, u32 requested_bits, ++ bool fully_seeded); ++ u32 (*curr_entropy)(u32 requested_bits); ++ u32 (*max_entropy)(void); ++ void (*state)(unsigned char *buf, size_t buflen); ++ void (*reset)(void); ++ int (*switch_hash)(struct lrng_drng *drng, int node, ++ const struct lrng_hash_cb *new_cb, void *new_hash, ++ const struct lrng_hash_cb *old_cb); ++}; ++ ++/* Allow entropy sources to tell the ES manager that new entropy is there */ ++void lrng_es_add_entropy(void); ++ ++/* Cap to maximum entropy that can ever be generated with given hash */ ++#define lrng_cap_requested(__digestsize_bits, __requested_bits) \ ++ do { \ ++ if (__digestsize_bits < __requested_bits) { \ ++ pr_debug("Cannot satisfy requested entropy %u due to insufficient hash size %u\n",\ ++ __requested_bits, __digestsize_bits); \ ++ __requested_bits = __digestsize_bits; \ ++ } \ ++ } while (0) ++ ++#endif /* _LRNG_ES_MGR_CB_H */ +diff --git a/drivers/char/lrng/lrng_es_sched.c b/drivers/char/lrng/lrng_es_sched.c +new file mode 100644 +index 000000000000..c3abaa7ff310 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_sched.c +@@ -0,0 +1,562 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Slow Entropy Source: Scheduler-based data collection ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "lrng_es_aux.h" ++#include "lrng_es_sched.h" ++#include "lrng_es_timer_common.h" ++#include "lrng_health.h" ++#include "lrng_numa.h" ++#include "lrng_testing.h" ++ ++/* ++ * Number of scheduler-based context switches to be recorded to assume that ++ * DRNG security strength bits of entropy are received. ++ * Note: a value below the DRNG security strength should not be defined as this ++ * may imply the DRNG can never be fully seeded in case other noise ++ * sources are unavailable. ++ */ ++#define LRNG_SCHED_ENTROPY_BITS \ ++ LRNG_UINT32_C(CONFIG_LRNG_SCHED_ENTROPY_RATE) ++ ++/* Number of events required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ ++static u32 lrng_sched_entropy_bits = LRNG_SCHED_ENTROPY_BITS; ++ ++static u32 sched_entropy __read_mostly = LRNG_SCHED_ENTROPY_BITS; ++#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG ++module_param(sched_entropy, uint, 0444); ++MODULE_PARM_DESC(sched_entropy, ++ "How many scheduler-based context switches must be collected for obtaining 256 bits of entropy\n"); ++#endif ++ ++/* Per-CPU array holding concatenated entropy events */ ++static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_sched_array) ++ __aligned(LRNG_KCAPI_ALIGN); ++static DEFINE_PER_CPU(u32, lrng_sched_array_ptr) = 0; ++static DEFINE_PER_CPU(atomic_t, lrng_sched_array_events) = ATOMIC_INIT(0); ++ ++/* ++ * Per-CPU entropy pool with compressed entropy event ++ * ++ * The per-CPU entropy pool is defined as the hash state. New data is simply ++ * inserted into the entropy pool by performing a hash update operation. ++ * To read the entropy pool, a hash final must be invoked. However, before ++ * the entropy pool is released again after a hash final, the hash init must ++ * be performed. ++ */ ++static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_sched_pool) ++ __aligned(LRNG_KCAPI_ALIGN); ++/* ++ * Lock to allow other CPUs to read the pool - as this is only done during ++ * reseed which is infrequent, this lock is hardly contended. ++ */ ++static DEFINE_PER_CPU(spinlock_t, lrng_sched_lock); ++static DEFINE_PER_CPU(bool, lrng_sched_lock_init) = false; ++ ++static bool lrng_sched_pool_online(int cpu) ++{ ++ return per_cpu(lrng_sched_lock_init, cpu); ++} ++ ++static void __init lrng_sched_check_compression_state(void) ++{ ++ /* One pool should hold sufficient entropy for disabled compression */ ++ u32 max_ent = min_t(u32, lrng_get_digestsize(), ++ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, ++ lrng_sched_entropy_bits)); ++ if (max_ent < lrng_security_strength()) { ++ pr_devel("Scheduler entropy source will never provide %u bits of entropy required for fully seeding the DRNG all by itself\n", ++ lrng_security_strength()); ++ } ++} ++ ++void __init lrng_sched_es_init(bool highres_timer) ++{ ++ /* Set a minimum number of scheduler events that must be collected */ ++ sched_entropy = max_t(u32, LRNG_SCHED_ENTROPY_BITS, sched_entropy); ++ ++ if (highres_timer) { ++ lrng_sched_entropy_bits = sched_entropy; ++ } else { ++ u32 new_entropy = sched_entropy * LRNG_ES_OVERSAMPLING_FACTOR; ++ ++ lrng_sched_entropy_bits = (sched_entropy < new_entropy) ? ++ new_entropy : sched_entropy; ++ pr_warn("operating without high-resolution timer and applying oversampling factor %u\n", ++ LRNG_ES_OVERSAMPLING_FACTOR); ++ } ++ ++ lrng_sched_check_compression_state(); ++} ++ ++static u32 lrng_sched_avail_pool_size(void) ++{ ++ u32 max_pool = lrng_get_digestsize(), ++ max_size = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); ++ int cpu; ++ ++ for_each_online_cpu(cpu) ++ max_size += max_pool; ++ ++ return max_size; ++} ++ ++/* Return entropy of unused scheduler events present in all per-CPU pools. */ ++static u32 lrng_sched_avail_entropy(u32 __unused) ++{ ++ u32 digestsize_events, events = 0; ++ int cpu; ++ ++ /* Only deliver entropy when SP800-90B self test is completed */ ++ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) ++ return 0; ++ ++ /* Obtain the cap of maximum numbers of scheduler events we count */ ++ digestsize_events = lrng_entropy_to_data(lrng_get_digestsize(), ++ lrng_sched_entropy_bits); ++ /* Cap to max. number of scheduler events the array can hold */ ++ digestsize_events = min_t(u32, digestsize_events, LRNG_DATA_NUM_VALUES); ++ ++ for_each_online_cpu(cpu) { ++ events += min_t(u32, digestsize_events, ++ atomic_read_u32(per_cpu_ptr(&lrng_sched_array_events, ++ cpu))); ++ } ++ ++ /* Consider oversampling rate */ ++ return lrng_reduce_by_osr( ++ lrng_data_to_entropy(events, lrng_sched_entropy_bits)); ++} ++ ++/* ++ * Reset all per-CPU pools - reset entropy estimator but leave the pool data ++ * that may or may not have entropy unchanged. ++ */ ++static void lrng_sched_reset(void) ++{ ++ int cpu; ++ ++ /* Trigger GCD calculation anew. */ ++ lrng_gcd_set(0); ++ ++ for_each_online_cpu(cpu) ++ atomic_set(per_cpu_ptr(&lrng_sched_array_events, cpu), 0); ++} ++ ++/* ++ * Trigger a switch of the hash implementation for the per-CPU pool. ++ * ++ * For each per-CPU pool, obtain the message digest with the old hash ++ * implementation, initialize the per-CPU pool again with the new hash ++ * implementation and inject the message digest into the new state. ++ * ++ * Assumption: the caller must guarantee that the new_cb is available during the ++ * entire operation (e.g. it must hold the lock against pointer updating). ++ */ ++static int ++lrng_sched_switch_hash(struct lrng_drng *drng, int node, ++ const struct lrng_hash_cb *new_cb, void *new_hash, ++ const struct lrng_hash_cb *old_cb) ++{ ++ u8 digest[LRNG_MAX_DIGESTSIZE]; ++ u32 digestsize_events, found_events; ++ int ret = 0, cpu; ++ ++ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) ++ return -EOPNOTSUPP; ++ ++ for_each_online_cpu(cpu) { ++ struct shash_desc *pcpu_shash; ++ ++ /* ++ * Only switch the per-CPU pools for the current node because ++ * the hash_cb only applies NUMA-node-wide. ++ */ ++ if (cpu_to_node(cpu) != node || !lrng_sched_pool_online(cpu)) ++ continue; ++ ++ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, ++ cpu); ++ ++ digestsize_events = old_cb->hash_digestsize(pcpu_shash); ++ digestsize_events = lrng_entropy_to_data(digestsize_events << 3, ++ lrng_sched_entropy_bits); ++ ++ if (pcpu_shash->tfm == new_hash) ++ continue; ++ ++ /* Get the per-CPU pool hash with old digest ... */ ++ ret = old_cb->hash_final(pcpu_shash, digest) ?: ++ /* ... re-initialize the hash with the new digest ... */ ++ new_cb->hash_init(pcpu_shash, new_hash) ?: ++ /* ++ * ... feed the old hash into the new state. We may feed ++ * uninitialized memory into the new state, but this is ++ * considered no issue and even good as we have some more ++ * uncertainty here. ++ */ ++ new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); ++ if (ret) ++ goto out; ++ ++ /* ++ * In case the new digest is larger than the old one, cap ++ * the available entropy to the old message digest used to ++ * process the existing data. ++ */ ++ found_events = atomic_xchg_relaxed( ++ per_cpu_ptr(&lrng_sched_array_events, cpu), 0); ++ found_events = min_t(u32, found_events, digestsize_events); ++ atomic_add_return_relaxed(found_events, ++ per_cpu_ptr(&lrng_sched_array_events, cpu)); ++ ++ pr_debug("Re-initialize per-CPU scheduler entropy pool for CPU %d on NUMA node %d with hash %s\n", ++ cpu, node, new_cb->hash_name()); ++ } ++ ++out: ++ memzero_explicit(digest, sizeof(digest)); ++ return ret; ++} ++ ++static u32 ++lrng_sched_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, ++ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) ++{ ++ struct shash_desc *pcpu_shash = ++ (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, cpu); ++ spinlock_t *lock = per_cpu_ptr(&lrng_sched_lock, cpu); ++ unsigned long flags; ++ u32 digestsize_events, found_events; ++ ++ if (unlikely(!per_cpu(lrng_sched_lock_init, cpu))) { ++ if (pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash)) { ++ pr_warn("Initialization of hash failed\n"); ++ return 0; ++ } ++ spin_lock_init(lock); ++ per_cpu(lrng_sched_lock_init, cpu) = true; ++ pr_debug("Initializing per-CPU scheduler entropy pool for CPU %d with hash %s\n", ++ raw_smp_processor_id(), pcpu_hash_cb->hash_name()); ++ } ++ ++ /* Lock guarding against reading / writing to per-CPU pool */ ++ spin_lock_irqsave(lock, flags); ++ ++ *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); ++ digestsize_events = lrng_entropy_to_data(*digestsize << 3, ++ lrng_sched_entropy_bits); ++ ++ /* Obtain entropy statement like for the entropy pool */ ++ found_events = atomic_xchg_relaxed( ++ per_cpu_ptr(&lrng_sched_array_events, cpu), 0); ++ /* Cap to maximum amount of data we can hold in hash */ ++ found_events = min_t(u32, found_events, digestsize_events); ++ ++ /* Cap to maximum amount of data we can hold in array */ ++ found_events = min_t(u32, found_events, LRNG_DATA_NUM_VALUES); ++ ++ /* Store all not-yet compressed data in data array into hash, ... */ ++ if (pcpu_hash_cb->hash_update(pcpu_shash, ++ (u8 *)per_cpu_ptr(lrng_sched_array, cpu), ++ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: ++ /* ... get the per-CPU pool digest, ... */ ++ pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: ++ /* ... re-initialize the hash, ... */ ++ pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: ++ /* ... feed the old hash into the new state. */ ++ pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) ++ found_events = 0; ++ ++ spin_unlock_irqrestore(lock, flags); ++ return found_events; ++} ++ ++/* ++ * Hash all per-CPU arrays and return the digest to be used as seed data for ++ * seeding a DRNG. The caller must guarantee backtracking resistance. ++ * The function will only copy as much data as entropy is available into the ++ * caller-provided output buffer. ++ * ++ * This function handles the translation from the number of received scheduler ++ * events into an entropy statement. The conversion depends on ++ * LRNG_SCHED_ENTROPY_BITS which defines how many scheduler events must be ++ * received to obtain 256 bits of entropy. With this value, the function ++ * lrng_data_to_entropy converts a given data size (received scheduler events, ++ * requested amount of data, etc.) into an entropy statement. ++ * lrng_entropy_to_data does the reverse. ++ * ++ * @eb: entropy buffer to store entropy ++ * @requested_bits: Requested amount of entropy ++ * @fully_seeded: indicator whether LRNG is fully seeded ++ */ ++static void lrng_sched_pool_hash(struct entropy_buf *eb, u32 requested_bits, ++ bool fully_seeded) ++{ ++ SHASH_DESC_ON_STACK(shash, NULL); ++ const struct lrng_hash_cb *hash_cb; ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ struct lrng_drng *drng = lrng_drng_init_instance(); ++ u8 digest[LRNG_MAX_DIGESTSIZE]; ++ unsigned long flags, flags2; ++ u32 found_events, collected_events = 0, collected_ent_bits, ++ requested_events, returned_ent_bits; ++ int ret, cpu; ++ void *hash; ++ ++ /* Only deliver entropy when SP800-90B self test is completed */ ++ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) { ++ eb->e_bits[lrng_int_es_sched] = 0; ++ return; ++ } ++ ++ /* Lock guarding replacement of per-NUMA hash */ ++ read_lock_irqsave(&drng->hash_lock, flags); ++ ++ hash_cb = drng->hash_cb; ++ hash = drng->hash; ++ ++ /* The hash state of filled with all per-CPU pool hashes. */ ++ ret = hash_cb->hash_init(shash, hash); ++ if (ret) ++ goto err; ++ ++ /* Cap to maximum entropy that can ever be generated with given hash */ ++ lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); ++ requested_events = lrng_entropy_to_data(requested_bits + ++ lrng_compress_osr(), ++ lrng_sched_entropy_bits); ++ ++ /* ++ * Harvest entropy from each per-CPU hash state - even though we may ++ * have collected sufficient entropy, we will hash all per-CPU pools. ++ */ ++ for_each_online_cpu(cpu) { ++ struct lrng_drng *pcpu_drng = drng; ++ u32 digestsize, unused_events = 0; ++ int node = cpu_to_node(cpu); ++ ++ if (lrng_drng && lrng_drng[node]) ++ pcpu_drng = lrng_drng[node]; ++ ++ if (pcpu_drng == drng) { ++ found_events = lrng_sched_pool_hash_one(hash_cb, hash, ++ cpu, digest, ++ &digestsize); ++ } else { ++ read_lock_irqsave(&pcpu_drng->hash_lock, flags2); ++ found_events = ++ lrng_sched_pool_hash_one(pcpu_drng->hash_cb, ++ pcpu_drng->hash, cpu, ++ digest, &digestsize); ++ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); ++ } ++ ++ /* Store all not-yet compressed data in data array into hash */ ++ ret = hash_cb->hash_update(shash, digest, digestsize); ++ if (ret) ++ goto err; ++ ++ collected_events += found_events; ++ if (collected_events > requested_events) { ++ unused_events = collected_events - requested_events; ++ atomic_add_return_relaxed(unused_events, ++ per_cpu_ptr(&lrng_sched_array_events, cpu)); ++ collected_events = requested_events; ++ } ++ pr_debug("%u scheduler-based events used from entropy array of CPU %d, %u scheduler-based events remain unused\n", ++ found_events - unused_events, cpu, unused_events); ++ } ++ ++ ret = hash_cb->hash_final(shash, digest); ++ if (ret) ++ goto err; ++ ++ collected_ent_bits = lrng_data_to_entropy(collected_events, ++ lrng_sched_entropy_bits); ++ /* Apply oversampling: discount requested oversampling rate */ ++ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); ++ ++ pr_debug("obtained %u bits by collecting %u bits of entropy from scheduler-based noise source\n", ++ returned_ent_bits, collected_ent_bits); ++ ++ /* ++ * Truncate to available entropy as implicitly allowed by SP800-90B ++ * section 3.1.5.1.1 table 1 which awards truncated hashes full ++ * entropy. ++ * ++ * During boot time, we read requested_bits data with ++ * returned_ent_bits entropy. In case our conservative entropy ++ * estimate underestimates the available entropy we can transport as ++ * much available entropy as possible. ++ */ ++ memcpy(eb->e[lrng_int_es_sched], digest, ++ fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); ++ eb->e_bits[lrng_int_es_sched] = returned_ent_bits; ++ ++out: ++ hash_cb->hash_desc_zero(shash); ++ read_unlock_irqrestore(&drng->hash_lock, flags); ++ memzero_explicit(digest, sizeof(digest)); ++ return; ++ ++err: ++ eb->e_bits[lrng_int_es_sched] = 0; ++ goto out; ++} ++ ++/* ++ * Concatenate full 32 bit word at the end of time array even when current ++ * ptr is not aligned to sizeof(data). ++ */ ++static void lrng_sched_array_add_u32(u32 data) ++{ ++ /* Increment pointer by number of slots taken for input value */ ++ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_sched_array_ptr, ++ LRNG_DATA_SLOTS_PER_UINT); ++ unsigned int pre_array; ++ ++ lrng_data_split_u32(&ptr, &pre_ptr, &mask); ++ ++ /* MSB of data go into previous unit */ ++ pre_array = lrng_data_idx2array(pre_ptr); ++ /* zeroization of slot to ensure the following OR adds the data */ ++ this_cpu_and(lrng_sched_array[pre_array], ~(0xffffffff & ~mask)); ++ this_cpu_or(lrng_sched_array[pre_array], data & ~mask); ++ ++ /* ++ * Continuous compression is not allowed for scheduler noise source, ++ * so do not call lrng_sched_array_to_hash here. ++ */ ++ ++ /* LSB of data go into current unit */ ++ this_cpu_write(lrng_sched_array[lrng_data_idx2array(ptr)], ++ data & mask); ++} ++ ++/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ ++static void lrng_sched_array_add_slot(u32 data) ++{ ++ /* Get slot */ ++ u32 ptr = this_cpu_inc_return(lrng_sched_array_ptr) & ++ LRNG_DATA_WORD_MASK; ++ unsigned int array = lrng_data_idx2array(ptr); ++ unsigned int slot = lrng_data_idx2slot(ptr); ++ ++ /* zeroization of slot to ensure the following OR adds the data */ ++ this_cpu_and(lrng_sched_array[array], ++ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, ++ slot))); ++ /* Store data into slot */ ++ this_cpu_or(lrng_sched_array[array], lrng_data_slot_val(data, slot)); ++ ++ /* ++ * Continuous compression is not allowed for scheduler noise source, ++ * so do not call lrng_sched_array_to_hash here. ++ */ ++} ++ ++static void ++lrng_time_process_common(u32 time, void(*add_time)(u32 data)) ++{ ++ enum lrng_health_res health_test; ++ ++ if (lrng_raw_sched_hires_entropy_store(time)) ++ return; ++ ++ health_test = lrng_health_test(time, lrng_int_es_sched); ++ if (health_test > lrng_health_fail_use) ++ return; ++ ++ if (health_test == lrng_health_pass) ++ atomic_inc_return(this_cpu_ptr(&lrng_sched_array_events)); ++ ++ add_time(time); ++ ++ /* ++ * We cannot call lrng_es_add_entropy() as this would call a schedule ++ * operation that is not permissible in scheduler context. ++ * As the scheduler ES provides a high bandwidth of entropy, we assume ++ * that other reseed triggers happen to pick up the scheduler ES ++ * entropy in due time. ++ */ ++} ++ ++/* Batching up of entropy in per-CPU array */ ++static void lrng_sched_time_process(void) ++{ ++ u32 now_time = random_get_entropy(); ++ ++ if (unlikely(!lrng_gcd_tested())) { ++ /* When GCD is unknown, we process the full time stamp */ ++ lrng_time_process_common(now_time, lrng_sched_array_add_u32); ++ lrng_gcd_add_value(now_time); ++ } else { ++ /* GCD is known and applied */ ++ lrng_time_process_common((now_time / lrng_gcd_get()) & ++ LRNG_DATA_SLOTSIZE_MASK, ++ lrng_sched_array_add_slot); ++ } ++ ++ lrng_sched_perf_time(now_time); ++} ++ ++void add_sched_randomness(const struct task_struct *p, int cpu) ++{ ++ if (lrng_highres_timer()) { ++ lrng_sched_time_process(); ++ } else { ++ u32 tmp = cpu; ++ ++ tmp ^= lrng_raw_sched_pid_entropy_store(p->pid) ? ++ 0 : (u32)p->pid; ++ tmp ^= lrng_raw_sched_starttime_entropy_store(p->start_time) ? ++ 0 : (u32)p->start_time; ++ tmp ^= lrng_raw_sched_nvcsw_entropy_store(p->nvcsw) ? ++ 0 : (u32)p->nvcsw; ++ ++ lrng_sched_time_process(); ++ lrng_sched_array_add_u32(tmp); ++ } ++} ++EXPORT_SYMBOL(add_sched_randomness); ++ ++static void lrng_sched_es_state(unsigned char *buf, size_t buflen) ++{ ++ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ ++ /* Assume the lrng_drng_init lock is taken by caller */ ++ snprintf(buf, buflen, ++ " Hash for operating entropy pool: %s\n" ++ " Available entropy: %u\n" ++ " per-CPU scheduler event collection size: %u\n" ++ " Standards compliance: %s\n" ++ " High-resolution timer: %s\n", ++ lrng_drng_init->hash_cb->hash_name(), ++ lrng_sched_avail_entropy(0), ++ LRNG_DATA_NUM_VALUES, ++ lrng_sp80090b_compliant(lrng_int_es_sched) ? "SP800-90B " : "", ++ lrng_highres_timer() ? "true" : "false"); ++} ++ ++struct lrng_es_cb lrng_es_sched = { ++ .name = "Scheduler", ++ .get_ent = lrng_sched_pool_hash, ++ .curr_entropy = lrng_sched_avail_entropy, ++ .max_entropy = lrng_sched_avail_pool_size, ++ .state = lrng_sched_es_state, ++ .reset = lrng_sched_reset, ++ .switch_hash = lrng_sched_switch_hash, ++}; +diff --git a/drivers/char/lrng/lrng_es_sched.h b/drivers/char/lrng/lrng_es_sched.h +new file mode 100644 +index 000000000000..f1e596dd89d9 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_sched.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_SCHED_H ++#define _LRNG_ES_SCHED_H ++ ++#include "lrng_es_mgr_cb.h" ++ ++#ifdef CONFIG_LRNG_SCHED ++void lrng_sched_es_init(bool highres_timer); ++ ++extern struct lrng_es_cb lrng_es_sched; ++ ++#else /* CONFIG_LRNG_SCHED */ ++static inline void lrng_sched_es_init(bool highres_timer) { } ++#endif /* CONFIG_LRNG_SCHED */ ++ ++#endif /* _LRNG_ES_SCHED_H */ +diff --git a/drivers/char/lrng/lrng_es_timer_common.c b/drivers/char/lrng/lrng_es_timer_common.c +new file mode 100644 +index 000000000000..70f3ff074fe6 +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_timer_common.c +@@ -0,0 +1,144 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Slow Entropy Source: Interrupt data collection ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++ ++#include "lrng_es_irq.h" ++#include "lrng_es_sched.h" ++#include "lrng_es_timer_common.h" ++#include "lrng_health.h" ++ ++/* Is high-resolution timer present? */ ++static bool lrng_highres_timer_val = false; ++ ++/* Number of time stamps analyzed to calculate a GCD */ ++#define LRNG_GCD_WINDOW_SIZE 100 ++static u32 lrng_gcd_history[LRNG_GCD_WINDOW_SIZE]; ++static atomic_t lrng_gcd_history_ptr = ATOMIC_INIT(-1); ++ ++/* The common divisor for all timestamps */ ++static u32 lrng_gcd_timer = 0; ++ ++bool lrng_gcd_tested(void) ++{ ++ return (lrng_gcd_timer != 0); ++} ++ ++u32 lrng_gcd_get(void) ++{ ++ return lrng_gcd_timer; ++} ++ ++/* Set the GCD for use in IRQ ES - if 0, the GCD calculation is restarted. */ ++void lrng_gcd_set(u32 running_gcd) ++{ ++ lrng_gcd_timer = running_gcd; ++ /* Ensure that update to global variable lrng_gcd_timer is visible */ ++ mb(); ++} ++ ++static void lrng_gcd_set_check(u32 running_gcd) ++{ ++ if (!lrng_gcd_tested()) { ++ lrng_gcd_set(running_gcd); ++ pr_debug("Setting GCD to %u\n", running_gcd); ++ } ++} ++ ++u32 lrng_gcd_analyze(u32 *history, size_t nelem) ++{ ++ u32 running_gcd = 0; ++ size_t i; ++ ++ /* Now perform the analysis on the accumulated time data. */ ++ for (i = 0; i < nelem; i++) { ++ /* ++ * NOTE: this would be the place to add more analysis on the ++ * appropriateness of the timer like checking the presence ++ * of sufficient variations in the timer. ++ */ ++ ++ /* ++ * This calculates the gcd of all the time values. that is ++ * gcd(time_1, time_2, ..., time_nelem) ++ * ++ * Some timers increment by a fixed (non-1) amount each step. ++ * This code checks for such increments, and allows the library ++ * to output the number of such changes have occurred. ++ */ ++ running_gcd = (u32)gcd(history[i], running_gcd); ++ ++ /* Zeroize data */ ++ history[i] = 0; ++ } ++ ++ return running_gcd; ++} ++ ++void lrng_gcd_add_value(u32 time) ++{ ++ u32 ptr = (u32)atomic_inc_return_relaxed(&lrng_gcd_history_ptr); ++ ++ if (ptr < LRNG_GCD_WINDOW_SIZE) { ++ lrng_gcd_history[ptr] = time; ++ } else if (ptr == LRNG_GCD_WINDOW_SIZE) { ++ u32 gcd = lrng_gcd_analyze(lrng_gcd_history, ++ LRNG_GCD_WINDOW_SIZE); ++ ++ if (!gcd) ++ gcd = 1; ++ ++ /* ++ * Ensure that we have variations in the time stamp below the ++ * given value. This is just a safety measure to prevent the GCD ++ * becoming too large. ++ */ ++ if (gcd >= 1000) { ++ pr_warn("calculated GCD is larger than expected: %u\n", ++ gcd); ++ gcd = 1000; ++ } ++ ++ /* Adjust all deltas by the observed (small) common factor. */ ++ lrng_gcd_set_check(gcd); ++ atomic_set(&lrng_gcd_history_ptr, 0); ++ } ++} ++ ++/* Return boolean whether LRNG identified presence of high-resolution timer */ ++bool lrng_highres_timer(void) ++{ ++ return lrng_highres_timer_val; ++} ++ ++static int __init lrng_init_time_source(void) ++{ ++ if ((random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK) || ++ (random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK)) { ++ /* ++ * As the highres timer is identified here, previous interrupts ++ * obtained during boot time are treated like a lowres-timer ++ * would have been present. ++ */ ++ lrng_highres_timer_val = true; ++ } else { ++ lrng_health_disable(); ++ lrng_highres_timer_val = false; ++ } ++ ++ lrng_irq_es_init(lrng_highres_timer_val); ++ lrng_sched_es_init(lrng_highres_timer_val); ++ ++ /* Ensure that changes to global variables are visible */ ++ mb(); ++ ++ return 0; ++} ++core_initcall(lrng_init_time_source); +diff --git a/drivers/char/lrng/lrng_es_timer_common.h b/drivers/char/lrng/lrng_es_timer_common.h +new file mode 100644 +index 000000000000..b45b9f683dea +--- /dev/null ++++ b/drivers/char/lrng/lrng_es_timer_common.h +@@ -0,0 +1,83 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * LRNG Slow Noise Source: Time stamp array handling ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_ES_TIMER_COMMON_H ++#define _LRNG_ES_TIMER_COMMON_H ++ ++bool lrng_gcd_tested(void); ++void lrng_gcd_set(u32 running_gcd); ++u32 lrng_gcd_get(void); ++u32 lrng_gcd_analyze(u32 *history, size_t nelem); ++void lrng_gcd_add_value(u32 time); ++bool lrng_highres_timer(void); ++ ++/* ++ * To limit the impact on the interrupt handling, the LRNG concatenates ++ * entropic LSB parts of the time stamps in a per-CPU array and only ++ * injects them into the entropy pool when the array is full. ++ */ ++ ++/* Store multiple integers in one u32 */ ++#define LRNG_DATA_SLOTSIZE_BITS (8) ++#define LRNG_DATA_SLOTSIZE_MASK ((1 << LRNG_DATA_SLOTSIZE_BITS) - 1) ++#define LRNG_DATA_ARRAY_MEMBER_BITS (4 << 3) /* ((sizeof(u32)) << 3) */ ++#define LRNG_DATA_SLOTS_PER_UINT (LRNG_DATA_ARRAY_MEMBER_BITS / \ ++ LRNG_DATA_SLOTSIZE_BITS) ++ ++/* ++ * Number of time values to store in the array - in small environments ++ * only one atomic_t variable per CPU is used. ++ */ ++#define LRNG_DATA_NUM_VALUES (CONFIG_LRNG_COLLECTION_SIZE) ++/* Mask of LSB of time stamp to store */ ++#define LRNG_DATA_WORD_MASK (LRNG_DATA_NUM_VALUES - 1) ++ ++#define LRNG_DATA_SLOTS_MASK (LRNG_DATA_SLOTS_PER_UINT - 1) ++#define LRNG_DATA_ARRAY_SIZE (LRNG_DATA_NUM_VALUES / \ ++ LRNG_DATA_SLOTS_PER_UINT) ++ ++/* Starting bit index of slot */ ++static inline unsigned int lrng_data_slot2bitindex(unsigned int slot) ++{ ++ return (LRNG_DATA_SLOTSIZE_BITS * slot); ++} ++ ++/* Convert index into the array index */ ++static inline unsigned int lrng_data_idx2array(unsigned int idx) ++{ ++ return idx / LRNG_DATA_SLOTS_PER_UINT; ++} ++ ++/* Convert index into the slot of a given array index */ ++static inline unsigned int lrng_data_idx2slot(unsigned int idx) ++{ ++ return idx & LRNG_DATA_SLOTS_MASK; ++} ++ ++/* Convert value into slot value */ ++static inline unsigned int lrng_data_slot_val(unsigned int val, ++ unsigned int slot) ++{ ++ return val << lrng_data_slot2bitindex(slot); ++} ++ ++/* ++ * Return the pointers for the previous and current units to inject a u32 into. ++ * Also return the mask which the u32 word is to be processed. ++ */ ++static inline void lrng_data_split_u32(u32 *ptr, u32 *pre_ptr, u32 *mask) ++{ ++ /* ptr to previous unit */ ++ *pre_ptr = (*ptr - LRNG_DATA_SLOTS_PER_UINT) & LRNG_DATA_WORD_MASK; ++ *ptr &= LRNG_DATA_WORD_MASK; ++ ++ /* mask to split data into the two parts for the two units */ ++ *mask = ((1 << (*pre_ptr & (LRNG_DATA_SLOTS_PER_UINT - 1)) * ++ LRNG_DATA_SLOTSIZE_BITS)) - 1; ++} ++ ++#endif /* _LRNG_ES_TIMER_COMMON_H */ +\ No newline at end of file +diff --git a/drivers/char/lrng/lrng_hash_kcapi.c b/drivers/char/lrng/lrng_hash_kcapi.c +new file mode 100644 +index 000000000000..13e62db9b6c8 +--- /dev/null ++++ b/drivers/char/lrng/lrng_hash_kcapi.c +@@ -0,0 +1,140 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Backend for providing the hash primitive using the kernel crypto API. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++ ++ ++static char *lrng_hash_name = "sha512"; ++ ++/* The parameter must be r/o in sysfs as otherwise races appear. */ ++module_param(lrng_hash_name, charp, 0444); ++MODULE_PARM_DESC(lrng_hash_name, "Kernel crypto API hash name"); ++ ++struct lrng_hash_info { ++ struct crypto_shash *tfm; ++}; ++ ++static const char *lrng_kcapi_hash_name(void) ++{ ++ return lrng_hash_name; ++} ++ ++static void _lrng_kcapi_hash_free(struct lrng_hash_info *lrng_hash) ++{ ++ struct crypto_shash *tfm = lrng_hash->tfm; ++ ++ crypto_free_shash(tfm); ++ kfree(lrng_hash); ++} ++ ++static void *lrng_kcapi_hash_alloc(const char *name) ++{ ++ struct lrng_hash_info *lrng_hash; ++ struct crypto_shash *tfm; ++ int ret; ++ ++ if (!name) { ++ pr_err("Hash name missing\n"); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ tfm = crypto_alloc_shash(name, 0, 0); ++ if (IS_ERR(tfm)) { ++ pr_err("could not allocate hash %s\n", name); ++ return ERR_CAST(tfm); ++ } ++ ++ ret = sizeof(struct lrng_hash_info); ++ lrng_hash = kmalloc(ret, GFP_KERNEL); ++ if (!lrng_hash) { ++ crypto_free_shash(tfm); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ lrng_hash->tfm = tfm; ++ ++ pr_info("Hash %s allocated\n", name); ++ ++ return lrng_hash; ++} ++ ++static void *lrng_kcapi_hash_name_alloc(void) ++{ ++ return lrng_kcapi_hash_alloc(lrng_kcapi_hash_name()); ++} ++ ++static u32 lrng_kcapi_hash_digestsize(void *hash) ++{ ++ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; ++ struct crypto_shash *tfm = lrng_hash->tfm; ++ ++ return crypto_shash_digestsize(tfm); ++} ++ ++static void lrng_kcapi_hash_dealloc(void *hash) ++{ ++ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; ++ ++ _lrng_kcapi_hash_free(lrng_hash); ++ pr_info("Hash deallocated\n"); ++} ++ ++static int lrng_kcapi_hash_init(struct shash_desc *shash, void *hash) ++{ ++ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; ++ struct crypto_shash *tfm = lrng_hash->tfm; ++ ++ shash->tfm = tfm; ++ return crypto_shash_init(shash); ++} ++ ++static int lrng_kcapi_hash_update(struct shash_desc *shash, const u8 *inbuf, ++ u32 inbuflen) ++{ ++ return crypto_shash_update(shash, inbuf, inbuflen); ++} ++ ++static int lrng_kcapi_hash_final(struct shash_desc *shash, u8 *digest) ++{ ++ return crypto_shash_final(shash, digest); ++} ++ ++static void lrng_kcapi_hash_zero(struct shash_desc *shash) ++{ ++ shash_desc_zero(shash); ++} ++ ++static const struct lrng_hash_cb lrng_kcapi_hash_cb = { ++ .hash_name = lrng_kcapi_hash_name, ++ .hash_alloc = lrng_kcapi_hash_name_alloc, ++ .hash_dealloc = lrng_kcapi_hash_dealloc, ++ .hash_digestsize = lrng_kcapi_hash_digestsize, ++ .hash_init = lrng_kcapi_hash_init, ++ .hash_update = lrng_kcapi_hash_update, ++ .hash_final = lrng_kcapi_hash_final, ++ .hash_desc_zero = lrng_kcapi_hash_zero, ++}; ++ ++static int __init lrng_kcapi_init(void) ++{ ++ return lrng_set_hash_cb(&lrng_kcapi_hash_cb); ++} ++ ++static void __exit lrng_kcapi_exit(void) ++{ ++ lrng_set_hash_cb(NULL); ++} ++ ++late_initcall(lrng_kcapi_init); ++module_exit(lrng_kcapi_exit); ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_AUTHOR("Stephan Mueller "); ++MODULE_DESCRIPTION("Entropy Source and DRNG Manager - Kernel crypto API hash backend"); +diff --git a/drivers/char/lrng/lrng_health.c b/drivers/char/lrng/lrng_health.c +new file mode 100644 +index 000000000000..a60c8378eea5 +--- /dev/null ++++ b/drivers/char/lrng/lrng_health.c +@@ -0,0 +1,420 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Entropy Source and DRNG Manager (LRNG) Health Testing ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++ ++#include "lrng_es_mgr.h" ++#include "lrng_health.h" ++ ++/* Stuck Test */ ++struct lrng_stuck_test { ++ u32 last_time; /* Stuck test: time of previous IRQ */ ++ u32 last_delta; /* Stuck test: delta of previous IRQ */ ++ u32 last_delta2; /* Stuck test: 2. time derivation of prev IRQ */ ++}; ++ ++/* Repetition Count Test */ ++struct lrng_rct { ++ atomic_t rct_count; /* Number of stuck values */ ++}; ++ ++/* Adaptive Proportion Test */ ++struct lrng_apt { ++ /* Data window size */ ++#define LRNG_APT_WINDOW_SIZE 512 ++ /* LSB of time stamp to process */ ++#define LRNG_APT_LSB 16 ++#define LRNG_APT_WORD_MASK (LRNG_APT_LSB - 1) ++ atomic_t apt_count; /* APT counter */ ++ atomic_t apt_base; /* APT base reference */ ++ ++ atomic_t apt_trigger; ++ bool apt_base_set; /* Is APT base set? */ ++}; ++ ++/* Health data collected for one entropy source */ ++struct lrng_health_es_state { ++ struct lrng_rct rct; ++ struct lrng_apt apt; ++ ++ /* SP800-90B startup health tests */ ++#define LRNG_SP80090B_STARTUP_SAMPLES 1024 ++#define LRNG_SP80090B_STARTUP_BLOCKS ((LRNG_SP80090B_STARTUP_SAMPLES + \ ++ LRNG_APT_WINDOW_SIZE - 1) / \ ++ LRNG_APT_WINDOW_SIZE) ++ bool sp80090b_startup_done; ++ atomic_t sp80090b_startup_blocks; ++}; ++ ++#define LRNG_HEALTH_ES_INIT(x) \ ++ x.rct.rct_count = ATOMIC_INIT(0), \ ++ x.apt.apt_count = ATOMIC_INIT(0), \ ++ x.apt.apt_base = ATOMIC_INIT(-1), \ ++ x.apt.apt_trigger = ATOMIC_INIT(LRNG_APT_WINDOW_SIZE), \ ++ x.apt.apt_base_set = false, \ ++ x.sp80090b_startup_blocks = ATOMIC_INIT(LRNG_SP80090B_STARTUP_BLOCKS), \ ++ x.sp80090b_startup_done = false, ++ ++/* The health test code must operate lock-less */ ++struct lrng_health { ++ bool health_test_enabled; ++ struct lrng_health_es_state es_state[lrng_int_es_last]; ++}; ++ ++static struct lrng_health lrng_health = { ++ .health_test_enabled = true, ++ ++#ifdef CONFIG_LRNG_IRQ ++ LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_irq]) ++#endif ++#ifdef CONFIG_LRNG_SCHED ++ LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_sched]) ++#endif ++}; ++ ++static DEFINE_PER_CPU(struct lrng_stuck_test[lrng_int_es_last], ++ lrng_stuck_test_array); ++ ++static bool lrng_sp80090b_health_requested(void) ++{ ++ /* Health tests are only requested in FIPS mode */ ++ return fips_enabled; ++} ++ ++static bool lrng_sp80090b_health_enabled(void) ++{ ++ struct lrng_health *health = &lrng_health; ++ ++ return lrng_sp80090b_health_requested() && health->health_test_enabled; ++} ++ ++/*************************************************************************** ++ * SP800-90B Compliance ++ * ++ * If the Linux-RNG is booted into FIPS mode, the following interfaces ++ * provide an SP800-90B compliant noise source: ++ * ++ * * /dev/random ++ * * getrandom(2) ++ * * get_random_bytes_full ++ * ++ * All other interfaces, including /dev/urandom or get_random_bytes without ++ * the add_random_ready_callback cannot claim to use an SP800-90B compliant ++ * noise source. ++ ***************************************************************************/ ++ ++/* ++ * Perform SP800-90B startup testing ++ */ ++static void lrng_sp80090b_startup(struct lrng_health *health, ++ enum lrng_internal_es es) ++{ ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ ++ if (!es_state->sp80090b_startup_done && ++ atomic_dec_and_test(&es_state->sp80090b_startup_blocks)) { ++ es_state->sp80090b_startup_done = true; ++ pr_info("SP800-90B startup health tests for internal entropy source %u completed\n", ++ es); ++ lrng_drng_force_reseed(); ++ ++ /* ++ * We cannot call lrng_es_add_entropy() as this may cause a ++ * schedule operation while in scheduler context for the ++ * scheduler ES. ++ */ ++ } ++} ++ ++/* ++ * Handle failure of SP800-90B startup testing ++ */ ++static void lrng_sp80090b_startup_failure(struct lrng_health *health, ++ enum lrng_internal_es es) ++{ ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ ++ /* Reset of LRNG and its entropy - NOTE: we are in atomic context */ ++ lrng_reset(); ++ ++ /* ++ * Reset the SP800-90B startup test. ++ * ++ * NOTE SP800-90B section 4.3 bullet 4 does not specify what ++ * exactly is to be done in case of failure! Thus, we do what ++ * makes sense, i.e. restarting the health test and thus gating ++ * the output function of /dev/random and getrandom(2). ++ */ ++ atomic_set(&es_state->sp80090b_startup_blocks, ++ LRNG_SP80090B_STARTUP_BLOCKS); ++} ++ ++/* ++ * Handle failure of SP800-90B runtime testing ++ */ ++static void lrng_sp80090b_runtime_failure(struct lrng_health *health, ++ enum lrng_internal_es es) ++{ ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ ++ lrng_sp80090b_startup_failure(health, es); ++ es_state->sp80090b_startup_done = false; ++} ++ ++static void lrng_sp80090b_failure(struct lrng_health *health, ++ enum lrng_internal_es es) ++{ ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ ++ if (es_state->sp80090b_startup_done) { ++ pr_err("SP800-90B runtime health test failure for internal entropy source %u - invalidating all existing entropy and initiate SP800-90B startup\n", es); ++ lrng_sp80090b_runtime_failure(health, es); ++ } else { ++ pr_err("SP800-90B startup test failure for internal entropy source %u - resetting\n", es); ++ lrng_sp80090b_startup_failure(health, es); ++ } ++} ++ ++bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) ++{ ++ struct lrng_health *health = &lrng_health; ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ ++ if (!lrng_sp80090b_health_enabled()) ++ return true; ++ ++ return es_state->sp80090b_startup_done; ++} ++ ++bool lrng_sp80090b_compliant(enum lrng_internal_es es) ++{ ++ struct lrng_health *health = &lrng_health; ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ ++ return lrng_sp80090b_health_enabled() && ++ es_state->sp80090b_startup_done; ++} ++ ++/*************************************************************************** ++ * Adaptive Proportion Test ++ * ++ * This test complies with SP800-90B section 4.4.2. ++ ***************************************************************************/ ++ ++/* ++ * Reset the APT counter ++ * ++ * @health [in] Reference to health state ++ */ ++static void lrng_apt_reset(struct lrng_apt *apt, unsigned int time_masked) ++{ ++ /* Reset APT */ ++ atomic_set(&apt->apt_count, 0); ++ atomic_set(&apt->apt_base, time_masked); ++} ++ ++static void lrng_apt_restart(struct lrng_apt *apt) ++{ ++ atomic_set(&apt->apt_trigger, LRNG_APT_WINDOW_SIZE); ++} ++ ++/* ++ * Insert a new entropy event into APT ++ * ++ * This function does is void as it does not decide about the fate of a time ++ * stamp. An APT failure can only happen at the same time of a stuck test ++ * failure. Thus, the stuck failure will already decide how the time stamp ++ * is handled. ++ * ++ * @health [in] Reference to health state ++ * @now_time [in] Time stamp to process ++ */ ++static void lrng_apt_insert(struct lrng_health *health, ++ unsigned int now_time, enum lrng_internal_es es) ++{ ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ struct lrng_apt *apt = &es_state->apt; ++ ++ if (!lrng_sp80090b_health_requested()) ++ return; ++ ++ now_time &= LRNG_APT_WORD_MASK; ++ ++ /* Initialization of APT */ ++ if (!apt->apt_base_set) { ++ atomic_set(&apt->apt_base, now_time); ++ apt->apt_base_set = true; ++ return; ++ } ++ ++ if (now_time == (unsigned int)atomic_read(&apt->apt_base)) { ++ u32 apt_val = (u32)atomic_inc_return_relaxed(&apt->apt_count); ++ ++ if (apt_val >= CONFIG_LRNG_APT_CUTOFF) ++ lrng_sp80090b_failure(health, es); ++ } ++ ++ if (atomic_dec_and_test(&apt->apt_trigger)) { ++ lrng_apt_restart(apt); ++ lrng_apt_reset(apt, now_time); ++ lrng_sp80090b_startup(health, es); ++ } ++} ++ ++/*************************************************************************** ++ * Repetition Count Test ++ * ++ * The LRNG uses an enhanced version of the Repetition Count Test ++ * (RCT) specified in SP800-90B section 4.4.1. Instead of counting identical ++ * back-to-back values, the input to the RCT is the counting of the stuck ++ * values while filling the entropy pool. ++ * ++ * The RCT is applied with an alpha of 2^-30 compliant to FIPS 140-2 IG 9.8. ++ * ++ * During the counting operation, the LRNG always calculates the RCT ++ * cut-off value of C. If that value exceeds the allowed cut-off value, ++ * the LRNG will invalidate all entropy for the entropy pool which implies ++ * that no data can be extracted from the entropy pool unless new entropy ++ * is received. ++ ***************************************************************************/ ++ ++/* ++ * Hot code path - Insert data for Repetition Count Test ++ * ++ * @health: Reference to health information ++ * @stuck: Decision of stuck test ++ */ ++static void lrng_rct(struct lrng_health *health, enum lrng_internal_es es, ++ int stuck) ++{ ++ struct lrng_health_es_state *es_state = &health->es_state[es]; ++ struct lrng_rct *rct = &es_state->rct; ++ ++ if (!lrng_sp80090b_health_requested()) ++ return; ++ ++ if (stuck) { ++ u32 rct_count = atomic_add_return_relaxed(1, &rct->rct_count); ++ ++ /* ++ * The cutoff value is based on the following consideration: ++ * alpha = 2^-30 as recommended in FIPS 140-2 IG 9.8. ++ * In addition, we imply an entropy value H of 1 bit as this ++ * is the minimum entropy required to provide full entropy. ++ * ++ * Note, rct_count (which equals to value B in the ++ * pseudo code of SP800-90B section 4.4.1) starts with zero. ++ * Hence we need to subtract one from the cutoff value as ++ * calculated following SP800-90B. ++ */ ++ if (rct_count >= CONFIG_LRNG_RCT_CUTOFF) { ++ atomic_set(&rct->rct_count, 0); ++ ++ /* ++ * APT must start anew as we consider all previously ++ * recorded data to contain no entropy. ++ */ ++ lrng_apt_restart(&es_state->apt); ++ ++ lrng_sp80090b_failure(health, es); ++ } ++ } else { ++ atomic_set(&rct->rct_count, 0); ++ } ++} ++ ++/*************************************************************************** ++ * Stuck Test ++ * ++ * Checking the: ++ * 1st derivative of the event occurrence (time delta) ++ * 2nd derivative of the event occurrence (delta of time deltas) ++ * 3rd derivative of the event occurrence (delta of delta of time deltas) ++ * ++ * All values must always be non-zero. The stuck test is only valid disabled if ++ * high-resolution time stamps are identified after initialization. ++ ***************************************************************************/ ++ ++static u32 lrng_delta(u32 prev, u32 next) ++{ ++ /* ++ * Note that this (unsigned) subtraction does yield the correct value ++ * in the wraparound-case, i.e. when next < prev. ++ */ ++ return (next - prev); ++} ++ ++/* ++ * Hot code path ++ * ++ * @health: Reference to health information ++ * @now: Event time ++ * @return: 0 event occurrence not stuck (good time stamp) ++ * != 0 event occurrence stuck (reject time stamp) ++ */ ++static int lrng_irq_stuck(enum lrng_internal_es es, u32 now_time) ++{ ++ struct lrng_stuck_test *stuck = this_cpu_ptr(lrng_stuck_test_array); ++ u32 delta = lrng_delta(stuck[es].last_time, now_time); ++ u32 delta2 = lrng_delta(stuck[es].last_delta, delta); ++ u32 delta3 = lrng_delta(stuck[es].last_delta2, delta2); ++ ++ stuck[es].last_time = now_time; ++ stuck[es].last_delta = delta; ++ stuck[es].last_delta2 = delta2; ++ ++ if (!delta || !delta2 || !delta3) ++ return 1; ++ ++ return 0; ++} ++ ++/*************************************************************************** ++ * Health test interfaces ++ ***************************************************************************/ ++ ++/* ++ * Disable all health tests ++ */ ++void lrng_health_disable(void) ++{ ++ struct lrng_health *health = &lrng_health; ++ ++ health->health_test_enabled = false; ++ ++ if (lrng_sp80090b_health_requested()) ++ pr_warn("SP800-90B compliance requested but the Linux RNG is NOT SP800-90B compliant\n"); ++} ++ ++/* ++ * Hot code path - Perform health test on time stamp received from an event ++ * ++ * @now_time Time stamp ++ */ ++enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es) ++{ ++ struct lrng_health *health = &lrng_health; ++ int stuck; ++ ++ if (!health->health_test_enabled) ++ return lrng_health_pass; ++ ++ lrng_apt_insert(health, now_time, es); ++ ++ stuck = lrng_irq_stuck(es, now_time); ++ lrng_rct(health, es, stuck); ++ if (stuck) { ++ /* SP800-90B disallows using a failing health test time stamp */ ++ return lrng_sp80090b_health_requested() ? ++ lrng_health_fail_drop : lrng_health_fail_use; ++ } ++ ++ return lrng_health_pass; ++} +diff --git a/drivers/char/lrng/lrng_health.h b/drivers/char/lrng/lrng_health.h +new file mode 100644 +index 000000000000..4f9f5033fc30 +--- /dev/null ++++ b/drivers/char/lrng/lrng_health.h +@@ -0,0 +1,42 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_HEALTH_H ++#define _LRNG_HEALTH_H ++ ++#include "lrng_es_mgr.h" ++ ++enum lrng_health_res { ++ lrng_health_pass, /* Health test passes on time stamp */ ++ lrng_health_fail_use, /* Time stamp unhealthy, but mix in */ ++ lrng_health_fail_drop /* Time stamp unhealthy, drop it */ ++}; ++ ++#ifdef CONFIG_LRNG_HEALTH_TESTS ++bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es); ++bool lrng_sp80090b_compliant(enum lrng_internal_es es); ++ ++enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es); ++void lrng_health_disable(void); ++#else /* CONFIG_LRNG_HEALTH_TESTS */ ++static inline bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) ++{ ++ return true; ++} ++ ++static inline bool lrng_sp80090b_compliant(enum lrng_internal_es es) ++{ ++ return false; ++} ++ ++static inline enum lrng_health_res ++lrng_health_test(u32 now_time, enum lrng_internal_es es) ++{ ++ return lrng_health_pass; ++} ++static inline void lrng_health_disable(void) { } ++#endif /* CONFIG_LRNG_HEALTH_TESTS */ ++ ++#endif /* _LRNG_HEALTH_H */ +diff --git a/drivers/char/lrng/lrng_interface_aux.c b/drivers/char/lrng/lrng_interface_aux.c +new file mode 100644 +index 000000000000..0eb49bd6b48f +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_aux.c +@@ -0,0 +1,126 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG auxiliary interfaces ++ * ++ * Copyright (C) 2022 Stephan Mueller ++ * Copyright (C) 2017 Jason A. Donenfeld . All ++ * Rights Reserved. ++ * Copyright (C) 2016 Jason Cooper ++ */ ++ ++#include ++#include ++#include ++ ++#include "lrng_es_mgr.h" ++#include "lrng_interface_random_kernel.h" ++ ++struct batched_entropy { ++ union { ++ u64 entropy_u64[LRNG_DRNG_BLOCKSIZE / sizeof(u64)]; ++ u32 entropy_u32[LRNG_DRNG_BLOCKSIZE / sizeof(u32)]; ++ }; ++ unsigned int position; ++ spinlock_t batch_lock; ++}; ++ ++/* ++ * Get a random word for internal kernel use only. The quality of the random ++ * number is as good as /dev/urandom, but there is no backtrack protection, ++ * with the goal of being quite fast and not depleting entropy. ++ */ ++static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { ++ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), ++}; ++ ++u64 get_random_u64(void) ++{ ++ u64 ret; ++ unsigned long flags; ++ struct batched_entropy *batch; ++ ++ lrng_debug_report_seedlevel("get_random_u64"); ++ ++ batch = raw_cpu_ptr(&batched_entropy_u64); ++ spin_lock_irqsave(&batch->batch_lock, flags); ++ if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { ++ lrng_get_random_bytes(batch->entropy_u64, LRNG_DRNG_BLOCKSIZE); ++ batch->position = 0; ++ } ++ ret = batch->entropy_u64[batch->position++]; ++ spin_unlock_irqrestore(&batch->batch_lock, flags); ++ return ret; ++} ++EXPORT_SYMBOL(get_random_u64); ++ ++static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { ++ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), ++}; ++ ++u32 get_random_u32(void) ++{ ++ u32 ret; ++ unsigned long flags; ++ struct batched_entropy *batch; ++ ++ lrng_debug_report_seedlevel("get_random_u32"); ++ ++ batch = raw_cpu_ptr(&batched_entropy_u32); ++ spin_lock_irqsave(&batch->batch_lock, flags); ++ if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { ++ lrng_get_random_bytes(batch->entropy_u32, LRNG_DRNG_BLOCKSIZE); ++ batch->position = 0; ++ } ++ ret = batch->entropy_u32[batch->position++]; ++ spin_unlock_irqrestore(&batch->batch_lock, flags); ++ return ret; ++} ++EXPORT_SYMBOL(get_random_u32); ++ ++#ifdef CONFIG_SMP ++/* ++ * This function is called when the CPU is coming up, with entry ++ * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP. ++ */ ++int random_prepare_cpu(unsigned int cpu) ++{ ++ /* ++ * When the cpu comes back online, immediately invalidate all batches, ++ * so that we serve fresh randomness. ++ */ ++ per_cpu_ptr(&batched_entropy_u32, cpu)->position = 0; ++ per_cpu_ptr(&batched_entropy_u64, cpu)->position = 0; ++ return 0; ++} ++ ++int random_online_cpu(unsigned int cpu) ++{ ++ return 0; ++} ++#endif ++ ++/* ++ * It's important to invalidate all potential batched entropy that might ++ * be stored before the crng is initialized, which we can do lazily by ++ * simply resetting the counter to zero so that it's re-extracted on the ++ * next usage. ++ */ ++void invalidate_batched_entropy(void) ++{ ++ int cpu; ++ unsigned long flags; ++ ++ for_each_possible_cpu(cpu) { ++ struct batched_entropy *batched_entropy; ++ ++ batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); ++ spin_lock_irqsave(&batched_entropy->batch_lock, flags); ++ batched_entropy->position = 0; ++ spin_unlock(&batched_entropy->batch_lock); ++ ++ batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); ++ spin_lock(&batched_entropy->batch_lock); ++ batched_entropy->position = 0; ++ spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); ++ } ++} +diff --git a/drivers/char/lrng/lrng_interface_dev.c b/drivers/char/lrng/lrng_interface_dev.c +new file mode 100644 +index 000000000000..e60060d402b3 +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_dev.c +@@ -0,0 +1,35 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG user space device file interface ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#include ++#include ++ ++#include "lrng_interface_dev_common.h" ++ ++static const struct file_operations lrng_fops = { ++ .read = lrng_drng_read_block, ++ .write = lrng_drng_write, ++ .poll = lrng_random_poll, ++ .unlocked_ioctl = lrng_ioctl, ++ .compat_ioctl = compat_ptr_ioctl, ++ .fasync = lrng_fasync, ++ .llseek = noop_llseek, ++}; ++ ++static struct miscdevice lrng_miscdev = { ++ .minor = MISC_DYNAMIC_MINOR, ++ .name = "lrng", ++ .nodename = "lrng", ++ .fops = &lrng_fops, ++ .mode = 0666 ++}; ++ ++static int __init lrng_dev_if_mod_init(void) ++{ ++ return misc_register(&lrng_miscdev); ++} ++device_initcall(lrng_dev_if_mod_init); +diff --git a/drivers/char/lrng/lrng_interface_dev_common.c b/drivers/char/lrng/lrng_interface_dev_common.c +new file mode 100644 +index 000000000000..09d0563074a4 +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_dev_common.c +@@ -0,0 +1,314 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG User and kernel space interfaces ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_mgr.h" ++#include "lrng_interface_dev_common.h" ++ ++DECLARE_WAIT_QUEUE_HEAD(lrng_write_wait); ++static struct fasync_struct *fasync; ++ ++static bool lrng_seed_hw = true; /* Allow HW to provide seed */ ++static bool lrng_seed_user = true; /* Allow user space to provide seed */ ++ ++/********************************** Helper ***********************************/ ++ ++static u32 lrng_get_aux_ent(void) ++{ ++ return lrng_es[lrng_ext_es_aux]->curr_entropy(0); ++} ++ ++/* Is the DRNG seed level too low? */ ++bool lrng_need_entropy(void) ++{ ++ return (lrng_get_aux_ent() < lrng_write_wakeup_bits); ++} ++ ++void lrng_writer_wakeup(void) ++{ ++ if (lrng_need_entropy() && wq_has_sleeper(&lrng_write_wait)) { ++ wake_up_interruptible(&lrng_write_wait); ++ kill_fasync(&fasync, SIGIO, POLL_OUT); ++ } ++} ++ ++void lrng_init_wakeup_dev(void) ++{ ++ kill_fasync(&fasync, SIGIO, POLL_IN); ++} ++ ++/* External entropy provider is allowed to provide seed data */ ++bool lrng_state_exseed_allow(enum lrng_external_noise_source source) ++{ ++ if (source == lrng_noise_source_hw) ++ return lrng_seed_hw; ++ return lrng_seed_user; ++} ++ ++/* Enable / disable external entropy provider to furnish seed */ ++void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) ++{ ++ /* ++ * If the LRNG is not yet operational, allow all entropy sources ++ * to deliver data unconditionally to get fully seeded asap. ++ */ ++ if (!lrng_state_operational()) ++ return; ++ ++ if (source == lrng_noise_source_hw) ++ lrng_seed_hw = type; ++ else ++ lrng_seed_user = type; ++} ++ ++void lrng_state_exseed_allow_all(void) ++{ ++ lrng_state_exseed_set(lrng_noise_source_hw, true); ++ lrng_state_exseed_set(lrng_noise_source_user, true); ++} ++ ++/************************ LRNG user output interfaces *************************/ ++ ++ssize_t lrng_read_seed(char __user *buf, size_t nbytes, unsigned int flags) ++{ ++ ssize_t ret = 0; ++ u64 t[(sizeof(struct entropy_buf) + 3 * sizeof(u64) - 1) / sizeof(u64)]; ++ ++ memset(t, 0, sizeof(t)); ++ ret = lrng_get_seed(t, min_t(size_t, nbytes, sizeof(t)), flags); ++ if (ret == -EMSGSIZE && copy_to_user(buf, t, sizeof(u64))) { ++ ret = -EFAULT; ++ } else if (ret > 0 && copy_to_user(buf, t, ret)) { ++ ret = -EFAULT; ++ } ++ memzero_explicit(t, sizeof(t)); ++ ++ return ret; ++} ++ ++ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr) ++{ ++ ssize_t ret = 0; ++ u8 tmpbuf[LRNG_DRNG_BLOCKSIZE] __aligned(LRNG_KCAPI_ALIGN); ++ u8 *tmp_large = NULL, *tmp = tmpbuf; ++ u32 tmplen = sizeof(tmpbuf); ++ ++ if (nbytes == 0) ++ return 0; ++ ++ /* ++ * Satisfy large read requests -- as the common case are smaller ++ * request sizes, such as 16 or 32 bytes, avoid a kmalloc overhead for ++ * those by using the stack variable of tmpbuf. ++ */ ++ if (!CONFIG_BASE_SMALL && (nbytes > sizeof(tmpbuf))) { ++ tmplen = min_t(u32, nbytes, LRNG_DRNG_MAX_REQSIZE); ++ tmp_large = kmalloc(tmplen + LRNG_KCAPI_ALIGN, GFP_KERNEL); ++ if (!tmp_large) ++ tmplen = sizeof(tmpbuf); ++ else ++ tmp = PTR_ALIGN(tmp_large, LRNG_KCAPI_ALIGN); ++ } ++ ++ while (nbytes) { ++ u32 todo = min_t(u32, nbytes, tmplen); ++ int rc = 0; ++ ++ /* Reschedule if we received a large request. */ ++ if ((tmp_large) && need_resched()) { ++ if (signal_pending(current)) { ++ if (ret == 0) ++ ret = -ERESTARTSYS; ++ break; ++ } ++ schedule(); ++ } ++ ++ rc = lrng_drng_get_sleep(tmp, todo, pr); ++ if (rc <= 0) { ++ if (rc < 0) ++ ret = rc; ++ break; ++ } ++ if (copy_to_user(buf, tmp, rc)) { ++ ret = -EFAULT; ++ break; ++ } ++ ++ nbytes -= rc; ++ buf += rc; ++ ret += rc; ++ } ++ ++ /* Wipe data just returned from memory */ ++ if (tmp_large) ++ kfree_sensitive(tmp_large); ++ else ++ memzero_explicit(tmpbuf, sizeof(tmpbuf)); ++ ++ return ret; ++} ++ ++ssize_t lrng_read_common_block(int nonblock, int pr, ++ char __user *buf, size_t nbytes) ++{ ++ int ret; ++ ++ if (nbytes == 0) ++ return 0; ++ ++ ret = lrng_drng_sleep_while_nonoperational(nonblock); ++ if (ret) ++ return ret; ++ ++ return lrng_read_common(buf, nbytes, !!pr); ++} ++ ++ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, ++ loff_t *ppos) ++{ ++ return lrng_read_common_block(file->f_flags & O_NONBLOCK, ++ file->f_flags & O_SYNC, buf, nbytes); ++} ++ ++__poll_t lrng_random_poll(struct file *file, poll_table *wait) ++{ ++ __poll_t mask; ++ ++ poll_wait(file, &lrng_init_wait, wait); ++ poll_wait(file, &lrng_write_wait, wait); ++ mask = 0; ++ if (lrng_state_operational()) ++ mask |= EPOLLIN | EPOLLRDNORM; ++ if (lrng_need_entropy() || ++ lrng_state_exseed_allow(lrng_noise_source_user)) { ++ lrng_state_exseed_set(lrng_noise_source_user, false); ++ mask |= EPOLLOUT | EPOLLWRNORM; ++ } ++ return mask; ++} ++ ++ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, ++ u32 entropy_bits) ++{ ++ ssize_t ret = 0; ++ u8 buf[64] __aligned(LRNG_KCAPI_ALIGN); ++ const char __user *p = buffer; ++ u32 orig_entropy_bits = entropy_bits; ++ ++ if (!lrng_get_available()) { ++ ret = lrng_drng_initalize(); ++ if (!ret) ++ return ret; ++ } ++ ++ count = min_t(size_t, count, INT_MAX); ++ while (count > 0) { ++ size_t bytes = min_t(size_t, count, sizeof(buf)); ++ u32 ent = min_t(u32, bytes<<3, entropy_bits); ++ ++ if (copy_from_user(&buf, p, bytes)) ++ return -EFAULT; ++ /* Inject data into entropy pool */ ++ lrng_pool_insert_aux(buf, bytes, ent); ++ ++ count -= bytes; ++ p += bytes; ++ ret += bytes; ++ entropy_bits -= ent; ++ ++ cond_resched(); ++ } ++ ++ /* Force reseed of DRNG during next data request. */ ++ if (!orig_entropy_bits) ++ lrng_drng_force_reseed(); ++ ++ return ret; ++} ++ ++ssize_t lrng_drng_write(struct file *file, const char __user *buffer, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_drng_write_common(buffer, count, 0); ++} ++ ++long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg) ++{ ++ u32 digestsize_bits; ++ int size, ent_count_bits; ++ int __user *p = (int __user *)arg; ++ ++ switch (cmd) { ++ case RNDGETENTCNT: ++ ent_count_bits = lrng_avail_entropy_aux(); ++ if (put_user(ent_count_bits, p)) ++ return -EFAULT; ++ return 0; ++ case RNDADDTOENTCNT: ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ if (get_user(ent_count_bits, p)) ++ return -EFAULT; ++ ent_count_bits = (int)lrng_get_aux_ent() + ent_count_bits; ++ if (ent_count_bits < 0) ++ ent_count_bits = 0; ++ digestsize_bits = lrng_get_digestsize(); ++ if (ent_count_bits > digestsize_bits) ++ ent_count_bits = digestsize_bits; ++ lrng_pool_set_entropy(ent_count_bits); ++ return 0; ++ case RNDADDENTROPY: ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ if (get_user(ent_count_bits, p++)) ++ return -EFAULT; ++ if (ent_count_bits < 0) ++ return -EINVAL; ++ if (get_user(size, p++)) ++ return -EFAULT; ++ if (size < 0) ++ return -EINVAL; ++ /* there cannot be more entropy than data */ ++ ent_count_bits = min(ent_count_bits, size<<3); ++ return lrng_drng_write_common((const char __user *)p, size, ++ ent_count_bits); ++ case RNDZAPENTCNT: ++ case RNDCLEARPOOL: ++ /* Clear the entropy pool counter. */ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ lrng_pool_set_entropy(0); ++ return 0; ++ case RNDRESEEDCRNG: ++ /* ++ * We leave the capability check here since it is present ++ * in the upstream's RNG implementation. Yet, user space ++ * can trigger a reseed as easy as writing into /dev/random ++ * or /dev/urandom where no privilege is needed. ++ */ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ /* Force a reseed of all DRNGs */ ++ lrng_drng_force_reseed(); ++ return 0; ++ default: ++ return -EINVAL; ++ } ++} ++EXPORT_SYMBOL(lrng_ioctl); ++ ++int lrng_fasync(int fd, struct file *filp, int on) ++{ ++ return fasync_helper(fd, filp, on, &fasync); ++} +diff --git a/drivers/char/lrng/lrng_interface_dev_common.h b/drivers/char/lrng/lrng_interface_dev_common.h +new file mode 100644 +index 000000000000..9e6603ad8af4 +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_dev_common.h +@@ -0,0 +1,51 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_INTERFACE_DEV_COMMON_H ++#define _LRNG_INTERFACE_DEV_COMMON_H ++ ++#include ++#include ++ ++/******************* Upstream functions hooked into the LRNG ******************/ ++enum lrng_external_noise_source { ++ lrng_noise_source_hw, ++ lrng_noise_source_user ++}; ++ ++#ifdef CONFIG_LRNG_COMMON_DEV_IF ++void lrng_writer_wakeup(void); ++void lrng_init_wakeup_dev(void); ++void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type); ++void lrng_state_exseed_allow_all(void); ++#else /* CONFIG_LRNG_COMMON_DEV_IF */ ++static inline void lrng_writer_wakeup(void) { } ++static inline void lrng_init_wakeup_dev(void) { } ++static inline void ++lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) { } ++static inline void lrng_state_exseed_allow_all(void) { } ++#endif /* CONFIG_LRNG_COMMON_DEV_IF */ ++ ++/****** Downstream service functions to actual interface implementations ******/ ++ ++bool lrng_state_exseed_allow(enum lrng_external_noise_source source); ++int lrng_fasync(int fd, struct file *filp, int on); ++long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg); ++ssize_t lrng_drng_write(struct file *file, const char __user *buffer, ++ size_t count, loff_t *ppos); ++ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, ++ u32 entropy_bits); ++__poll_t lrng_random_poll(struct file *file, poll_table *wait); ++ssize_t lrng_read_common_block(int nonblock, int pr, ++ char __user *buf, size_t nbytes); ++ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, ++ loff_t *ppos); ++ssize_t lrng_read_seed(char __user *buf, size_t nbytes, unsigned int flags); ++ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr); ++bool lrng_need_entropy(void); ++ ++extern struct wait_queue_head lrng_write_wait; ++ ++#endif /* _LRNG_INTERFACE_DEV_COMMON_H */ +diff --git a/drivers/char/lrng/lrng_interface_hwrand.c b/drivers/char/lrng/lrng_interface_hwrand.c +new file mode 100644 +index 000000000000..e841eea13348 +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_hwrand.c +@@ -0,0 +1,68 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG interface with the HW-Random framework ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#include ++#include ++#include ++ ++static int lrng_hwrand_if_random(struct hwrng *rng, void *buf, size_t max, ++ bool wait) ++{ ++ /* ++ * lrng_get_random_bytes_full not called as we cannot block. ++ * ++ * Note: We should either adjust .quality below depending on ++ * rng_is_initialized() or block here, but neither is not supported by ++ * the hw_rand framework. ++ */ ++ lrng_get_random_bytes(buf, max); ++ return (int)max; ++} ++ ++static struct hwrng lrng_hwrand = { ++ .name = "lrng", ++ .init = NULL, ++ .cleanup = NULL, ++ .read = lrng_hwrand_if_random, ++ ++ /* ++ * We set .quality only in case the LRNG does not provide the common ++ * interfaces or does not use the legacy RNG as entropy source. This ++ * shall avoid that the LRNG automatically spawns the hw_rand ++ * framework's hwrng kernel thread to feed data into ++ * add_hwgenerator_randomness. When the LRNG implements the common ++ * interfaces, this function feeds the data directly into the LRNG. ++ * If the LRNG uses the legacy RNG as entropy source, ++ * add_hwgenerator_randomness is implemented by the legacy RNG, but ++ * still eventually feeds the data into the LRNG. We should avoid such ++ * circular loops. ++ * ++ * We can specify full entropy here, because the LRNG is designed ++ * to provide full entropy. ++ */ ++#if !defined(CONFIG_LRNG_RANDOM_IF) && \ ++ !defined(CONFIG_LRNG_KERNEL_RNG) ++ .quality = 1024, ++#endif ++}; ++ ++static int __init lrng_hwrand_if_mod_init(void) ++{ ++ return hwrng_register(&lrng_hwrand); ++} ++ ++static void __exit lrng_hwrand_if_mod_exit(void) ++{ ++ hwrng_unregister(&lrng_hwrand); ++} ++ ++module_init(lrng_hwrand_if_mod_init); ++module_exit(lrng_hwrand_if_mod_exit); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_AUTHOR("Stephan Mueller "); ++MODULE_DESCRIPTION("Entropy Source and DRNG Manager HW-Random Interface"); +diff --git a/drivers/char/lrng/lrng_interface_kcapi.c b/drivers/char/lrng/lrng_interface_kcapi.c +new file mode 100644 +index 000000000000..4cb511f8088e +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_kcapi.c +@@ -0,0 +1,129 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG interface with the RNG framework of the kernel crypto API ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#include ++#include ++#include ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_aux.h" ++ ++static int lrng_kcapi_if_init(struct crypto_tfm *tfm) ++{ ++ return 0; ++} ++ ++static void lrng_kcapi_if_cleanup(struct crypto_tfm *tfm) { } ++ ++static int lrng_kcapi_if_reseed(const u8 *src, unsigned int slen) ++{ ++ int ret; ++ ++ if (!slen) ++ return 0; ++ ++ /* Insert caller-provided data without crediting entropy */ ++ ret = lrng_pool_insert_aux((u8 *)src, slen, 0); ++ if (ret) ++ return ret; ++ ++ /* Make sure the new data is immediately available to DRNG */ ++ lrng_drng_force_reseed(); ++ ++ return 0; ++} ++ ++static int lrng_kcapi_if_random(struct crypto_rng *tfm, ++ const u8 *src, unsigned int slen, ++ u8 *rdata, unsigned int dlen) ++{ ++ int ret = lrng_kcapi_if_reseed(src, slen); ++ ++ if (!ret) ++ lrng_get_random_bytes_full(rdata, dlen); ++ ++ return ret; ++} ++ ++static int lrng_kcapi_if_reset(struct crypto_rng *tfm, ++ const u8 *seed, unsigned int slen) ++{ ++ return lrng_kcapi_if_reseed(seed, slen); ++} ++ ++static struct rng_alg lrng_alg = { ++ .generate = lrng_kcapi_if_random, ++ .seed = lrng_kcapi_if_reset, ++ .seedsize = 0, ++ .base = { ++ .cra_name = "stdrng", ++ .cra_driver_name = "lrng", ++ .cra_priority = 500, ++ .cra_ctxsize = 0, ++ .cra_module = THIS_MODULE, ++ .cra_init = lrng_kcapi_if_init, ++ .cra_exit = lrng_kcapi_if_cleanup, ++ ++ } ++}; ++ ++#ifdef CONFIG_LRNG_DRNG_ATOMIC ++static int lrng_kcapi_if_random_atomic(struct crypto_rng *tfm, ++ const u8 *src, unsigned int slen, ++ u8 *rdata, unsigned int dlen) ++{ ++ int ret = lrng_kcapi_if_reseed(src, slen); ++ ++ if (!ret) ++ lrng_get_random_bytes(rdata, dlen); ++ ++ return ret; ++} ++ ++static struct rng_alg lrng_alg_atomic = { ++ .generate = lrng_kcapi_if_random_atomic, ++ .seed = lrng_kcapi_if_reset, ++ .seedsize = 0, ++ .base = { ++ .cra_name = "lrng_atomic", ++ .cra_driver_name = "lrng_atomic", ++ .cra_priority = 100, ++ .cra_ctxsize = 0, ++ .cra_module = THIS_MODULE, ++ .cra_init = lrng_kcapi_if_init, ++ .cra_exit = lrng_kcapi_if_cleanup, ++ ++ } ++}; ++#endif /* CONFIG_LRNG_DRNG_ATOMIC */ ++ ++static int __init lrng_kcapi_if_mod_init(void) ++{ ++ return ++#ifdef CONFIG_LRNG_DRNG_ATOMIC ++ crypto_register_rng(&lrng_alg_atomic) ?: ++#endif ++ crypto_register_rng(&lrng_alg); ++} ++ ++static void __exit lrng_kcapi_if_mod_exit(void) ++{ ++ crypto_unregister_rng(&lrng_alg); ++#ifdef CONFIG_LRNG_DRNG_ATOMIC ++ crypto_unregister_rng(&lrng_alg_atomic); ++#endif ++} ++ ++module_init(lrng_kcapi_if_mod_init); ++module_exit(lrng_kcapi_if_mod_exit); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_AUTHOR("Stephan Mueller "); ++MODULE_DESCRIPTION("Entropy Source and DRNG Manager kernel crypto API RNG framework interface"); ++MODULE_ALIAS_CRYPTO("lrng"); ++MODULE_ALIAS_CRYPTO("lrng_atomic"); ++MODULE_ALIAS_CRYPTO("stdrng"); +diff --git a/drivers/char/lrng/lrng_interface_random_kernel.c b/drivers/char/lrng/lrng_interface_random_kernel.c +new file mode 100644 +index 000000000000..471390bfd70c +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_random_kernel.c +@@ -0,0 +1,214 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Kernel space interfaces API/ABI compliant to linux/random.h ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "lrng_es_aux.h" ++#include "lrng_es_irq.h" ++#include "lrng_es_mgr.h" ++#include "lrng_interface_dev_common.h" ++#include "lrng_interface_random_kernel.h" ++ ++/********************************** Helper ***********************************/ ++ ++static bool lrng_trust_bootloader __initdata = ++ IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER); ++ ++static int __init lrng_parse_trust_bootloader(char *arg) ++{ ++ return kstrtobool(arg, &lrng_trust_bootloader); ++} ++early_param("random.trust_bootloader", lrng_parse_trust_bootloader); ++ ++int __init random_init(const char *command_line) ++{ ++ int ret = lrng_rand_initialize(); ++ ++ lrng_pool_insert_aux(command_line, strlen(command_line), 0); ++ return ret; ++} ++ ++/************************ LRNG kernel input interfaces ************************/ ++ ++/* ++ * add_hwgenerator_randomness() - Interface for in-kernel drivers of true ++ * hardware RNGs. ++ * ++ * Those devices may produce endless random bits and will be throttled ++ * when our pool is full. ++ * ++ * @buffer: buffer holding the entropic data from HW noise sources to be used to ++ * insert into entropy pool. ++ * @count: length of buffer ++ * @entropy_bits: amount of entropy in buffer (value is in bits) ++ */ ++void add_hwgenerator_randomness(const void *buffer, size_t count, ++ size_t entropy_bits) ++{ ++ /* ++ * Suspend writing if we are fully loaded with entropy. ++ * We'll be woken up again once below lrng_write_wakeup_thresh, ++ * or when the calling thread is about to terminate. ++ */ ++ wait_event_interruptible(lrng_write_wait, ++ lrng_need_entropy() || ++ lrng_state_exseed_allow(lrng_noise_source_hw) || ++ kthread_should_stop()); ++ lrng_state_exseed_set(lrng_noise_source_hw, false); ++ lrng_pool_insert_aux(buffer, count, entropy_bits); ++} ++EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); ++ ++/* ++ * add_bootloader_randomness() - Handle random seed passed by bootloader. ++ * ++ * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise ++ * it would be regarded as device data. ++ * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. ++ * ++ * @buf: buffer holding the entropic data from HW noise sources to be used to ++ * insert into entropy pool. ++ * @size: length of buffer ++ */ ++void __init add_bootloader_randomness(const void *buf, size_t size) ++{ ++ lrng_pool_insert_aux(buf, size, lrng_trust_bootloader ? size * 8 : 0); ++} ++ ++/* ++ * Callback for HID layer -- use the HID event values to stir the entropy pool ++ */ ++void add_input_randomness(unsigned int type, unsigned int code, ++ unsigned int value) ++{ ++ static unsigned char last_value; ++ ++ /* ignore autorepeat and the like */ ++ if (value == last_value) ++ return; ++ ++ last_value = value; ++ ++ lrng_irq_array_add_u32((type << 4) ^ code ^ (code >> 4) ^ value); ++} ++EXPORT_SYMBOL_GPL(add_input_randomness); ++ ++/* ++ * add_device_randomness() - Add device- or boot-specific data to the entropy ++ * pool to help initialize it. ++ * ++ * None of this adds any entropy; it is meant to avoid the problem of ++ * the entropy pool having similar initial state across largely ++ * identical devices. ++ * ++ * @buf: buffer holding the entropic data from HW noise sources to be used to ++ * insert into entropy pool. ++ * @size: length of buffer ++ */ ++void add_device_randomness(const void *buf, size_t size) ++{ ++ lrng_pool_insert_aux((u8 *)buf, size, 0); ++} ++EXPORT_SYMBOL(add_device_randomness); ++ ++#ifdef CONFIG_BLOCK ++void rand_initialize_disk(struct gendisk *disk) { } ++void add_disk_randomness(struct gendisk *disk) { } ++EXPORT_SYMBOL(add_disk_randomness); ++#endif ++ ++#ifndef CONFIG_LRNG_IRQ ++void add_interrupt_randomness(int irq) { } ++EXPORT_SYMBOL(add_interrupt_randomness); ++#endif ++ ++#if IS_ENABLED(CONFIG_VMGENID) ++static BLOCKING_NOTIFIER_HEAD(lrng_vmfork_chain); ++ ++/* ++ * Handle a new unique VM ID, which is unique, not secret, so we ++ * don't credit it, but we do immediately force a reseed after so ++ * that it's used by the crng posthaste. ++ */ ++void add_vmfork_randomness(const void *unique_vm_id, size_t size) ++{ ++ add_device_randomness(unique_vm_id, size); ++ if (lrng_state_operational()) ++ lrng_drng_force_reseed(); ++ blocking_notifier_call_chain(&lrng_vmfork_chain, 0, NULL); ++} ++#if IS_MODULE(CONFIG_VMGENID) ++EXPORT_SYMBOL_GPL(add_vmfork_randomness); ++#endif ++ ++int register_random_vmfork_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_register(&lrng_vmfork_chain, nb); ++} ++EXPORT_SYMBOL_GPL(register_random_vmfork_notifier); ++ ++int unregister_random_vmfork_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_unregister(&lrng_vmfork_chain, nb); ++} ++EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier); ++#endif ++ ++/*********************** LRNG kernel output interfaces ************************/ ++ ++/* ++ * get_random_bytes() - Provider of cryptographic strong random numbers for ++ * kernel-internal usage. ++ * ++ * This function is appropriate for all in-kernel use cases. However, ++ * it will always use the ChaCha20 DRNG. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ */ ++void get_random_bytes(void *buf, size_t nbytes) ++{ ++ lrng_get_random_bytes(buf, nbytes); ++} ++EXPORT_SYMBOL(get_random_bytes); ++ ++/* ++ * wait_for_random_bytes() - Wait for the LRNG to be seeded and thus ++ * guaranteed to supply cryptographically secure random numbers. ++ * ++ * This applies to: the /dev/urandom device, the get_random_bytes function, ++ * and the get_random_{u32,u64,int,long} family of functions. Using any of ++ * these functions without first calling this function forfeits the guarantee ++ * of security. ++ * ++ * Return: ++ * * 0 if the LRNG has been seeded. ++ * * -ERESTARTSYS if the function was interrupted by a signal. ++ */ ++int wait_for_random_bytes(void) ++{ ++ return lrng_drng_sleep_while_non_min_seeded(); ++} ++EXPORT_SYMBOL(wait_for_random_bytes); ++ ++/* ++ * Returns whether or not the LRNG has been seeded. ++ * ++ * Returns: true if the urandom pool has been seeded. ++ * false if the urandom pool has not been seeded. ++ */ ++bool rng_is_initialized(void) ++{ ++ return lrng_state_operational(); ++} ++EXPORT_SYMBOL(rng_is_initialized); +diff --git a/drivers/char/lrng/lrng_interface_random_kernel.h b/drivers/char/lrng/lrng_interface_random_kernel.h +new file mode 100644 +index 000000000000..f4d09d1fbb14 +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_random_kernel.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_INTERFACE_RANDOM_H ++#define _LRNG_INTERFACE_RANDOM_H ++ ++#ifdef CONFIG_LRNG_RANDOM_IF ++void invalidate_batched_entropy(void); ++#else /* CONFIG_LRNG_RANDOM_IF */ ++static inline void invalidate_batched_entropy(void) { } ++#endif /* CONFIG_LRNG_RANDOM_IF */ ++ ++#endif /* _LRNG_INTERFACE_RANDOM_H */ +diff --git a/drivers/char/lrng/lrng_interface_random_user.c b/drivers/char/lrng/lrng_interface_random_user.c +new file mode 100644 +index 000000000000..d12e883804d9 +--- /dev/null ++++ b/drivers/char/lrng/lrng_interface_random_user.c +@@ -0,0 +1,104 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG Common user space interfaces compliant to random(4), random(7) and ++ * getrandom(2) man pages. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++ ++#include "lrng_es_mgr.h" ++#include "lrng_interface_dev_common.h" ++ ++static ssize_t lrng_drng_read(struct file *file, char __user *buf, ++ size_t nbytes, loff_t *ppos) ++{ ++ if (!lrng_state_min_seeded()) ++ pr_notice_ratelimited("%s - use of insufficiently seeded DRNG (%zu bytes read)\n", ++ current->comm, nbytes); ++ else if (!lrng_state_operational()) ++ pr_debug_ratelimited("%s - use of not fully seeded DRNG (%zu bytes read)\n", ++ current->comm, nbytes); ++ ++ return lrng_read_common(buf, nbytes, false); ++} ++ ++const struct file_operations random_fops = { ++ .read = lrng_drng_read_block, ++ .write = lrng_drng_write, ++ .poll = lrng_random_poll, ++ .unlocked_ioctl = lrng_ioctl, ++ .compat_ioctl = compat_ptr_ioctl, ++ .fasync = lrng_fasync, ++ .llseek = noop_llseek, ++}; ++ ++const struct file_operations urandom_fops = { ++ .read = lrng_drng_read, ++ .write = lrng_drng_write, ++ .unlocked_ioctl = lrng_ioctl, ++ .compat_ioctl = compat_ptr_ioctl, ++ .fasync = lrng_fasync, ++ .llseek = noop_llseek, ++}; ++ ++/* ++ * GRND_SEED ++ * ++ * This flag requests to provide the data directly from the entropy sources. ++ * ++ * The behavior of the call is exactly as outlined for the function ++ * lrng_get_seed in lrng.h. ++ */ ++#define GRND_SEED 0x0010 ++ ++/* ++ * GRND_FULLY_SEEDED ++ * ++ * This flag indicates whether the caller wants to reseed a DRNG that is already ++ * fully seeded. See esdm_get_seed in lrng.h for details. ++ */ ++#define GRND_FULLY_SEEDED 0x0020 ++ ++SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, ++ unsigned int, flags) ++{ ++ if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE| ++ GRND_SEED|GRND_FULLY_SEEDED)) ++ return -EINVAL; ++ ++ /* ++ * Requesting insecure and blocking randomness at the same time makes ++ * no sense. ++ */ ++ if ((flags & ++ (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) ++ return -EINVAL; ++ if ((flags & ++ (GRND_INSECURE|GRND_SEED)) == (GRND_INSECURE|GRND_SEED)) ++ return -EINVAL; ++ if ((flags & ++ (GRND_RANDOM|GRND_SEED)) == (GRND_RANDOM|GRND_SEED)) ++ return -EINVAL; ++ ++ if (count > INT_MAX) ++ count = INT_MAX; ++ ++ if (flags & GRND_INSECURE) { ++ return lrng_drng_read(NULL, buf, count, NULL); ++ } else if (flags & GRND_SEED) { ++ unsigned int seed_flags = (flags & GRND_NONBLOCK) ? ++ LRNG_GET_SEED_NONBLOCK : 0; ++ ++ seed_flags |= (flags & GRND_FULLY_SEEDED) ? ++ LRNG_GET_SEED_FULLY_SEEDED : 0; ++ return lrng_read_seed(buf, count, seed_flags); ++ } ++ ++ return lrng_read_common_block(flags & GRND_NONBLOCK, ++ flags & GRND_RANDOM, buf, count); ++} +diff --git a/drivers/char/lrng/lrng_numa.c b/drivers/char/lrng/lrng_numa.c +new file mode 100644 +index 000000000000..d74dd8df2843 +--- /dev/null ++++ b/drivers/char/lrng/lrng_numa.c +@@ -0,0 +1,124 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG NUMA support ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_irq.h" ++#include "lrng_es_mgr.h" ++#include "lrng_numa.h" ++#include "lrng_proc.h" ++ ++static struct lrng_drng **lrng_drng __read_mostly = NULL; ++ ++struct lrng_drng **lrng_drng_instances(void) ++{ ++ /* counterpart to cmpxchg_release in _lrng_drngs_numa_alloc */ ++ return READ_ONCE(lrng_drng); ++} ++ ++/* Allocate the data structures for the per-NUMA node DRNGs */ ++static void _lrng_drngs_numa_alloc(struct work_struct *work) ++{ ++ struct lrng_drng **drngs; ++ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ u32 node; ++ bool init_drng_used = false; ++ ++ mutex_lock(&lrng_crypto_cb_update); ++ ++ /* per-NUMA-node DRNGs are already present */ ++ if (lrng_drng) ++ goto unlock; ++ ++ /* Make sure the initial DRNG is initialized and its drng_cb is set */ ++ if (lrng_drng_initalize()) ++ goto err; ++ ++ drngs = kcalloc(nr_node_ids, sizeof(void *), GFP_KERNEL|__GFP_NOFAIL); ++ for_each_online_node(node) { ++ struct lrng_drng *drng; ++ ++ if (!init_drng_used) { ++ drngs[node] = lrng_drng_init; ++ init_drng_used = true; ++ continue; ++ } ++ ++ drng = kmalloc_node(sizeof(struct lrng_drng), ++ GFP_KERNEL|__GFP_NOFAIL, node); ++ memset(drng, 0, sizeof(lrng_drng)); ++ ++ if (lrng_drng_alloc_common(drng, lrng_drng_init->drng_cb)) { ++ kfree(drng); ++ goto err; ++ } ++ ++ drng->hash_cb = lrng_drng_init->hash_cb; ++ drng->hash = lrng_drng_init->hash_cb->hash_alloc(); ++ if (IS_ERR(drng->hash)) { ++ lrng_drng_init->drng_cb->drng_dealloc(drng->drng); ++ kfree(drng); ++ goto err; ++ } ++ ++ mutex_init(&drng->lock); ++ rwlock_init(&drng->hash_lock); ++ ++ /* ++ * No reseeding of NUMA DRNGs from previous DRNGs as this ++ * would complicate the code. Let it simply reseed. ++ */ ++ drngs[node] = drng; ++ ++ lrng_pool_inc_numa_node(); ++ pr_info("DRNG and entropy pool read hash for NUMA node %d allocated\n", ++ node); ++ } ++ ++ /* counterpart to READ_ONCE in lrng_drng_instances */ ++ if (!cmpxchg_release(&lrng_drng, NULL, drngs)) { ++ lrng_pool_all_numa_nodes_seeded(false); ++ goto unlock; ++ } ++ ++err: ++ for_each_online_node(node) { ++ struct lrng_drng *drng = drngs[node]; ++ ++ if (drng == lrng_drng_init) ++ continue; ++ ++ if (drng) { ++ drng->hash_cb->hash_dealloc(drng->hash); ++ drng->drng_cb->drng_dealloc(drng->drng); ++ kfree(drng); ++ } ++ } ++ kfree(drngs); ++ ++unlock: ++ mutex_unlock(&lrng_crypto_cb_update); ++} ++ ++static DECLARE_WORK(lrng_drngs_numa_alloc_work, _lrng_drngs_numa_alloc); ++ ++static void lrng_drngs_numa_alloc(void) ++{ ++ schedule_work(&lrng_drngs_numa_alloc_work); ++} ++ ++static int __init lrng_numa_init(void) ++{ ++ lrng_drngs_numa_alloc(); ++ return 0; ++} ++ ++late_initcall(lrng_numa_init); +diff --git a/drivers/char/lrng/lrng_numa.h b/drivers/char/lrng/lrng_numa.h +new file mode 100644 +index 000000000000..dc8dff9816ee +--- /dev/null ++++ b/drivers/char/lrng/lrng_numa.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_NUMA_H ++#define _LRNG_NUMA_H ++ ++#ifdef CONFIG_NUMA ++struct lrng_drng **lrng_drng_instances(void); ++#else /* CONFIG_NUMA */ ++static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; } ++#endif /* CONFIG_NUMA */ ++ ++#endif /* _LRNG_NUMA_H */ +diff --git a/drivers/char/lrng/lrng_proc.c b/drivers/char/lrng/lrng_proc.c +new file mode 100644 +index 000000000000..92329cc6ef28 +--- /dev/null ++++ b/drivers/char/lrng/lrng_proc.c +@@ -0,0 +1,74 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG proc interfaces ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_aux.h" ++#include "lrng_es_mgr.h" ++#include "lrng_proc.h" ++ ++/* Number of online DRNGs */ ++static u32 numa_drngs = 1; ++ ++void lrng_pool_inc_numa_node(void) ++{ ++ numa_drngs++; ++} ++ ++static int lrng_proc_type_show(struct seq_file *m, void *v) ++{ ++ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ unsigned char buf[270]; ++ u32 i; ++ ++ mutex_lock(&lrng_drng_init->lock); ++ snprintf(buf, sizeof(buf), ++ "DRNG name: %s\n" ++ "LRNG security strength in bits: %d\n" ++ "Number of DRNG instances: %u\n" ++ "Standards compliance: %s%s\n" ++ "LRNG minimally seeded: %s\n" ++ "LRNG fully seeded: %s\n" ++ "LRNG entropy level: %u\n", ++ lrng_drng_init->drng_cb->drng_name(), ++ lrng_security_strength(), ++ numa_drngs, ++ lrng_sp80090c_compliant() ? "SP800-90C " : "", ++ lrng_ntg1_compliant() ? "NTG.1 " : "", ++ lrng_state_min_seeded() ? "true" : "false", ++ lrng_state_fully_seeded() ? "true" : "false", ++ lrng_avail_entropy()); ++ seq_write(m, buf, strlen(buf)); ++ ++ for_each_lrng_es(i) { ++ snprintf(buf, sizeof(buf), ++ "Entropy Source %u properties:\n" ++ " Name: %s\n", ++ i, lrng_es[i]->name); ++ seq_write(m, buf, strlen(buf)); ++ ++ buf[0] = '\0'; ++ lrng_es[i]->state(buf, sizeof(buf)); ++ seq_write(m, buf, strlen(buf)); ++ } ++ ++ mutex_unlock(&lrng_drng_init->lock); ++ ++ return 0; ++} ++ ++static int __init lrng_proc_type_init(void) ++{ ++ proc_create_single("lrng_type", 0444, NULL, &lrng_proc_type_show); ++ return 0; ++} ++ ++module_init(lrng_proc_type_init); +diff --git a/drivers/char/lrng/lrng_proc.h b/drivers/char/lrng/lrng_proc.h +new file mode 100644 +index 000000000000..c653274f1954 +--- /dev/null ++++ b/drivers/char/lrng/lrng_proc.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_PROC_H ++#define _LRNG_PROC_H ++ ++#ifdef CONFIG_SYSCTL ++void lrng_pool_inc_numa_node(void); ++#else ++static inline void lrng_pool_inc_numa_node(void) { } ++#endif ++ ++#endif /* _LRNG_PROC_H */ +diff --git a/drivers/char/lrng/lrng_selftest.c b/drivers/char/lrng/lrng_selftest.c +new file mode 100644 +index 000000000000..15f1e4a2a719 +--- /dev/null ++++ b/drivers/char/lrng/lrng_selftest.c +@@ -0,0 +1,397 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG power-on and on-demand self-test ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++/* ++ * In addition to the self-tests below, the following LRNG components ++ * are covered with self-tests during regular operation: ++ * ++ * * power-on self-test: SP800-90A DRBG provided by the Linux kernel crypto API ++ * * power-on self-test: PRNG provided by the Linux kernel crypto API ++ * * runtime test: Raw noise source data testing including SP800-90B compliant ++ * tests when enabling CONFIG_LRNG_HEALTH_TESTS ++ * ++ * Additional developer tests present with LRNG code: ++ * * SP800-90B APT and RCT test enforcement validation when enabling ++ * CONFIG_LRNG_APT_BROKEN or CONFIG_LRNG_RCT_BROKEN. ++ * * Collection of raw entropy from the interrupt noise source when enabling ++ * CONFIG_LRNG_TESTING and pulling the data from the kernel with the provided ++ * interface. ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++ ++#include "lrng_drng_chacha20.h" ++#include "lrng_sha.h" ++ ++#define LRNG_SELFTEST_PASSED 0 ++#define LRNG_SEFLTEST_ERROR_TIME (1 << 0) ++#define LRNG_SEFLTEST_ERROR_CHACHA20 (1 << 1) ++#define LRNG_SEFLTEST_ERROR_HASH (1 << 2) ++#define LRNG_SEFLTEST_ERROR_GCD (1 << 3) ++#define LRNG_SELFTEST_NOT_EXECUTED 0xffffffff ++ ++#ifdef CONFIG_LRNG_TIMER_COMMON ++ ++#include "lrng_es_timer_common.h" ++ ++static u32 lrng_data_selftest_ptr = 0; ++static u32 lrng_data_selftest[LRNG_DATA_ARRAY_SIZE]; ++ ++static void lrng_data_process_selftest_insert(u32 time) ++{ ++ u32 ptr = lrng_data_selftest_ptr++ & LRNG_DATA_WORD_MASK; ++ unsigned int array = lrng_data_idx2array(ptr); ++ unsigned int slot = lrng_data_idx2slot(ptr); ++ ++ /* zeroization of slot to ensure the following OR adds the data */ ++ lrng_data_selftest[array] &= ++ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, ++ slot)); ++ lrng_data_selftest[array] |= ++ lrng_data_slot_val(time & LRNG_DATA_SLOTSIZE_MASK, slot); ++} ++ ++static void lrng_data_process_selftest_u32(u32 data) ++{ ++ u32 pre_ptr, ptr, mask; ++ unsigned int pre_array; ++ ++ /* Increment pointer by number of slots taken for input value */ ++ lrng_data_selftest_ptr += LRNG_DATA_SLOTS_PER_UINT; ++ ++ /* ptr to current unit */ ++ ptr = lrng_data_selftest_ptr; ++ ++ lrng_data_split_u32(&ptr, &pre_ptr, &mask); ++ ++ /* MSB of data go into previous unit */ ++ pre_array = lrng_data_idx2array(pre_ptr); ++ /* zeroization of slot to ensure the following OR adds the data */ ++ lrng_data_selftest[pre_array] &= ~(0xffffffff & ~mask); ++ lrng_data_selftest[pre_array] |= data & ~mask; ++ ++ /* LSB of data go into current unit */ ++ lrng_data_selftest[lrng_data_idx2array(ptr)] = data & mask; ++} ++ ++static unsigned int lrng_data_process_selftest(void) ++{ ++ u32 time; ++ u32 idx_zero_compare = (0 << 0) | (1 << 8) | (2 << 16) | (3 << 24); ++ u32 idx_one_compare = (4 << 0) | (5 << 8) | (6 << 16) | (7 << 24); ++ u32 idx_last_compare = ++ (((LRNG_DATA_NUM_VALUES - 4) & LRNG_DATA_SLOTSIZE_MASK) << 0) | ++ (((LRNG_DATA_NUM_VALUES - 3) & LRNG_DATA_SLOTSIZE_MASK) << 8) | ++ (((LRNG_DATA_NUM_VALUES - 2) & LRNG_DATA_SLOTSIZE_MASK) << 16) | ++ (((LRNG_DATA_NUM_VALUES - 1) & LRNG_DATA_SLOTSIZE_MASK) << 24); ++ ++ (void)idx_one_compare; ++ ++ /* "poison" the array to verify the operation of the zeroization */ ++ lrng_data_selftest[0] = 0xffffffff; ++ lrng_data_selftest[1] = 0xffffffff; ++ ++ lrng_data_process_selftest_insert(0); ++ /* ++ * Note, when using lrng_data_process_u32() on unaligned ptr, ++ * the first slots will go into next word, and the last slots go ++ * into the previous word. ++ */ ++ lrng_data_process_selftest_u32((4 << 0) | (1 << 8) | (2 << 16) | ++ (3 << 24)); ++ lrng_data_process_selftest_insert(5); ++ lrng_data_process_selftest_insert(6); ++ lrng_data_process_selftest_insert(7); ++ ++ if ((lrng_data_selftest[0] != idx_zero_compare) || ++ (lrng_data_selftest[1] != idx_one_compare)) ++ goto err; ++ ++ /* Reset for next test */ ++ lrng_data_selftest[0] = 0; ++ lrng_data_selftest[1] = 0; ++ lrng_data_selftest_ptr = 0; ++ ++ for (time = 0; time < LRNG_DATA_NUM_VALUES; time++) ++ lrng_data_process_selftest_insert(time); ++ ++ if ((lrng_data_selftest[0] != idx_zero_compare) || ++ (lrng_data_selftest[1] != idx_one_compare) || ++ (lrng_data_selftest[LRNG_DATA_ARRAY_SIZE - 1] != idx_last_compare)) ++ goto err; ++ ++ return LRNG_SELFTEST_PASSED; ++ ++err: ++ pr_err("LRNG data array self-test FAILED\n"); ++ return LRNG_SEFLTEST_ERROR_TIME; ++} ++ ++static unsigned int lrng_gcd_selftest(void) ++{ ++ u32 history[10]; ++ unsigned int i; ++ ++#define LRNG_GCD_SELFTEST 3 ++ for (i = 0; i < ARRAY_SIZE(history); i++) ++ history[i] = i * LRNG_GCD_SELFTEST; ++ ++ if (lrng_gcd_analyze(history, ARRAY_SIZE(history)) == LRNG_GCD_SELFTEST) ++ return LRNG_SELFTEST_PASSED; ++ ++ pr_err("LRNG GCD self-test FAILED\n"); ++ return LRNG_SEFLTEST_ERROR_GCD; ++} ++ ++#else /* CONFIG_LRNG_TIMER_COMMON */ ++ ++static unsigned int lrng_data_process_selftest(void) ++{ ++ return LRNG_SELFTEST_PASSED; ++} ++ ++static unsigned int lrng_gcd_selftest(void) ++{ ++ return LRNG_SELFTEST_PASSED; ++} ++ ++#endif /* CONFIG_LRNG_TIMER_COMMON */ ++ ++/* The test vectors are taken from crypto/testmgr.h */ ++static unsigned int lrng_hash_selftest(void) ++{ ++ SHASH_DESC_ON_STACK(shash, NULL); ++ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; ++ static const u8 lrng_hash_selftest_result[] = ++#ifdef CONFIG_CRYPTO_LIB_SHA256 ++ { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, ++ 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, ++ 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, ++ 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad }; ++#else /* CONFIG_CRYPTO_LIB_SHA256 */ ++ { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e, ++ 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d }; ++#endif /* CONFIG_CRYPTO_LIB_SHA256 */ ++ static const u8 hash_input[] = { 0x61, 0x62, 0x63 }; /* "abc" */ ++ u8 digest[sizeof(lrng_hash_selftest_result)] __aligned(sizeof(u32)); ++ ++ if (sizeof(digest) != hash_cb->hash_digestsize(NULL)) ++ return LRNG_SEFLTEST_ERROR_HASH; ++ ++ if (!hash_cb->hash_init(shash, NULL) && ++ !hash_cb->hash_update(shash, hash_input, ++ sizeof(hash_input)) && ++ !hash_cb->hash_final(shash, digest) && ++ !memcmp(digest, lrng_hash_selftest_result, sizeof(digest))) ++ return 0; ++ ++ pr_err("LRNG %s Hash self-test FAILED\n", hash_cb->hash_name()); ++ return LRNG_SEFLTEST_ERROR_HASH; ++} ++ ++#ifdef CONFIG_LRNG_DRNG_CHACHA20 ++ ++static void lrng_selftest_bswap32(u32 *ptr, u32 words) ++{ ++ u32 i; ++ ++ /* Byte-swap data which is an LE representation */ ++ for (i = 0; i < words; i++) { ++ __le32 *p = (__le32 *)ptr; ++ ++ *p = cpu_to_le32(*ptr); ++ ptr++; ++ } ++} ++ ++/* ++ * The test vectors were generated using the ChaCha20 DRNG from ++ * https://www.chronox.de/chacha20.html ++ */ ++static unsigned int lrng_chacha20_drng_selftest(void) ++{ ++ const struct lrng_drng_cb *drng_cb = &lrng_cc20_drng_cb; ++ u8 seed[CHACHA_KEY_SIZE * 2] = { ++ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ++ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, ++ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, ++ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ++ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, ++ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, ++ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, ++ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, ++ }; ++ struct chacha20_block chacha20; ++ int ret; ++ u8 outbuf[CHACHA_KEY_SIZE * 2] __aligned(sizeof(u32)); ++ ++ /* ++ * Expected result when ChaCha20 DRNG state is zero: ++ * * constants are set to "expand 32-byte k" ++ * * remaining state is 0 ++ * and pulling one half ChaCha20 DRNG block. ++ */ ++ static const u8 expected_halfblock[CHACHA_KEY_SIZE] = { ++ 0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, ++ 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86, 0xbd, 0x28, ++ 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, ++ 0xa8, 0x36, 0xef, 0xcc, 0x8b, 0x77, 0x0d, 0xc7 }; ++ ++ /* ++ * Expected result when ChaCha20 DRNG state is zero: ++ * * constants are set to "expand 32-byte k" ++ * * remaining state is 0 ++ * followed by a reseed with two keyblocks ++ * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ++ * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, ++ * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, ++ * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ++ * 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, ++ * 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, ++ * 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, ++ * 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f ++ * and pulling one ChaCha20 DRNG block. ++ */ ++ static const u8 expected_oneblock[CHACHA_KEY_SIZE * 2] = { ++ 0xe3, 0xb0, 0x8a, 0xcc, 0x34, 0xc3, 0x17, 0x0e, ++ 0xc3, 0xd8, 0xc3, 0x40, 0xe7, 0x73, 0xe9, 0x0d, ++ 0xd1, 0x62, 0xa3, 0x5d, 0x7d, 0xf2, 0xf1, 0x4a, ++ 0x24, 0x42, 0xb7, 0x1e, 0xb0, 0x05, 0x17, 0x07, ++ 0xb9, 0x35, 0x10, 0x69, 0x8b, 0x46, 0xfb, 0x51, ++ 0xe9, 0x91, 0x3f, 0x46, 0xf2, 0x4d, 0xea, 0xd0, ++ 0x81, 0xc1, 0x1b, 0xa9, 0x5d, 0x52, 0x91, 0x5f, ++ 0xcd, 0xdc, 0xc6, 0xd6, 0xc3, 0x7c, 0x50, 0x23 }; ++ ++ /* ++ * Expected result when ChaCha20 DRNG state is zero: ++ * * constants are set to "expand 32-byte k" ++ * * remaining state is 0 ++ * followed by a reseed with one key block plus one byte ++ * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ++ * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, ++ * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, ++ * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ++ * 0x20 ++ * and pulling less than one ChaCha20 DRNG block. ++ */ ++ static const u8 expected_block_nonalinged[CHACHA_KEY_SIZE + 4] = { ++ 0x9c, 0xfc, 0x5e, 0x31, 0x21, 0x62, 0x11, 0x85, ++ 0xd3, 0x77, 0xd3, 0x69, 0x0f, 0xa8, 0x16, 0x55, ++ 0xb4, 0x4c, 0xf6, 0x52, 0xf3, 0xa8, 0x37, 0x99, ++ 0x38, 0x76, 0xa0, 0x66, 0xec, 0xbb, 0xce, 0xa9, ++ 0x9c, 0x95, 0xa1, 0xfd }; ++ ++ BUILD_BUG_ON(sizeof(seed) % sizeof(u32)); ++ ++ memset(&chacha20, 0, sizeof(chacha20)); ++ lrng_cc20_init_rfc7539(&chacha20); ++ lrng_selftest_bswap32((u32 *)seed, sizeof(seed) / sizeof(u32)); ++ ++ /* Generate with zero state */ ++ ret = drng_cb->drng_generate(&chacha20, outbuf, ++ sizeof(expected_halfblock)); ++ if (ret != sizeof(expected_halfblock)) ++ goto err; ++ if (memcmp(outbuf, expected_halfblock, sizeof(expected_halfblock))) ++ goto err; ++ ++ /* Clear state of DRNG */ ++ memset(&chacha20.key.u[0], 0, 48); ++ ++ /* Reseed with 2 key blocks */ ++ ret = drng_cb->drng_seed(&chacha20, seed, sizeof(expected_oneblock)); ++ if (ret < 0) ++ goto err; ++ ret = drng_cb->drng_generate(&chacha20, outbuf, ++ sizeof(expected_oneblock)); ++ if (ret != sizeof(expected_oneblock)) ++ goto err; ++ if (memcmp(outbuf, expected_oneblock, sizeof(expected_oneblock))) ++ goto err; ++ ++ /* Clear state of DRNG */ ++ memset(&chacha20.key.u[0], 0, 48); ++ ++ /* Reseed with 1 key block and one byte */ ++ ret = drng_cb->drng_seed(&chacha20, seed, ++ sizeof(expected_block_nonalinged)); ++ if (ret < 0) ++ goto err; ++ ret = drng_cb->drng_generate(&chacha20, outbuf, ++ sizeof(expected_block_nonalinged)); ++ if (ret != sizeof(expected_block_nonalinged)) ++ goto err; ++ if (memcmp(outbuf, expected_block_nonalinged, ++ sizeof(expected_block_nonalinged))) ++ goto err; ++ ++ return LRNG_SELFTEST_PASSED; ++ ++err: ++ pr_err("LRNG ChaCha20 DRNG self-test FAILED\n"); ++ return LRNG_SEFLTEST_ERROR_CHACHA20; ++} ++ ++#else /* CONFIG_LRNG_DRNG_CHACHA20 */ ++ ++static unsigned int lrng_chacha20_drng_selftest(void) ++{ ++ return LRNG_SELFTEST_PASSED; ++} ++ ++#endif /* CONFIG_LRNG_DRNG_CHACHA20 */ ++ ++static unsigned int lrng_selftest_status = LRNG_SELFTEST_NOT_EXECUTED; ++ ++static int lrng_selftest(void) ++{ ++ unsigned int ret = lrng_data_process_selftest(); ++ ++ ret |= lrng_chacha20_drng_selftest(); ++ ret |= lrng_hash_selftest(); ++ ret |= lrng_gcd_selftest(); ++ ++ if (ret) { ++ if (IS_ENABLED(CONFIG_LRNG_SELFTEST_PANIC)) ++ panic("LRNG self-tests failed: %u\n", ret); ++ } else { ++ pr_info("LRNG self-tests passed\n"); ++ } ++ ++ lrng_selftest_status = ret; ++ ++ if (lrng_selftest_status) ++ return -EFAULT; ++ return 0; ++} ++ ++#ifdef CONFIG_SYSFS ++/* Re-perform self-test when any value is written to the sysfs file. */ ++static int lrng_selftest_sysfs_set(const char *val, ++ const struct kernel_param *kp) ++{ ++ return lrng_selftest(); ++} ++ ++static const struct kernel_param_ops lrng_selftest_sysfs = { ++ .set = lrng_selftest_sysfs_set, ++ .get = param_get_uint, ++}; ++module_param_cb(selftest_status, &lrng_selftest_sysfs, &lrng_selftest_status, ++ 0644); ++#endif /* CONFIG_SYSFS */ ++ ++static int __init lrng_selftest_init(void) ++{ ++ return lrng_selftest(); ++} ++ ++module_init(lrng_selftest_init); +diff --git a/drivers/char/lrng/lrng_sha.h b/drivers/char/lrng/lrng_sha.h +new file mode 100644 +index 000000000000..d2f134f54773 +--- /dev/null ++++ b/drivers/char/lrng/lrng_sha.h +@@ -0,0 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * LRNG SHA definition usable in atomic contexts right from the start of the ++ * kernel. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_SHA_H ++#define _LRNG_SHA_H ++ ++extern const struct lrng_hash_cb lrng_sha_hash_cb; ++ ++#endif /* _LRNG_SHA_H */ +diff --git a/drivers/char/lrng/lrng_sha1.c b/drivers/char/lrng/lrng_sha1.c +new file mode 100644 +index 000000000000..9cbc7a6fee49 +--- /dev/null ++++ b/drivers/char/lrng/lrng_sha1.c +@@ -0,0 +1,88 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Backend for the LRNG providing the SHA-1 implementation that can be used ++ * without the kernel crypto API available including during early boot and in ++ * atomic contexts. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++ ++#include "lrng_sha.h" ++ ++/* ++ * If the SHA-256 support is not compiled, we fall back to SHA-1 that is always ++ * compiled and present in the kernel. ++ */ ++static u32 lrng_sha1_hash_digestsize(void *hash) ++{ ++ return SHA1_DIGEST_SIZE; ++} ++ ++static void lrng_sha1_block_fn(struct sha1_state *sctx, const u8 *src, ++ int blocks) ++{ ++ u32 temp[SHA1_WORKSPACE_WORDS]; ++ ++ while (blocks--) { ++ sha1_transform(sctx->state, src, temp); ++ src += SHA1_BLOCK_SIZE; ++ } ++ memzero_explicit(temp, sizeof(temp)); ++} ++ ++static int lrng_sha1_hash_init(struct shash_desc *shash, void *hash) ++{ ++ /* ++ * We do not need a TFM - we only need sufficient space for ++ * struct sha1_state on the stack. ++ */ ++ sha1_base_init(shash); ++ return 0; ++} ++ ++static int lrng_sha1_hash_update(struct shash_desc *shash, ++ const u8 *inbuf, u32 inbuflen) ++{ ++ return sha1_base_do_update(shash, inbuf, inbuflen, lrng_sha1_block_fn); ++} ++ ++static int lrng_sha1_hash_final(struct shash_desc *shash, u8 *digest) ++{ ++ return sha1_base_do_finalize(shash, lrng_sha1_block_fn) ?: ++ sha1_base_finish(shash, digest); ++} ++ ++static const char *lrng_sha1_hash_name(void) ++{ ++ return "SHA-1"; ++} ++ ++static void lrng_sha1_hash_desc_zero(struct shash_desc *shash) ++{ ++ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha1_state)); ++} ++ ++static void *lrng_sha1_hash_alloc(void) ++{ ++ pr_info("Hash %s allocated\n", lrng_sha1_hash_name()); ++ return NULL; ++} ++ ++static void lrng_sha1_hash_dealloc(void *hash) { } ++ ++const struct lrng_hash_cb lrng_sha_hash_cb = { ++ .hash_name = lrng_sha1_hash_name, ++ .hash_alloc = lrng_sha1_hash_alloc, ++ .hash_dealloc = lrng_sha1_hash_dealloc, ++ .hash_digestsize = lrng_sha1_hash_digestsize, ++ .hash_init = lrng_sha1_hash_init, ++ .hash_update = lrng_sha1_hash_update, ++ .hash_final = lrng_sha1_hash_final, ++ .hash_desc_zero = lrng_sha1_hash_desc_zero, ++}; +diff --git a/drivers/char/lrng/lrng_sha256.c b/drivers/char/lrng/lrng_sha256.c +new file mode 100644 +index 000000000000..50705351a71c +--- /dev/null ++++ b/drivers/char/lrng/lrng_sha256.c +@@ -0,0 +1,72 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Backend for the LRNG providing the SHA-256 implementation that can be used ++ * without the kernel crypto API available including during early boot and in ++ * atomic contexts. ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++ ++#include "lrng_sha.h" ++ ++static u32 lrng_sha256_hash_digestsize(void *hash) ++{ ++ return SHA256_DIGEST_SIZE; ++} ++ ++static int lrng_sha256_hash_init(struct shash_desc *shash, void *hash) ++{ ++ /* ++ * We do not need a TFM - we only need sufficient space for ++ * struct sha256_state on the stack. ++ */ ++ sha256_init(shash_desc_ctx(shash)); ++ return 0; ++} ++ ++static int lrng_sha256_hash_update(struct shash_desc *shash, ++ const u8 *inbuf, u32 inbuflen) ++{ ++ sha256_update(shash_desc_ctx(shash), inbuf, inbuflen); ++ return 0; ++} ++ ++static int lrng_sha256_hash_final(struct shash_desc *shash, u8 *digest) ++{ ++ sha256_final(shash_desc_ctx(shash), digest); ++ return 0; ++} ++ ++static const char *lrng_sha256_hash_name(void) ++{ ++ return "SHA-256"; ++} ++ ++static void lrng_sha256_hash_desc_zero(struct shash_desc *shash) ++{ ++ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha256_state)); ++} ++ ++static void *lrng_sha256_hash_alloc(void) ++{ ++ pr_info("Hash %s allocated\n", lrng_sha256_hash_name()); ++ return NULL; ++} ++ ++static void lrng_sha256_hash_dealloc(void *hash) { } ++ ++const struct lrng_hash_cb lrng_sha_hash_cb = { ++ .hash_name = lrng_sha256_hash_name, ++ .hash_alloc = lrng_sha256_hash_alloc, ++ .hash_dealloc = lrng_sha256_hash_dealloc, ++ .hash_digestsize = lrng_sha256_hash_digestsize, ++ .hash_init = lrng_sha256_hash_init, ++ .hash_update = lrng_sha256_hash_update, ++ .hash_final = lrng_sha256_hash_final, ++ .hash_desc_zero = lrng_sha256_hash_desc_zero, ++}; +diff --git a/drivers/char/lrng/lrng_switch.c b/drivers/char/lrng/lrng_switch.c +new file mode 100644 +index 000000000000..aae75594926f +--- /dev/null ++++ b/drivers/char/lrng/lrng_switch.c +@@ -0,0 +1,286 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG DRNG switching support ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++ ++#include "lrng_es_aux.h" ++#include "lrng_es_mgr.h" ++#include "lrng_interface_dev_common.h" ++#include "lrng_numa.h" ++ ++static int __maybe_unused ++lrng_hash_switch(struct lrng_drng *drng_store, const void *cb, int node) ++{ ++ const struct lrng_hash_cb *new_cb = (const struct lrng_hash_cb *)cb; ++ const struct lrng_hash_cb *old_cb = drng_store->hash_cb; ++ unsigned long flags; ++ u32 i; ++ void *new_hash, *old_hash; ++ int ret; ++ ++ if (node == -1) ++ return 0; ++ ++ new_hash = new_cb->hash_alloc(); ++ old_hash = drng_store->hash; ++ ++ if (IS_ERR(new_hash)) { ++ pr_warn("could not allocate new LRNG pool hash (%ld)\n", ++ PTR_ERR(new_hash)); ++ return PTR_ERR(new_hash); ++ } ++ ++ if (new_cb->hash_digestsize(new_hash) > LRNG_MAX_DIGESTSIZE) { ++ pr_warn("digest size of newly requested hash too large\n"); ++ new_cb->hash_dealloc(new_hash); ++ return -EINVAL; ++ } ++ ++ write_lock_irqsave(&drng_store->hash_lock, flags); ++ ++ /* Trigger the switch for each entropy source */ ++ for_each_lrng_es(i) { ++ if (!lrng_es[i]->switch_hash) ++ continue; ++ ret = lrng_es[i]->switch_hash(drng_store, node, new_cb, ++ new_hash, old_cb); ++ if (ret) { ++ u32 j; ++ ++ /* Revert all already executed operations */ ++ for (j = 0; j < i; j++) { ++ if (!lrng_es[j]->switch_hash) ++ continue; ++ WARN_ON(lrng_es[j]->switch_hash(drng_store, ++ node, old_cb, ++ old_hash, ++ new_cb)); ++ } ++ goto err; ++ } ++ } ++ ++ drng_store->hash = new_hash; ++ drng_store->hash_cb = new_cb; ++ old_cb->hash_dealloc(old_hash); ++ pr_info("Conditioning function allocated for DRNG for NUMA node %d\n", ++ node); ++ ++err: ++ write_unlock_irqrestore(&drng_store->hash_lock, flags); ++ return ret; ++} ++ ++static int __maybe_unused ++lrng_drng_switch(struct lrng_drng *drng_store, const void *cb, int node) ++{ ++ const struct lrng_drng_cb *new_cb = (const struct lrng_drng_cb *)cb; ++ const struct lrng_drng_cb *old_cb = drng_store->drng_cb; ++ int ret; ++ u8 seed[LRNG_DRNG_SECURITY_STRENGTH_BYTES]; ++ void *new_drng = new_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); ++ void *old_drng = drng_store->drng; ++ u32 current_security_strength; ++ bool reset_drng = !lrng_get_available(); ++ ++ if (IS_ERR(new_drng)) { ++ pr_warn("could not allocate new DRNG for NUMA node %d (%ld)\n", ++ node, PTR_ERR(new_drng)); ++ return PTR_ERR(new_drng); ++ } ++ ++ current_security_strength = lrng_security_strength(); ++ mutex_lock(&drng_store->lock); ++ ++ /* ++ * Pull from existing DRNG to seed new DRNG regardless of seed status ++ * of old DRNG -- the entropy state for the DRNG is left unchanged which ++ * implies that als the new DRNG is reseeded when deemed necessary. This ++ * seeding of the new DRNG shall only ensure that the new DRNG has the ++ * same entropy as the old DRNG. ++ */ ++ ret = old_cb->drng_generate(old_drng, seed, sizeof(seed)); ++ mutex_unlock(&drng_store->lock); ++ ++ if (ret < 0) { ++ reset_drng = true; ++ pr_warn("getting random data from DRNG failed for NUMA node %d (%d)\n", ++ node, ret); ++ } else { ++ /* seed new DRNG with data */ ++ ret = new_cb->drng_seed(new_drng, seed, ret); ++ memzero_explicit(seed, sizeof(seed)); ++ if (ret < 0) { ++ reset_drng = true; ++ pr_warn("seeding of new DRNG failed for NUMA node %d (%d)\n", ++ node, ret); ++ } else { ++ pr_debug("seeded new DRNG of NUMA node %d instance from old DRNG instance\n", ++ node); ++ } ++ } ++ ++ mutex_lock(&drng_store->lock); ++ ++ if (reset_drng) ++ lrng_drng_reset(drng_store); ++ ++ drng_store->drng = new_drng; ++ drng_store->drng_cb = new_cb; ++ ++ /* Reseed if previous LRNG security strength was insufficient */ ++ if (current_security_strength < lrng_security_strength()) ++ drng_store->force_reseed = true; ++ ++ /* Force oversampling seeding as we initialize DRNG */ ++ if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) ++ lrng_unset_fully_seeded(drng_store); ++ ++ if (lrng_state_min_seeded()) ++ lrng_set_entropy_thresh(lrng_get_seed_entropy_osr( ++ drng_store->fully_seeded)); ++ ++ old_cb->drng_dealloc(old_drng); ++ ++ pr_info("DRNG of NUMA node %d switched\n", node); ++ ++ mutex_unlock(&drng_store->lock); ++ return ret; ++} ++ ++/* ++ * Switch the existing DRNG and hash instances with new using the new crypto ++ * callbacks. The caller must hold the lrng_crypto_cb_update lock. ++ */ ++static int lrng_switch(const void *cb, ++ int(*switcher)(struct lrng_drng *drng_store, ++ const void *cb, int node)) ++{ ++ struct lrng_drng **lrng_drng = lrng_drng_instances(); ++ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ struct lrng_drng *lrng_drng_pr = lrng_drng_pr_instance(); ++ int ret = 0; ++ ++ if (lrng_drng) { ++ u32 node; ++ ++ for_each_online_node(node) { ++ if (lrng_drng[node]) ++ ret |= switcher(lrng_drng[node], cb, node); ++ } ++ } else { ++ ret |= switcher(lrng_drng_init, cb, 0); ++ } ++ ++ ret |= switcher(lrng_drng_pr, cb, -1); ++ ++ return ret; ++} ++ ++/* ++ * lrng_set_drng_cb - Register new cryptographic callback functions for DRNG ++ * The registering implies that all old DRNG states are replaced with new ++ * DRNG states. ++ * ++ * drng_cb: Callback functions to be registered -- if NULL, use the default ++ * callbacks defined at compile time. ++ * ++ * Return: ++ * * 0 on success ++ * * < 0 on error ++ */ ++int lrng_set_drng_cb(const struct lrng_drng_cb *drng_cb) ++{ ++ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_LRNG_SWITCH_DRNG)) ++ return -EOPNOTSUPP; ++ ++ if (!drng_cb) ++ drng_cb = lrng_default_drng_cb; ++ ++ mutex_lock(&lrng_crypto_cb_update); ++ ++ /* ++ * If a callback other than the default is set, allow it only to be ++ * set back to the default callback. This ensures that multiple ++ * different callbacks can be registered at the same time. If a ++ * callback different from the current callback and the default ++ * callback shall be set, the current callback must be deregistered ++ * (e.g. the kernel module providing it must be unloaded) and the new ++ * implementation can be registered. ++ */ ++ if ((drng_cb != lrng_default_drng_cb) && ++ (lrng_drng_init->drng_cb != lrng_default_drng_cb)) { ++ pr_warn("disallow setting new DRNG callbacks, unload the old callbacks first!\n"); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ ret = lrng_switch(drng_cb, lrng_drng_switch); ++ /* The switch may imply new entropy due to larger DRNG sec strength. */ ++ if (!ret) ++ lrng_es_add_entropy(); ++ ++out: ++ mutex_unlock(&lrng_crypto_cb_update); ++ return ret; ++} ++EXPORT_SYMBOL(lrng_set_drng_cb); ++ ++/* ++ * lrng_set_hash_cb - Register new cryptographic callback functions for hash ++ * The registering implies that all old hash states are replaced with new ++ * hash states. ++ * ++ * @hash_cb: Callback functions to be registered -- if NULL, use the default ++ * callbacks defined at compile time. ++ * ++ * Return: ++ * * 0 on success ++ * * < 0 on error ++ */ ++int lrng_set_hash_cb(const struct lrng_hash_cb *hash_cb) ++{ ++ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_LRNG_SWITCH_HASH)) ++ return -EOPNOTSUPP; ++ ++ if (!hash_cb) ++ hash_cb = lrng_default_hash_cb; ++ ++ mutex_lock(&lrng_crypto_cb_update); ++ ++ /* Comment from lrng_set_drng_cb applies. */ ++ if ((hash_cb != lrng_default_hash_cb) && ++ (lrng_drng_init->hash_cb != lrng_default_hash_cb)) { ++ pr_warn("disallow setting new hash callbacks, unload the old callbacks first!\n"); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ ret = lrng_switch(hash_cb, lrng_hash_switch); ++ /* ++ * The switch may imply new entropy due to larger digest size. But ++ * it may also offer more room in the aux pool which means we ping ++ * any waiting entropy providers. ++ */ ++ if (!ret) { ++ lrng_es_add_entropy(); ++ lrng_writer_wakeup(); ++ } ++ ++out: ++ mutex_unlock(&lrng_crypto_cb_update); ++ return ret; ++} ++EXPORT_SYMBOL(lrng_set_hash_cb); +diff --git a/drivers/char/lrng/lrng_sysctl.c b/drivers/char/lrng/lrng_sysctl.c +new file mode 100644 +index 000000000000..ecdd96a842b4 +--- /dev/null ++++ b/drivers/char/lrng/lrng_sysctl.c +@@ -0,0 +1,140 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG sysctl interfaces ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include "lrng_drng_mgr.h" ++#include "lrng_es_mgr.h" ++#include "lrng_sysctl.h" ++ ++/* ++ * This function is used to return both the bootid UUID, and random ++ * UUID. The difference is in whether table->data is NULL; if it is, ++ * then a new UUID is generated and returned to the user. ++ * ++ * If the user accesses this via the proc interface, the UUID will be ++ * returned as an ASCII string in the standard UUID format; if via the ++ * sysctl system call, as 16 bytes of binary data. ++ */ ++static int lrng_sysctl_do_uuid(struct ctl_table *table, int write, ++ void *buffer, size_t *lenp, loff_t *ppos) ++{ ++ struct ctl_table fake_table; ++ unsigned char buf[64], tmp_uuid[16], *uuid; ++ ++ uuid = table->data; ++ if (!uuid) { ++ uuid = tmp_uuid; ++ generate_random_uuid(uuid); ++ } else { ++ static DEFINE_SPINLOCK(bootid_spinlock); ++ ++ spin_lock(&bootid_spinlock); ++ if (!uuid[8]) ++ generate_random_uuid(uuid); ++ spin_unlock(&bootid_spinlock); ++ } ++ ++ sprintf(buf, "%pU", uuid); ++ ++ fake_table.data = buf; ++ fake_table.maxlen = sizeof(buf); ++ ++ return proc_dostring(&fake_table, write, buffer, lenp, ppos); ++} ++ ++static int lrng_sysctl_do_entropy(struct ctl_table *table, int write, ++ void *buffer, size_t *lenp, loff_t *ppos) ++{ ++ struct ctl_table fake_table; ++ int entropy_count = lrng_avail_entropy_aux(); ++ ++ fake_table.data = &entropy_count; ++ fake_table.maxlen = sizeof(entropy_count); ++ ++ return proc_dointvec(&fake_table, write, buffer, lenp, ppos); ++} ++ ++static int lrng_sysctl_do_poolsize(struct ctl_table *table, int write, ++ void *buffer, size_t *lenp, loff_t *ppos) ++{ ++ struct ctl_table fake_table; ++ u32 entropy_count = lrng_es[lrng_ext_es_aux]->max_entropy(); ++ ++ fake_table.data = &entropy_count; ++ fake_table.maxlen = sizeof(entropy_count); ++ ++ return proc_dointvec(&fake_table, write, buffer, lenp, ppos); ++} ++ ++static int lrng_min_write_thresh; ++static int lrng_max_write_thresh = (LRNG_WRITE_WAKEUP_ENTROPY << 3); ++static char lrng_sysctl_bootid[16]; ++static int lrng_drng_reseed_max_min; ++ ++void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) ++{ ++ lrng_max_write_thresh = (int)new_digestsize; ++ /* Ensure that changes to the global variable are visible */ ++ mb(); ++} ++ ++static struct ctl_table random_table[] = { ++ { ++ .procname = "poolsize", ++ .maxlen = sizeof(int), ++ .mode = 0444, ++ .proc_handler = lrng_sysctl_do_poolsize, ++ }, ++ { ++ .procname = "entropy_avail", ++ .maxlen = sizeof(int), ++ .mode = 0444, ++ .proc_handler = lrng_sysctl_do_entropy, ++ }, ++ { ++ .procname = "write_wakeup_threshold", ++ .data = &lrng_write_wakeup_bits, ++ .maxlen = sizeof(int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec_minmax, ++ .extra1 = &lrng_min_write_thresh, ++ .extra2 = &lrng_max_write_thresh, ++ }, ++ { ++ .procname = "boot_id", ++ .data = &lrng_sysctl_bootid, ++ .maxlen = 16, ++ .mode = 0444, ++ .proc_handler = lrng_sysctl_do_uuid, ++ }, ++ { ++ .procname = "uuid", ++ .maxlen = 16, ++ .mode = 0444, ++ .proc_handler = lrng_sysctl_do_uuid, ++ }, ++ { ++ .procname = "urandom_min_reseed_secs", ++ .data = &lrng_drng_reseed_max_time, ++ .maxlen = sizeof(int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec, ++ .extra1 = &lrng_drng_reseed_max_min, ++ }, ++ { } ++}; ++ ++static int __init random_sysctls_init(void) ++{ ++ register_sysctl_init("kernel/random", random_table); ++ return 0; ++} ++device_initcall(random_sysctls_init); +diff --git a/drivers/char/lrng/lrng_sysctl.h b/drivers/char/lrng/lrng_sysctl.h +new file mode 100644 +index 000000000000..4b487e5077ed +--- /dev/null ++++ b/drivers/char/lrng/lrng_sysctl.h +@@ -0,0 +1,15 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_SYSCTL_H ++#define _LRNG_SYSCTL_H ++ ++#ifdef CONFIG_LRNG_SYSCTL ++void lrng_sysctl_update_max_write_thresh(u32 new_digestsize); ++#else ++static inline void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) { } ++#endif ++ ++#endif /* _LRNG_SYSCTL_H */ +diff --git a/drivers/char/lrng/lrng_testing.c b/drivers/char/lrng/lrng_testing.c +new file mode 100644 +index 000000000000..101140085d81 +--- /dev/null ++++ b/drivers/char/lrng/lrng_testing.c +@@ -0,0 +1,901 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * LRNG testing interfaces to obtain raw entropy ++ * ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "lrng_definitions.h" ++#include "lrng_drng_chacha20.h" ++#include "lrng_sha.h" ++#include "lrng_testing.h" ++ ++#if defined(CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY) || \ ++ defined(CONFIG_LRNG_RAW_SCHED_PID_ENTROPY) || \ ++ defined(CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY) || \ ++ defined(CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY) || \ ++ defined(CONFIG_LRNG_SCHED_PERF) ++#define LRNG_TESTING_USE_BUSYLOOP ++#endif ++ ++#ifdef CONFIG_LRNG_TESTING_RECORDING ++ ++#define LRNG_TESTING_RINGBUFFER_SIZE 1024 ++#define LRNG_TESTING_RINGBUFFER_MASK (LRNG_TESTING_RINGBUFFER_SIZE - 1) ++ ++struct lrng_testing { ++ u32 lrng_testing_rb[LRNG_TESTING_RINGBUFFER_SIZE]; ++ u32 rb_reader; ++ atomic_t rb_writer; ++ atomic_t lrng_testing_enabled; ++ spinlock_t lock; ++ wait_queue_head_t read_wait; ++}; ++ ++/*************************** Generic Data Handling ****************************/ ++ ++/* ++ * boot variable: ++ * 0 ==> No boot test, gathering of runtime data allowed ++ * 1 ==> Boot test enabled and ready for collecting data, gathering runtime ++ * data is disabled ++ * 2 ==> Boot test completed and disabled, gathering of runtime data is ++ * disabled ++ */ ++ ++static void lrng_testing_reset(struct lrng_testing *data) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&data->lock, flags); ++ data->rb_reader = 0; ++ atomic_set(&data->rb_writer, 0); ++ spin_unlock_irqrestore(&data->lock, flags); ++} ++ ++static void lrng_testing_init(struct lrng_testing *data, u32 boot) ++{ ++ /* ++ * The boot time testing implies we have a running test. If the ++ * caller wants to clear it, he has to unset the boot_test flag ++ * at runtime via sysfs to enable regular runtime testing ++ */ ++ if (boot) ++ return; ++ ++ lrng_testing_reset(data); ++ atomic_set(&data->lrng_testing_enabled, 1); ++ pr_warn("Enabling data collection\n"); ++} ++ ++static void lrng_testing_fini(struct lrng_testing *data, u32 boot) ++{ ++ /* If we have boot data, we do not reset yet to allow data to be read */ ++ if (boot) ++ return; ++ ++ atomic_set(&data->lrng_testing_enabled, 0); ++ lrng_testing_reset(data); ++ pr_warn("Disabling data collection\n"); ++} ++ ++static bool lrng_testing_store(struct lrng_testing *data, u32 value, ++ u32 *boot) ++{ ++ unsigned long flags; ++ ++ if (!atomic_read(&data->lrng_testing_enabled) && (*boot != 1)) ++ return false; ++ ++ spin_lock_irqsave(&data->lock, flags); ++ ++ /* ++ * Disable entropy testing for boot time testing after ring buffer ++ * is filled. ++ */ ++ if (*boot) { ++ if (((u32)atomic_read(&data->rb_writer)) > ++ LRNG_TESTING_RINGBUFFER_SIZE) { ++ *boot = 2; ++ pr_warn_once("One time data collection test disabled\n"); ++ spin_unlock_irqrestore(&data->lock, flags); ++ return false; ++ } ++ ++ if (atomic_read(&data->rb_writer) == 1) ++ pr_warn("One time data collection test enabled\n"); ++ } ++ ++ data->lrng_testing_rb[((u32)atomic_read(&data->rb_writer)) & ++ LRNG_TESTING_RINGBUFFER_MASK] = value; ++ atomic_inc(&data->rb_writer); ++ ++ spin_unlock_irqrestore(&data->lock, flags); ++ ++#ifndef LRNG_TESTING_USE_BUSYLOOP ++ if (wq_has_sleeper(&data->read_wait)) ++ wake_up_interruptible(&data->read_wait); ++#endif ++ ++ return true; ++} ++ ++static bool lrng_testing_have_data(struct lrng_testing *data) ++{ ++ return ((((u32)atomic_read(&data->rb_writer)) & ++ LRNG_TESTING_RINGBUFFER_MASK) != ++ (data->rb_reader & LRNG_TESTING_RINGBUFFER_MASK)); ++} ++ ++static int lrng_testing_reader(struct lrng_testing *data, u32 *boot, ++ u8 *outbuf, u32 outbuflen) ++{ ++ unsigned long flags; ++ int collected_data = 0; ++ ++ lrng_testing_init(data, *boot); ++ ++ while (outbuflen) { ++ u32 writer = (u32)atomic_read(&data->rb_writer); ++ ++ spin_lock_irqsave(&data->lock, flags); ++ ++ /* We have no data or reached the writer. */ ++ if (!writer || (writer == data->rb_reader)) { ++ ++ spin_unlock_irqrestore(&data->lock, flags); ++ ++ /* ++ * Now we gathered all boot data, enable regular data ++ * collection. ++ */ ++ if (*boot) { ++ *boot = 0; ++ goto out; ++ } ++ ++#ifdef LRNG_TESTING_USE_BUSYLOOP ++ while (!lrng_testing_have_data(data)) ++ ; ++#else ++ wait_event_interruptible(data->read_wait, ++ lrng_testing_have_data(data)); ++#endif ++ if (signal_pending(current)) { ++ collected_data = -ERESTARTSYS; ++ goto out; ++ } ++ ++ continue; ++ } ++ ++ /* We copy out word-wise */ ++ if (outbuflen < sizeof(u32)) { ++ spin_unlock_irqrestore(&data->lock, flags); ++ goto out; ++ } ++ ++ memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], ++ sizeof(u32)); ++ data->rb_reader++; ++ ++ spin_unlock_irqrestore(&data->lock, flags); ++ ++ outbuf += sizeof(u32); ++ outbuflen -= sizeof(u32); ++ collected_data += sizeof(u32); ++ } ++ ++out: ++ lrng_testing_fini(data, *boot); ++ return collected_data; ++} ++ ++static int lrng_testing_extract_user(struct file *file, char __user *buf, ++ size_t nbytes, loff_t *ppos, ++ int (*reader)(u8 *outbuf, u32 outbuflen)) ++{ ++ u8 *tmp, *tmp_aligned; ++ int ret = 0, large_request = (nbytes > 256); ++ ++ if (!nbytes) ++ return 0; ++ ++ /* ++ * The intention of this interface is for collecting at least ++ * 1000 samples due to the SP800-90B requirements. So, we make no ++ * effort in avoiding allocating more memory that actually needed ++ * by the user. Hence, we allocate sufficient memory to always hold ++ * that amount of data. ++ */ ++ tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); ++ if (!tmp) ++ return -ENOMEM; ++ ++ tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); ++ ++ while (nbytes) { ++ int i; ++ ++ if (large_request && need_resched()) { ++ if (signal_pending(current)) { ++ if (ret == 0) ++ ret = -ERESTARTSYS; ++ break; ++ } ++ schedule(); ++ } ++ ++ i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); ++ i = reader(tmp_aligned, i); ++ if (i <= 0) { ++ if (i < 0) ++ ret = i; ++ break; ++ } ++ if (copy_to_user(buf, tmp_aligned, i)) { ++ ret = -EFAULT; ++ break; ++ } ++ ++ nbytes -= i; ++ buf += i; ++ ret += i; ++ } ++ ++ kfree_sensitive(tmp); ++ ++ if (ret > 0) ++ *ppos += ret; ++ ++ return ret; ++} ++ ++#endif /* CONFIG_LRNG_TESTING_RECORDING */ ++ ++/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ ++ ++#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY ++ ++static u32 boot_raw_hires_test = 0; ++module_param(boot_raw_hires_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); ++ ++static struct lrng_testing lrng_raw_hires = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) ++}; ++ ++bool lrng_raw_hires_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); ++} ++ ++static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, ++ outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_hires_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_hires_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_hires_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ ++ ++/********************* Raw Jiffies Entropy Data Handling **********************/ ++ ++#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY ++ ++static u32 boot_raw_jiffies_test = 0; ++module_param(boot_raw_jiffies_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); ++ ++static struct lrng_testing lrng_raw_jiffies = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) ++}; ++ ++bool lrng_raw_jiffies_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_jiffies, value, ++ &boot_raw_jiffies_test); ++} ++ ++static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, ++ outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_jiffies_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_jiffies_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_jiffies_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ ++ ++/************************** Raw IRQ Data Handling ****************************/ ++ ++#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY ++ ++static u32 boot_raw_irq_test = 0; ++module_param(boot_raw_irq_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); ++ ++static struct lrng_testing lrng_raw_irq = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) ++}; ++ ++bool lrng_raw_irq_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); ++} ++ ++static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, ++ outbuflen); ++} ++ ++static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_irq_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_irq_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_irq_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ ++ ++/************************ Raw _RET_IP_ Data Handling **************************/ ++ ++#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY ++ ++static u32 boot_raw_retip_test = 0; ++module_param(boot_raw_retip_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); ++ ++static struct lrng_testing lrng_raw_retip = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) ++}; ++ ++bool lrng_raw_retip_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); ++} ++ ++static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, ++ outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_retip_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_retip_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_retip_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ ++ ++/********************** Raw IRQ register Data Handling ************************/ ++ ++#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY ++ ++static u32 boot_raw_regs_test = 0; ++module_param(boot_raw_regs_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); ++ ++static struct lrng_testing lrng_raw_regs = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) ++}; ++ ++bool lrng_raw_regs_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); ++} ++ ++static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, ++ outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_regs_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_regs_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_regs_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ ++ ++/********************** Raw Entropy Array Data Handling ***********************/ ++ ++#ifdef CONFIG_LRNG_RAW_ARRAY ++ ++static u32 boot_raw_array = 0; ++module_param(boot_raw_array, uint, 0644); ++MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); ++ ++static struct lrng_testing lrng_raw_array = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) ++}; ++ ++bool lrng_raw_array_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); ++} ++ ++static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, ++ outbuflen); ++} ++ ++static ssize_t lrng_raw_array_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_array_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_array_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_array_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_ARRAY */ ++ ++/******************** Interrupt Performance Data Handling *********************/ ++ ++#ifdef CONFIG_LRNG_IRQ_PERF ++ ++static u32 boot_irq_perf = 0; ++module_param(boot_irq_perf, uint, 0644); ++MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); ++ ++static struct lrng_testing lrng_irq_perf = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) ++}; ++ ++bool lrng_perf_time(u32 start) ++{ ++ return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, ++ &boot_irq_perf); ++} ++ ++static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, ++ outbuflen); ++} ++ ++static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_irq_perf_reader); ++} ++ ++static const struct file_operations lrng_irq_perf_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_irq_perf_read, ++}; ++ ++#endif /* CONFIG_LRNG_IRQ_PERF */ ++ ++/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY ++ ++static u32 boot_raw_sched_hires_test = 0; ++module_param(boot_raw_sched_hires_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); ++ ++static struct lrng_testing lrng_raw_sched_hires = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), ++ .read_wait = ++ __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) ++}; ++ ++bool lrng_raw_sched_hires_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_sched_hires, value, ++ &boot_raw_sched_hires_test); ++} ++ ++static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_sched_hires, ++ &boot_raw_sched_hires_test, ++ outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_hires_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_sched_hires_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_hires_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ ++ ++/******************** Interrupt Performance Data Handling *********************/ ++ ++#ifdef CONFIG_LRNG_SCHED_PERF ++ ++static u32 boot_sched_perf = 0; ++module_param(boot_sched_perf, uint, 0644); ++MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); ++ ++static struct lrng_testing lrng_sched_perf = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) ++}; ++ ++bool lrng_sched_perf_time(u32 start) ++{ ++ return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, ++ &boot_sched_perf); ++} ++ ++static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, ++ outbuflen); ++} ++ ++static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_sched_perf_reader); ++} ++ ++static const struct file_operations lrng_sched_perf_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_sched_perf_read, ++}; ++ ++#endif /* CONFIG_LRNG_SCHED_PERF */ ++ ++/*************** Raw Scheduler task_struct->pid Data Handling *****************/ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++ ++static u32 boot_raw_sched_pid_test = 0; ++module_param(boot_raw_sched_pid_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); ++ ++static struct lrng_testing lrng_raw_sched_pid = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) ++}; ++ ++bool lrng_raw_sched_pid_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_sched_pid, value, ++ &boot_raw_sched_pid_test); ++} ++ ++static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_sched_pid, ++ &boot_raw_sched_pid_test, outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_pid_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_sched_pid_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_pid_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ ++ ++ ++/*********** Raw Scheduler task_struct->start_time Data Handling **************/ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++ ++static u32 boot_raw_sched_starttime_test = 0; ++module_param(boot_raw_sched_starttime_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); ++ ++static struct lrng_testing lrng_raw_sched_starttime = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) ++}; ++ ++bool lrng_raw_sched_starttime_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_sched_starttime, value, ++ &boot_raw_sched_starttime_test); ++} ++ ++static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_sched_starttime, ++ &boot_raw_sched_starttime_test, outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_starttime_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_sched_starttime_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_starttime_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++ ++/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++ ++static u32 boot_raw_sched_nvcsw_test = 0; ++module_param(boot_raw_sched_nvcsw_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); ++ ++static struct lrng_testing lrng_raw_sched_nvcsw = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) ++}; ++ ++bool lrng_raw_sched_nvcsw_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_sched_nvcsw, value, ++ &boot_raw_sched_nvcsw_test); ++} ++ ++static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_sched_nvcsw, ++ &boot_raw_sched_nvcsw_test, outbuf, outbuflen); ++} ++ ++static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_nvcsw_entropy_reader); ++} ++ ++static const struct file_operations lrng_raw_sched_nvcsw_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_nvcsw_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ ++ ++/*********************************** ACVT ************************************/ ++ ++#ifdef CONFIG_LRNG_ACVT_HASH ++ ++/* maximum amount of data to be hashed as defined by ACVP */ ++#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) ++ ++/* ++ * As we use static variables to store the data, it is clear that the ++ * test interface is only able to handle single threaded testing. This is ++ * considered to be sufficient for testing. If multi-threaded use of the ++ * ACVT test interface would be performed, the caller would get garbage ++ * but the kernel operation is unaffected by this. ++ */ ++static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] ++ __aligned(LRNG_KCAPI_ALIGN); ++static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); ++static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; ++ ++static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, ++ size_t nbytes, loff_t *ppos) ++{ ++ if (nbytes > LRNG_ACVT_MAX_SHA_MSG) ++ return -EINVAL; ++ ++ atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); ++ ++ return simple_write_to_buffer(lrng_acvt_hash_data, ++ LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); ++} ++ ++static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ SHASH_DESC_ON_STACK(shash, NULL); ++ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; ++ ssize_t ret; ++ ++ if (count > LRNG_ATOMIC_DIGEST_SIZE) ++ return -EINVAL; ++ ++ ret = hash_cb->hash_init(shash, NULL) ?: ++ hash_cb->hash_update(shash, lrng_acvt_hash_data, ++ atomic_read_u32(&lrng_acvt_hash_data_size)) ?: ++ hash_cb->hash_final(shash, lrng_acvt_hash_digest); ++ if (ret) ++ return ret; ++ ++ return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, ++ sizeof(lrng_acvt_hash_digest)); ++} ++ ++static const struct file_operations lrng_acvt_hash_fops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .llseek = default_llseek, ++ .read = lrng_acvt_hash_read, ++ .write = lrng_acvt_hash_write, ++}; ++ ++#endif /* CONFIG_LRNG_ACVT_DRNG */ ++ ++/************************************************************************** ++ * Debugfs interface ++ **************************************************************************/ ++ ++static int __init lrng_raw_init(void) ++{ ++ struct dentry *lrng_raw_debugfs_root; ++ ++ lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); ++ ++#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_hires", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_hires_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_jiffies_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, ++ NULL, &lrng_raw_irq_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_retip", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_retip_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_regs", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_regs_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_ARRAY ++ debugfs_create_file_unsafe("lrng_raw_array", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_array_fops); ++#endif ++#ifdef CONFIG_LRNG_IRQ_PERF ++ debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, ++ NULL, &lrng_irq_perf_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, ++ lrng_raw_debugfs_root, ++ NULL, &lrng_raw_sched_hires_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_sched_pid_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_sched_starttime_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_sched_nvcsw_fops); ++#endif ++#ifdef CONFIG_LRNG_SCHED_PERF ++ debugfs_create_file_unsafe("lrng_sched_perf", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_sched_perf_fops); ++#endif ++#ifdef CONFIG_LRNG_ACVT_HASH ++ debugfs_create_file_unsafe("lrng_acvt_hash", 0600, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_acvt_hash_fops); ++#endif ++ ++ return 0; ++} ++ ++module_init(lrng_raw_init); +diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h +new file mode 100644 +index 000000000000..672a7fca4595 +--- /dev/null ++++ b/drivers/char/lrng/lrng_testing.h +@@ -0,0 +1,85 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_TESTING_H ++#define _LRNG_TESTING_H ++ ++#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY ++bool lrng_raw_hires_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ ++static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY ++bool lrng_raw_jiffies_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ ++static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY ++bool lrng_raw_irq_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ ++static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY ++bool lrng_raw_retip_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ ++static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY ++bool lrng_raw_regs_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ ++static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_ARRAY ++bool lrng_raw_array_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_ARRAY */ ++static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_ARRAY */ ++ ++#ifdef CONFIG_LRNG_IRQ_PERF ++bool lrng_perf_time(u32 start); ++#else /* CONFIG_LRNG_IRQ_PERF */ ++static inline bool lrng_perf_time(u32 start) { return false; } ++#endif /*CONFIG_LRNG_IRQ_PERF */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY ++bool lrng_raw_sched_hires_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ ++static inline bool ++lrng_raw_sched_hires_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++bool lrng_raw_sched_pid_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ ++static inline bool ++lrng_raw_sched_pid_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++bool lrng_raw_sched_starttime_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++static inline bool ++lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++bool lrng_raw_sched_nvcsw_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ ++static inline bool ++lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_SCHED_PERF ++bool lrng_sched_perf_time(u32 start); ++#else /* CONFIG_LRNG_SCHED_PERF */ ++static inline bool lrng_sched_perf_time(u32 start) { return false; } ++#endif /*CONFIG_LRNG_SCHED_PERF */ ++ ++#endif /* _LRNG_TESTING_H */ +diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h +index af5ad51d3eef..b12ae9bdebf4 100644 +--- a/include/crypto/drbg.h ++++ b/include/crypto/drbg.h +@@ -283,4 +283,11 @@ enum drbg_prefixes { + DRBG_PREFIX3 + }; + ++extern int drbg_alloc_state(struct drbg_state *drbg); ++extern void drbg_dealloc_state(struct drbg_state *drbg); ++extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, ++ bool *pr); ++extern const struct drbg_core drbg_cores[]; ++extern unsigned short drbg_sec_strength(drbg_flag_t flags); ++ + #endif /* _DRBG_H */ +diff --git a/crypto/jitterentropy.h b/include/crypto/internal/jitterentropy.h +similarity index 100% +rename from crypto/jitterentropy.h +rename to include/crypto/internal/jitterentropy.h +diff --git a/include/linux/lrng.h b/include/linux/lrng.h +new file mode 100644 +index 000000000000..c0d31a03d51f +--- /dev/null ++++ b/include/linux/lrng.h +@@ -0,0 +1,251 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ ++ ++#ifndef _LRNG_H ++#define _LRNG_H ++ ++#include ++#include ++#include ++#include ++ ++/* ++ * struct lrng_drng_cb - cryptographic callback functions defining a DRNG ++ * @drng_name Name of DRNG ++ * @drng_alloc: Allocate DRNG -- the provided integer should be used for ++ * sanity checks. ++ * return: allocated data structure or PTR_ERR on error ++ * @drng_dealloc: Deallocate DRNG ++ * @drng_seed: Seed the DRNG with data of arbitrary length drng: is ++ * pointer to data structure allocated with drng_alloc ++ * return: >= 0 on success, < 0 on error ++ * @drng_generate: Generate random numbers from the DRNG with arbitrary ++ * length ++ */ ++struct lrng_drng_cb { ++ const char *(*drng_name)(void); ++ void *(*drng_alloc)(u32 sec_strength); ++ void (*drng_dealloc)(void *drng); ++ int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); ++ int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); ++}; ++ ++/* ++ * struct lrng_hash_cb - cryptographic callback functions defining a hash ++ * @hash_name Name of Hash used for reading entropy pool arbitrary ++ * length ++ * @hash_alloc: Allocate the hash for reading the entropy pool ++ * return: allocated data structure (NULL is success too) ++ * or ERR_PTR on error ++ * @hash_dealloc: Deallocate Hash ++ * @hash_digestsize: Return the digestsize for the used hash to read out ++ * entropy pool ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: size of digest of hash in bytes ++ * @hash_init: Initialize hash ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: 0 on success, < 0 on error ++ * @hash_update: Update hash operation ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: 0 on success, < 0 on error ++ * @hash_final Final hash operation ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: 0 on success, < 0 on error ++ * @hash_desc_zero Zeroization of hash state buffer ++ * ++ * Assumptions: ++ * ++ * 1. Hash operation will not sleep ++ * 2. The hash' volatile state information is provided with *shash by caller. ++ */ ++struct lrng_hash_cb { ++ const char *(*hash_name)(void); ++ void *(*hash_alloc)(void); ++ void (*hash_dealloc)(void *hash); ++ u32 (*hash_digestsize)(void *hash); ++ int (*hash_init)(struct shash_desc *shash, void *hash); ++ int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, ++ u32 inbuflen); ++ int (*hash_final)(struct shash_desc *shash, u8 *digest); ++ void (*hash_desc_zero)(struct shash_desc *shash); ++}; ++ ++/* Register cryptographic backend */ ++#ifdef CONFIG_LRNG_SWITCH ++int lrng_set_drng_cb(const struct lrng_drng_cb *cb); ++int lrng_set_hash_cb(const struct lrng_hash_cb *cb); ++#else /* CONFIG_LRNG_SWITCH */ ++static inline int ++lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } ++static inline int ++lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } ++#endif /* CONFIG_LRNG_SWITCH */ ++ ++/* Callback to feed events to the scheduler entropy source */ ++#ifdef CONFIG_LRNG_SCHED ++extern void add_sched_randomness(const struct task_struct *p, int cpu); ++#else ++static inline void ++add_sched_randomness(const struct task_struct *p, int cpu) { } ++#endif ++ ++/* ++ * lrng_get_random_bytes() - Provider of cryptographic strong random numbers ++ * for kernel-internal usage. ++ * ++ * This function is appropriate for in-kernel use cases operating in atomic ++ * contexts. It will always use the ChaCha20 DRNG and it may be the case that ++ * it is not fully seeded when being used. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ */ ++#ifdef CONFIG_LRNG_DRNG_ATOMIC ++void lrng_get_random_bytes(void *buf, int nbytes); ++#endif ++ ++/* ++ * lrng_get_random_bytes_full() - Provider of cryptographic strong ++ * random numbers for kernel-internal usage from a fully initialized LRNG. ++ * ++ * This function will always return random numbers from a fully seeded and ++ * fully initialized LRNG. ++ * ++ * This function is appropriate only for non-atomic use cases as this ++ * function may sleep. It provides access to the full functionality of LRNG ++ * including the switchable DRNG support, that may support other DRNGs such ++ * as the SP800-90A DRBG. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ */ ++#ifdef CONFIG_LRNG ++void lrng_get_random_bytes_full(void *buf, int nbytes); ++#endif ++ ++/* ++ * lrng_get_random_bytes_min() - Provider of cryptographic strong ++ * random numbers for kernel-internal usage from at least a minimally seeded ++ * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C ++ * oversampling applied in FIPS mode is not applied yet). ++ * ++ * This function is appropriate only for non-atomic use cases as this ++ * function may sleep. It provides access to the full functionality of LRNG ++ * including the switchable DRNG support, that may support other DRNGs such ++ * as the SP800-90A DRBG. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ */ ++#ifdef CONFIG_LRNG ++void lrng_get_random_bytes_min(void *buf, int nbytes); ++#endif ++ ++/* ++ * lrng_get_random_bytes_pr() - Provider of cryptographic strong ++ * random numbers for kernel-internal usage from a fully initialized LRNG and ++ * requiring a reseed from the entropy sources before. ++ * ++ * This function will always return random numbers from a fully seeded and ++ * fully initialized LRNG. ++ * ++ * This function is appropriate only for non-atomic use cases as this ++ * function may sleep. It provides access to the full functionality of LRNG ++ * including the switchable DRNG support, that may support other DRNGs such ++ * as the SP800-90A DRBG. ++ * ++ * This call only returns no more data than entropy was pulled from the ++ * entropy sources. Thus, it is likely that this call returns less data ++ * than requested by the caller. Also, the caller MUST be prepared that this ++ * call returns 0 bytes, i.e. it did not generate data. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ * ++ * @return: positive number indicates amount of generated bytes, < 0 on error ++ */ ++#ifdef CONFIG_LRNG ++int lrng_get_random_bytes_pr(void *buf, int nbytes); ++#endif ++ ++/* ++ * lrng_get_seed() - Fill buffer with data from entropy sources ++ * ++ * This call allows accessing the entropy sources directly and fill the buffer ++ * with data from all available entropy sources. This filled buffer is ++ * identical to the temporary seed buffer used by the LRNG to seed its DRNGs. ++ * ++ * The call is to allows users to seed their DRNG directly from the entropy ++ * sources in case the caller does not want to use the LRNG's DRNGs. This ++ * buffer can be directly used to seed the caller's DRNG from. ++ * ++ * The call blocks as long as one LRNG DRNG is not yet fully seeded. If ++ * LRNG_GET_SEED_NONBLOCK is specified, it does not block in this case, but ++ * returns with an error. ++ * ++ * Considering SP800-90C, there is a differentiation between the seeding ++ * requirements during instantiating a DRNG and at runtime of the DRNG. When ++ * specifying LRNG_GET_SEED_FULLY_SEEDED the caller indicates the DRNG was ++ * already fully seeded and the regular amount of entropy is requested. ++ * Otherwise, the LRNG will obtain the entropy rate required for initial ++ * seeding. The following minimum entropy rates will be obtained: ++ * ++ * * FIPS mode: ++ * * Initial seeding: 384 bits of entropy ++ * * Runtime seeding: 256 bits of entropy ++ * * Non-FIPS mode: ++ * * 128 bits of entropy in any case ++ * ++ * Albeit these are minimum entropy rates, the LRNG tries to request the ++ * given amount of entropy from each entropy source individually. If the ++ * minimum amount of entropy cannot be obtained collectively by all entropy ++ * sources, the LRNG will not fill the buffer. ++ * ++ * The return data in buf is structurally equivalent to the following ++ * definition: ++ * ++ * struct { ++ * u64 seedlen; ++ * u64 entropy_rate; ++ * struct entropy_buf seed; ++ * } __attribute((__packed__)); ++ * ++ * As struct entropy_buf is not known outsize of the LRNG, the LRNG fills ++ * seedlen first with the size of struct entropy_buf. If the caller-provided ++ * buffer buf is smaller than u64, then -EINVAL is returned ++ * and buf is not touched. If it is u64 or larger but smaller ++ * than the size of the structure above, -EMSGSIZE is returned and seedlen ++ * is filled with the size of the buffer. Finally, if buf is large ++ * enough to hold all data, it is filled with the seed data and the seedlen ++ * is set to sizeof(struct entropy_buf). The entropy rate is returned with ++ * the variable entropy_rate and provides the value in bits. ++ * ++ * The seed buffer is the data that should be handed to the caller's DRNG as ++ * seed data. ++ * ++ * @buf [out] Buffer to be filled with data from the entropy sources - note, the ++ * buffer is marked as u64 to ensure it is aligned to 64 bits. ++ * @nbytes [in] Size of the buffer allocated by the caller - this value ++ * provides size of @param buf in bytes. ++ * @flags [in] Flags field to adjust the behavior ++ * ++ * @return -EINVAL or -EMSGSIZE indicating the buffer is too small, -EAGAIN when ++ * the call would block, but NONBLOCK is specified, > 0 the size of ++ * the filled buffer. ++ */ ++#ifdef CONFIG_LRNG ++enum lrng_get_seed_flags { ++ LRNG_GET_SEED_NONBLOCK = 0x0001, /**< Do not block the call */ ++ LRNG_GET_SEED_FULLY_SEEDED = 0x0002, /**< DRNG is fully seeded */ ++}; ++ ++ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags); ++#endif ++ ++#endif /* _LRNG_H */ +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index dcdbb8f8dc52..a97ff97891b1 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -6,6 +6,7 @@ + * + * Copyright (C) 1991-2002 Linus Torvalds + */ ++#include + #include + #include + #include +@@ -3586,6 +3587,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) + { + struct rq *rq; + ++ add_sched_randomness(p, cpu); ++ + if (!schedstat_enabled()) + return; + +-- +2.38.0 + +From 4497c11cc44fa6f2f5777eb70d9828add7a1d17e Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 19 Sep 2022 14:40:14 +0200 +Subject: [PATCH 10/17] folios + +Signed-off-by: Peter Jung +--- + fs/afs/write.c | 114 +++++++++++++++++---------------- + fs/btrfs/extent_io.c | 57 +++++++++-------- + fs/ceph/addr.c | 138 ++++++++++++++++++++-------------------- + fs/cifs/file.c | 33 +++++++++- + fs/ext4/inode.c | 55 ++++++++-------- + fs/f2fs/checkpoint.c | 49 +++++++------- + fs/f2fs/compress.c | 13 ++-- + fs/f2fs/data.c | 69 ++++++++++---------- + fs/f2fs/f2fs.h | 5 +- + fs/f2fs/node.c | 72 +++++++++++---------- + fs/gfs2/aops.c | 64 ++++++++++--------- + fs/nilfs2/btree.c | 14 ++-- + fs/nilfs2/page.c | 59 ++++++++--------- + fs/nilfs2/segment.c | 44 +++++++------ + include/linux/pagemap.h | 32 +++++++--- + include/linux/pagevec.h | 8 --- + mm/filemap.c | 87 ++++++++++++------------- + mm/page-writeback.c | 44 +++++++------ + mm/swap.c | 10 --- + 19 files changed, 507 insertions(+), 460 deletions(-) + +diff --git a/fs/afs/write.c b/fs/afs/write.c +index 9ebdd36eaf2f..c17dbd82a38c 100644 +--- a/fs/afs/write.c ++++ b/fs/afs/write.c +@@ -699,82 +699,86 @@ static int afs_writepages_region(struct address_space *mapping, + loff_t start, loff_t end, loff_t *_next) + { + struct folio *folio; +- struct page *head_page; ++ struct folio_batch fbatch; + ssize_t ret; ++ unsigned int i; + int n, skips = 0; + + _enter("%llx,%llx,", start, end); ++ folio_batch_init(&fbatch); + + do { + pgoff_t index = start / PAGE_SIZE; + +- n = find_get_pages_range_tag(mapping, &index, end / PAGE_SIZE, +- PAGECACHE_TAG_DIRTY, 1, &head_page); ++ n = filemap_get_folios_tag(mapping, &index, end / PAGE_SIZE, ++ PAGECACHE_TAG_DIRTY, &fbatch); ++ + if (!n) + break; ++ for (i = 0; i < n; i++) { ++ folio = fbatch.folios[i]; ++ start = folio_pos(folio); /* May regress with THPs */ + +- folio = page_folio(head_page); +- start = folio_pos(folio); /* May regress with THPs */ +- +- _debug("wback %lx", folio_index(folio)); ++ _debug("wback %lx", folio_index(folio)); + +- /* At this point we hold neither the i_pages lock nor the +- * page lock: the page may be truncated or invalidated +- * (changing page->mapping to NULL), or even swizzled +- * back from swapper_space to tmpfs file mapping +- */ +- if (wbc->sync_mode != WB_SYNC_NONE) { +- ret = folio_lock_killable(folio); +- if (ret < 0) { +- folio_put(folio); +- return ret; +- } +- } else { +- if (!folio_trylock(folio)) { +- folio_put(folio); +- return 0; ++ /* At this point we hold neither the i_pages lock nor the ++ * page lock: the page may be truncated or invalidated ++ * (changing page->mapping to NULL), or even swizzled ++ * back from swapper_space to tmpfs file mapping ++ */ ++ if (wbc->sync_mode != WB_SYNC_NONE) { ++ ret = folio_lock_killable(folio); ++ if (ret < 0) { ++ folio_batch_release(&fbatch); ++ return ret; ++ } ++ } else { ++ if (!folio_trylock(folio)) ++ continue; + } +- } + +- if (folio_mapping(folio) != mapping || +- !folio_test_dirty(folio)) { +- start += folio_size(folio); +- folio_unlock(folio); +- folio_put(folio); +- continue; +- } ++ if (folio->mapping != mapping || ++ !folio_test_dirty(folio)) { ++ start += folio_size(folio); ++ folio_unlock(folio); ++ continue; ++ } + +- if (folio_test_writeback(folio) || +- folio_test_fscache(folio)) { +- folio_unlock(folio); +- if (wbc->sync_mode != WB_SYNC_NONE) { +- folio_wait_writeback(folio); ++ if (folio_test_writeback(folio) || ++ folio_test_fscache(folio)) { ++ folio_unlock(folio); ++ if (wbc->sync_mode != WB_SYNC_NONE) { ++ folio_wait_writeback(folio); + #ifdef CONFIG_AFS_FSCACHE +- folio_wait_fscache(folio); ++ folio_wait_fscache(folio); + #endif +- } else { +- start += folio_size(folio); ++ } else { ++ start += folio_size(folio); ++ } ++ if (wbc->sync_mode == WB_SYNC_NONE) { ++ if (skips >= 5 || need_resched()) { ++ *_next = start; ++ _leave(" = 0 [%llx]", *_next); ++ return 0; ++ } ++ skips++; ++ } ++ continue; + } +- folio_put(folio); +- if (wbc->sync_mode == WB_SYNC_NONE) { +- if (skips >= 5 || need_resched()) +- break; +- skips++; ++ ++ if (!folio_clear_dirty_for_io(folio)) ++ BUG(); ++ ret = afs_write_back_from_locked_folio(mapping, wbc, ++ folio, start, end); ++ if (ret < 0) { ++ _leave(" = %zd", ret); ++ folio_batch_release(&fbatch); ++ return ret; + } +- continue; +- } + +- if (!folio_clear_dirty_for_io(folio)) +- BUG(); +- ret = afs_write_back_from_locked_folio(mapping, wbc, folio, start, end); +- folio_put(folio); +- if (ret < 0) { +- _leave(" = %zd", ret); +- return ret; ++ start += ret; + } +- +- start += ret; +- ++ folio_batch_release(&fbatch); + cond_resched(); + } while (wbc->nr_to_write > 0); + +diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c +index cf4f19e80e2f..80fe313f8461 100644 +--- a/fs/btrfs/extent_io.c ++++ b/fs/btrfs/extent_io.c +@@ -4844,14 +4844,14 @@ int btree_write_cache_pages(struct address_space *mapping, + int ret = 0; + int done = 0; + int nr_to_write_done = 0; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ unsigned int nr_folios; + pgoff_t index; + pgoff_t end; /* Inclusive */ + int scanned = 0; + xa_mark_t tag; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + index = mapping->writeback_index; /* Start from prev offset */ + end = -1; +@@ -4874,14 +4874,15 @@ int btree_write_cache_pages(struct address_space *mapping, + if (wbc->sync_mode == WB_SYNC_ALL) + tag_pages_for_writeback(mapping, index, end); + while (!done && !nr_to_write_done && (index <= end) && +- (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, +- tag))) { ++ (nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ tag, &fbatch))) { + unsigned i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- ret = submit_eb_page(page, wbc, &epd, &eb_context); ++ ret = submit_eb_page(&folio->page, wbc, &epd, ++ &eb_context); + if (ret == 0) + continue; + if (ret < 0) { +@@ -4896,7 +4897,7 @@ int btree_write_cache_pages(struct address_space *mapping, + */ + nr_to_write_done = wbc->nr_to_write <= 0; + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + if (!scanned && !done) { +@@ -4971,8 +4972,8 @@ static int extent_write_cache_pages(struct address_space *mapping, + int ret = 0; + int done = 0; + int nr_to_write_done = 0; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ unsigned int nr_folios; + pgoff_t index; + pgoff_t end; /* Inclusive */ + pgoff_t done_index; +@@ -4992,7 +4993,7 @@ static int extent_write_cache_pages(struct address_space *mapping, + if (!igrab(inode)) + return 0; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + index = mapping->writeback_index; /* Start from prev offset */ + end = -1; +@@ -5030,14 +5031,14 @@ static int extent_write_cache_pages(struct address_space *mapping, + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && !nr_to_write_done && (index <= end) && +- (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, +- &index, end, tag))) { ++ (nr_folios = filemap_get_folios_tag(mapping, &index, ++ end, tag, &fbatch))) { + unsigned i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- done_index = page->index + 1; ++ done_index = folio->index + folio_nr_pages(folio); + /* + * At this point we hold neither the i_pages lock nor + * the page lock: the page may be truncated or +@@ -5045,29 +5046,29 @@ static int extent_write_cache_pages(struct address_space *mapping, + * or even swizzled back from swapper_space to + * tmpfs file mapping + */ +- if (!trylock_page(page)) { ++ if (!folio_trylock(folio)) { + submit_write_bio(epd, 0); +- lock_page(page); ++ folio_lock(folio); + } + +- if (unlikely(page->mapping != mapping)) { +- unlock_page(page); ++ if (unlikely(folio->mapping != mapping)) { ++ folio_unlock(folio); + continue; + } + + if (wbc->sync_mode != WB_SYNC_NONE) { +- if (PageWriteback(page)) ++ if (folio_test_writeback(folio)) + submit_write_bio(epd, 0); +- wait_on_page_writeback(page); ++ folio_wait_writeback(folio); + } + +- if (PageWriteback(page) || +- !clear_page_dirty_for_io(page)) { +- unlock_page(page); ++ if (folio_test_writeback(folio) || ++ !folio_clear_dirty_for_io(folio)) { ++ folio_unlock(folio); + continue; + } + +- ret = __extent_writepage(page, wbc, epd); ++ ret = __extent_writepage(&folio->page, wbc, epd); + if (ret < 0) { + done = 1; + break; +@@ -5080,7 +5081,7 @@ static int extent_write_cache_pages(struct address_space *mapping, + */ + nr_to_write_done = wbc->nr_to_write <= 0; + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + if (!scanned && !done) { +diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c +index dcf701b05cc1..33dbe55b08be 100644 +--- a/fs/ceph/addr.c ++++ b/fs/ceph/addr.c +@@ -792,7 +792,7 @@ static int ceph_writepages_start(struct address_space *mapping, + struct ceph_vino vino = ceph_vino(inode); + pgoff_t index, start_index, end = -1; + struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; +- struct pagevec pvec; ++ struct folio_batch fbatch; + int rc = 0; + unsigned int wsize = i_blocksize(inode); + struct ceph_osd_request *req = NULL; +@@ -821,7 +821,7 @@ static int ceph_writepages_start(struct address_space *mapping, + if (fsc->mount_options->wsize < wsize) + wsize = fsc->mount_options->wsize; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + + start_index = wbc->range_cyclic ? mapping->writeback_index : 0; + index = start_index; +@@ -869,9 +869,9 @@ static int ceph_writepages_start(struct address_space *mapping, + + while (!done && index <= end) { + int num_ops = 0, op_idx; +- unsigned i, pvec_pages, max_pages, locked_pages = 0; ++ unsigned i, nr_folios, max_pages, locked_pages = 0; + struct page **pages = NULL, **data_pages; +- struct page *page; ++ struct folio *folio; + pgoff_t strip_unit_end = 0; + u64 offset = 0, len = 0; + bool from_pool = false; +@@ -879,28 +879,28 @@ static int ceph_writepages_start(struct address_space *mapping, + max_pages = wsize >> PAGE_SHIFT; + + get_more_pages: +- pvec_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, +- end, PAGECACHE_TAG_DIRTY); +- dout("pagevec_lookup_range_tag got %d\n", pvec_pages); +- if (!pvec_pages && !locked_pages) ++ nr_folios = filemap_get_folios_tag(mapping, &index, ++ end, PAGECACHE_TAG_DIRTY, &fbatch); ++ dout("filemap_get_folios_tag got %d\n", nr_folios); ++ if (!nr_folios && !locked_pages) + break; +- for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { +- page = pvec.pages[i]; +- dout("? %p idx %lu\n", page, page->index); ++ for (i = 0; i < nr_folios && locked_pages < max_pages; i++) { ++ folio = fbatch.folios[i]; ++ dout("? %p idx %lu\n", folio, folio->index); + if (locked_pages == 0) +- lock_page(page); /* first page */ +- else if (!trylock_page(page)) ++ folio_lock(folio); /* first folio */ ++ else if (!folio_trylock(folio)) + break; + + /* only dirty pages, or our accounting breaks */ +- if (unlikely(!PageDirty(page)) || +- unlikely(page->mapping != mapping)) { +- dout("!dirty or !mapping %p\n", page); +- unlock_page(page); ++ if (unlikely(!folio_test_dirty(folio)) || ++ unlikely(folio->mapping != mapping)) { ++ dout("!dirty or !mapping %p\n", folio); ++ folio_unlock(folio); + continue; + } + /* only if matching snap context */ +- pgsnapc = page_snap_context(page); ++ pgsnapc = page_snap_context(&folio->page); + if (pgsnapc != snapc) { + dout("page snapc %p %lld != oldest %p %lld\n", + pgsnapc, pgsnapc->seq, snapc, snapc->seq); +@@ -908,11 +908,10 @@ static int ceph_writepages_start(struct address_space *mapping, + !ceph_wbc.head_snapc && + wbc->sync_mode != WB_SYNC_NONE) + should_loop = true; +- unlock_page(page); ++ folio_unlock(folio); + continue; + } +- if (page_offset(page) >= ceph_wbc.i_size) { +- struct folio *folio = page_folio(page); ++ if (folio_pos(folio) >= ceph_wbc.i_size) { + + dout("folio at %lu beyond eof %llu\n", + folio->index, ceph_wbc.i_size); +@@ -924,25 +923,26 @@ static int ceph_writepages_start(struct address_space *mapping, + folio_unlock(folio); + continue; + } +- if (strip_unit_end && (page->index > strip_unit_end)) { +- dout("end of strip unit %p\n", page); +- unlock_page(page); ++ if (strip_unit_end && (folio->index > strip_unit_end)) { ++ dout("end of strip unit %p\n", folio); ++ folio_unlock(folio); + break; + } +- if (PageWriteback(page) || PageFsCache(page)) { ++ if (folio_test_writeback(folio) || ++ folio_test_fscache(folio)) { + if (wbc->sync_mode == WB_SYNC_NONE) { +- dout("%p under writeback\n", page); +- unlock_page(page); ++ dout("%p under writeback\n", folio); ++ folio_unlock(folio); + continue; + } +- dout("waiting on writeback %p\n", page); +- wait_on_page_writeback(page); +- wait_on_page_fscache(page); ++ dout("waiting on writeback %p\n", folio); ++ folio_wait_writeback(folio); ++ folio_wait_fscache(folio); + } + +- if (!clear_page_dirty_for_io(page)) { +- dout("%p !clear_page_dirty_for_io\n", page); +- unlock_page(page); ++ if (!folio_clear_dirty_for_io(folio)) { ++ dout("%p !clear_page_dirty_for_io\n", folio); ++ folio_unlock(folio); + continue; + } + +@@ -958,7 +958,7 @@ static int ceph_writepages_start(struct address_space *mapping, + u32 xlen; + + /* prepare async write request */ +- offset = (u64)page_offset(page); ++ offset = (u64)folio_pos(folio); + ceph_calc_file_object_mapping(&ci->i_layout, + offset, wsize, + &objnum, &objoff, +@@ -966,7 +966,7 @@ static int ceph_writepages_start(struct address_space *mapping, + len = xlen; + + num_ops = 1; +- strip_unit_end = page->index + ++ strip_unit_end = folio->index + + ((len - 1) >> PAGE_SHIFT); + + BUG_ON(pages); +@@ -981,54 +981,53 @@ static int ceph_writepages_start(struct address_space *mapping, + } + + len = 0; +- } else if (page->index != ++ } else if (folio->index != + (offset + len) >> PAGE_SHIFT) { + if (num_ops >= (from_pool ? CEPH_OSD_SLAB_OPS : + CEPH_OSD_MAX_OPS)) { +- redirty_page_for_writepage(wbc, page); +- unlock_page(page); ++ folio_redirty_for_writepage(wbc, folio); ++ folio_unlock(folio); + break; + } + + num_ops++; +- offset = (u64)page_offset(page); ++ offset = (u64)folio_pos(folio); + len = 0; + } + +- /* note position of first page in pvec */ ++ /* note position of first page in fbatch */ + dout("%p will write page %p idx %lu\n", +- inode, page, page->index); ++ inode, folio, folio->index); + + if (atomic_long_inc_return(&fsc->writeback_count) > + CONGESTION_ON_THRESH( + fsc->mount_options->congestion_kb)) + fsc->write_congested = true; + +- pages[locked_pages++] = page; +- pvec.pages[i] = NULL; ++ pages[locked_pages++] = &folio->page; ++ fbatch.folios[i] = NULL; + +- len += thp_size(page); ++ len += folio_size(folio); + } + + /* did we get anything? */ + if (!locked_pages) +- goto release_pvec_pages; ++ goto release_folio_batches; + if (i) { + unsigned j, n = 0; +- /* shift unused page to beginning of pvec */ +- for (j = 0; j < pvec_pages; j++) { +- if (!pvec.pages[j]) ++ /* shift unused folio to the beginning of fbatch */ ++ for (j = 0; j < nr_folios; j++) { ++ if (!fbatch.folios[j]) + continue; + if (n < j) +- pvec.pages[n] = pvec.pages[j]; ++ fbatch.folios[n] = fbatch.folios[j]; + n++; + } +- pvec.nr = n; +- +- if (pvec_pages && i == pvec_pages && ++ fbatch.nr = n; ++ if (nr_folios && i == nr_folios && + locked_pages < max_pages) { +- dout("reached end pvec, trying for more\n"); +- pagevec_release(&pvec); ++ dout("reached end of fbatch, trying for more\n"); ++ folio_batch_release(&fbatch); + goto get_more_pages; + } + } +@@ -1056,7 +1055,7 @@ static int ceph_writepages_start(struct address_space *mapping, + BUG_ON(IS_ERR(req)); + } + BUG_ON(len < page_offset(pages[locked_pages - 1]) + +- thp_size(page) - offset); ++ folio_size(folio) - offset); + + req->r_callback = writepages_finish; + req->r_inode = inode; +@@ -1098,7 +1097,7 @@ static int ceph_writepages_start(struct address_space *mapping, + set_page_writeback(pages[i]); + if (caching) + ceph_set_page_fscache(pages[i]); +- len += thp_size(page); ++ len += folio_size(folio); + } + ceph_fscache_write_to_cache(inode, offset, len, caching); + +@@ -1108,7 +1107,7 @@ static int ceph_writepages_start(struct address_space *mapping, + /* writepages_finish() clears writeback pages + * according to the data length, so make sure + * data length covers all locked pages */ +- u64 min_len = len + 1 - thp_size(page); ++ u64 min_len = len + 1 - folio_size(folio); + len = get_writepages_data_length(inode, pages[i - 1], + offset); + len = max(len, min_len); +@@ -1164,10 +1163,10 @@ static int ceph_writepages_start(struct address_space *mapping, + if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) + done = true; + +-release_pvec_pages: +- dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, +- pvec.nr ? pvec.pages[0] : NULL); +- pagevec_release(&pvec); ++release_folio_batches: ++ dout("folio_batch_release on %d batches (%p)", (int) fbatch.nr, ++ fbatch.nr ? fbatch.folios[0] : NULL); ++ folio_batch_release(&fbatch); + } + + if (should_loop && !done) { +@@ -1180,19 +1179,22 @@ static int ceph_writepages_start(struct address_space *mapping, + if (wbc->sync_mode != WB_SYNC_NONE && + start_index == 0 && /* all dirty pages were checked */ + !ceph_wbc.head_snapc) { +- struct page *page; ++ struct folio *folio; + unsigned i, nr; + index = 0; + while ((index <= end) && +- (nr = pagevec_lookup_tag(&pvec, mapping, &index, +- PAGECACHE_TAG_WRITEBACK))) { ++ (nr = filemap_get_folios_tag(mapping, &index, ++ (pgoff_t)-1, ++ PAGECACHE_TAG_WRITEBACK, ++ &fbatch))) { + for (i = 0; i < nr; i++) { +- page = pvec.pages[i]; +- if (page_snap_context(page) != snapc) ++ folio = fbatch.folios[i]; ++ if (page_snap_context(&folio->page) != ++ snapc) + continue; +- wait_on_page_writeback(page); ++ folio_wait_writeback(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +diff --git a/fs/cifs/file.c b/fs/cifs/file.c +index 6f38b134a346..249a94896f78 100644 +--- a/fs/cifs/file.c ++++ b/fs/cifs/file.c +@@ -2517,14 +2517,41 @@ wdata_alloc_and_fillpages(pgoff_t tofind, struct address_space *mapping, + unsigned int *found_pages) + { + struct cifs_writedata *wdata; +- ++ struct folio_batch fbatch; ++ unsigned int i, idx, p, nr; + wdata = cifs_writedata_alloc((unsigned int)tofind, + cifs_writev_complete); + if (!wdata) + return NULL; + +- *found_pages = find_get_pages_range_tag(mapping, index, end, +- PAGECACHE_TAG_DIRTY, tofind, wdata->pages); ++ folio_batch_init(&fbatch); ++ *found_pages = 0; ++ ++again: ++ nr = filemap_get_folios_tag(mapping, index, end, ++ PAGECACHE_TAG_DIRTY, &fbatch); ++ if (!nr) ++ goto out; /* No dirty pages left in the range */ ++ ++ for (i = 0; i < nr; i++) { ++ struct folio *folio = fbatch.folios[i]; ++ ++ idx = 0; ++ p = folio_nr_pages(folio); ++add_more: ++ wdata->pages[*found_pages] = folio_page(folio, idx); ++ if (++*found_pages == tofind) { ++ folio_batch_release(&fbatch); ++ goto out; ++ } ++ if (++idx < p) { ++ folio_ref_inc(folio); ++ goto add_more; ++ } ++ } ++ folio_batch_release(&fbatch); ++ goto again; ++out: + return wdata; + } + +diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c +index 601214453c3a..fbd876e10a85 100644 +--- a/fs/ext4/inode.c ++++ b/fs/ext4/inode.c +@@ -2565,8 +2565,8 @@ static int ext4_da_writepages_trans_blocks(struct inode *inode) + static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) + { + struct address_space *mapping = mpd->inode->i_mapping; +- struct pagevec pvec; +- unsigned int nr_pages; ++ struct folio_batch fbatch; ++ unsigned int nr_folios; + long left = mpd->wbc->nr_to_write; + pgoff_t index = mpd->first_page; + pgoff_t end = mpd->last_page; +@@ -2580,18 +2580,17 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) + tag = PAGECACHE_TAG_TOWRITE; + else + tag = PAGECACHE_TAG_DIRTY; +- +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + mpd->map.m_len = 0; + mpd->next_page = index; + while (index <= end) { +- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, +- tag); +- if (nr_pages == 0) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ tag, &fbatch); ++ if (nr_folios == 0) + break; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + + /* + * Accumulated enough dirty pages? This doesn't apply +@@ -2605,10 +2604,10 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) + goto out; + + /* If we can't merge this page, we are done. */ +- if (mpd->map.m_len > 0 && mpd->next_page != page->index) ++ if (mpd->map.m_len > 0 && mpd->next_page != folio->index) + goto out; + +- lock_page(page); ++ folio_lock(folio); + /* + * If the page is no longer dirty, or its mapping no + * longer corresponds to inode we are writing (which +@@ -2616,16 +2615,16 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) + * page is already under writeback and we are not doing + * a data integrity writeback, skip the page + */ +- if (!PageDirty(page) || +- (PageWriteback(page) && ++ if (!folio_test_dirty(folio) || ++ (folio_test_writeback(folio) && + (mpd->wbc->sync_mode == WB_SYNC_NONE)) || +- unlikely(page->mapping != mapping)) { +- unlock_page(page); ++ unlikely(folio->mapping != mapping)) { ++ folio_unlock(folio); + continue; + } + +- wait_on_page_writeback(page); +- BUG_ON(PageWriteback(page)); ++ folio_wait_writeback(folio); ++ BUG_ON(folio_test_writeback(folio)); + + /* + * Should never happen but for buggy code in +@@ -2636,33 +2635,33 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) + * + * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz + */ +- if (!page_has_buffers(page)) { +- ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index); +- ClearPageDirty(page); +- unlock_page(page); ++ if (!folio_buffers(folio)) { ++ ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index); ++ folio_clear_dirty(folio); ++ folio_unlock(folio); + continue; + } + + if (mpd->map.m_len == 0) +- mpd->first_page = page->index; +- mpd->next_page = page->index + 1; ++ mpd->first_page = folio->index; ++ mpd->next_page = folio->index + folio_nr_pages(folio); + /* Add all dirty buffers to mpd */ +- lblk = ((ext4_lblk_t)page->index) << ++ lblk = ((ext4_lblk_t)folio->index) << + (PAGE_SHIFT - blkbits); +- head = page_buffers(page); ++ head = folio_buffers(folio); + err = mpage_process_page_bufs(mpd, head, head, lblk); + if (err <= 0) + goto out; + err = 0; +- left--; ++ left -= folio_nr_pages(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + mpd->scanned_until_end = 1; + return 0; + out: +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + return err; + } + +diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c +index 8259e0fa97e1..9f6694f7d723 100644 +--- a/fs/f2fs/checkpoint.c ++++ b/fs/f2fs/checkpoint.c +@@ -377,59 +377,62 @@ long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type, + { + struct address_space *mapping = META_MAPPING(sbi); + pgoff_t index = 0, prev = ULONG_MAX; +- struct pagevec pvec; ++ struct folio_batch fbatch; + long nwritten = 0; +- int nr_pages; ++ int nr_folios; + struct writeback_control wbc = { + .for_reclaim = 0, + }; + struct blk_plug plug; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + + blk_start_plug(&plug); + +- while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, +- PAGECACHE_TAG_DIRTY))) { ++ while ((nr_folios = filemap_get_folios_tag(mapping, &index, ++ (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch))) { + int i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- if (prev == ULONG_MAX) +- prev = page->index - 1; +- if (nr_to_write != LONG_MAX && page->index != prev + 1) { +- pagevec_release(&pvec); ++ if (nr_to_write != LONG_MAX && i != 0 && ++ folio->index != prev + ++ folio_nr_pages(fbatch.folios[i-1])) { ++ folio_batch_release(&fbatch); + goto stop; + } + +- lock_page(page); ++ folio_lock(folio); + +- if (unlikely(page->mapping != mapping)) { ++ if (unlikely(folio->mapping != mapping)) { + continue_unlock: +- unlock_page(page); ++ folio_unlock(folio); + continue; + } +- if (!PageDirty(page)) { ++ if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + +- f2fs_wait_on_page_writeback(page, META, true, true); ++ f2fs_wait_on_page_writeback(&folio->page, META, ++ true, true); + +- if (!clear_page_dirty_for_io(page)) ++ if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + +- if (__f2fs_write_meta_page(page, &wbc, io_type)) { +- unlock_page(page); ++ if (__f2fs_write_meta_page(&folio->page, &wbc, ++ io_type)) { ++ folio_unlock(folio); + break; + } +- nwritten++; +- prev = page->index; ++ nwritten += folio_nr_pages(folio); ++ prev = folio->index; + if (unlikely(nwritten >= nr_to_write)) + break; + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + stop: +@@ -1381,7 +1384,7 @@ static void commit_checkpoint(struct f2fs_sb_info *sbi, + }; + + /* +- * pagevec_lookup_tag and lock_page again will take ++ * filemap_get_folios_tag and lock_page again will take + * some extra time. Therefore, f2fs_update_meta_pages and + * f2fs_sync_meta_pages are combined in this function. + */ +diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c +index 70e97075e535..e1bd2e859f64 100644 +--- a/fs/f2fs/compress.c ++++ b/fs/f2fs/compress.c +@@ -841,10 +841,11 @@ bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) + return is_page_in_cluster(cc, index); + } + +-bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, +- int index, int nr_pages, bool uptodate) ++bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, ++ struct folio_batch *fbatch, ++ int index, int nr_folios, bool uptodate) + { +- unsigned long pgidx = pages[index]->index; ++ unsigned long pgidx = fbatch->folios[index]->index; + int i = uptodate ? 0 : 1; + + /* +@@ -854,13 +855,13 @@ bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, + if (uptodate && (pgidx % cc->cluster_size)) + return false; + +- if (nr_pages - index < cc->cluster_size) ++ if (nr_folios - index < cc->cluster_size) + return false; + + for (; i < cc->cluster_size; i++) { +- if (pages[index + i]->index != pgidx + i) ++ if (fbatch->folios[index + i]->index != pgidx + i) + return false; +- if (uptodate && !PageUptodate(pages[index + i])) ++ if (uptodate && !folio_test_uptodate(fbatch->folios[index + i])) + return false; + } + +diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c +index aa3ccddfa037..b14747598b39 100644 +--- a/fs/f2fs/data.c ++++ b/fs/f2fs/data.c +@@ -2917,7 +2917,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + { + int ret = 0; + int done = 0, retry = 0; +- struct page *pages[F2FS_ONSTACK_PAGES]; ++ struct folio_batch fbatch; + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct bio *bio = NULL; + sector_t last_block; +@@ -2938,7 +2938,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + .private = NULL, + }; + #endif +- int nr_pages; ++ int nr_folios; + pgoff_t index; + pgoff_t end; /* Inclusive */ + pgoff_t done_index; +@@ -2948,6 +2948,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + int submitted = 0; + int i; + ++ folio_batch_init(&fbatch); ++ + if (get_dirty_pages(mapping->host) <= + SM_I(F2FS_M_SB(mapping))->min_hot_blocks) + set_inode_flag(mapping->host, FI_HOT_DATA); +@@ -2973,13 +2975,13 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && !retry && (index <= end)) { +- nr_pages = find_get_pages_range_tag(mapping, &index, end, +- tag, F2FS_ONSTACK_PAGES, pages); +- if (nr_pages == 0) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ tag, &fbatch); ++ if (nr_folios == 0) + break; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + bool need_readd; + readd: + need_readd = false; +@@ -2996,7 +2998,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + } + + if (!f2fs_cluster_can_merge_page(&cc, +- page->index)) { ++ folio->index)) { + ret = f2fs_write_multi_pages(&cc, + &submitted, wbc, io_type); + if (!ret) +@@ -3005,27 +3007,28 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + } + + if (unlikely(f2fs_cp_error(sbi))) +- goto lock_page; ++ goto lock_folio; + + if (!f2fs_cluster_is_empty(&cc)) +- goto lock_page; ++ goto lock_folio; + + if (f2fs_all_cluster_page_ready(&cc, +- pages, i, nr_pages, true)) +- goto lock_page; ++ &fbatch, i, nr_folios, true)) ++ goto lock_folio; + + ret2 = f2fs_prepare_compress_overwrite( + inode, &pagep, +- page->index, &fsdata); ++ folio->index, &fsdata); + if (ret2 < 0) { + ret = ret2; + done = 1; + break; + } else if (ret2 && + (!f2fs_compress_write_end(inode, +- fsdata, page->index, 1) || ++ fsdata, folio->index, 1) || + !f2fs_all_cluster_page_ready(&cc, +- pages, i, nr_pages, false))) { ++ &fbatch, i, nr_folios, ++ false))) { + retry = 1; + break; + } +@@ -3038,46 +3041,47 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + break; + } + #ifdef CONFIG_F2FS_FS_COMPRESSION +-lock_page: ++lock_folio: + #endif +- done_index = page->index; ++ done_index = folio->index; + retry_write: +- lock_page(page); ++ folio_lock(folio); + +- if (unlikely(page->mapping != mapping)) { ++ if (unlikely(folio->mapping != mapping)) { + continue_unlock: +- unlock_page(page); ++ folio_unlock(folio); + continue; + } + +- if (!PageDirty(page)) { ++ if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + +- if (PageWriteback(page)) { ++ if (folio_test_writeback(folio)) { + if (wbc->sync_mode != WB_SYNC_NONE) +- f2fs_wait_on_page_writeback(page, ++ f2fs_wait_on_page_writeback( ++ &folio->page, + DATA, true, true); + else + goto continue_unlock; + } + +- if (!clear_page_dirty_for_io(page)) ++ if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + #ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { +- get_page(page); +- f2fs_compress_ctx_add_page(&cc, page); ++ folio_get(folio); ++ f2fs_compress_ctx_add_page(&cc, &folio->page); + continue; + } + #endif +- ret = f2fs_write_single_data_page(page, &submitted, +- &bio, &last_block, wbc, io_type, +- 0, true); ++ ret = f2fs_write_single_data_page(&folio->page, ++ &submitted, &bio, &last_block, ++ wbc, io_type, 0, true); + if (ret == AOP_WRITEPAGE_ACTIVATE) +- unlock_page(page); ++ folio_unlock(folio); + #ifdef CONFIG_F2FS_FS_COMPRESSION + result: + #endif +@@ -3101,7 +3105,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + } + goto next; + } +- done_index = page->index + 1; ++ done_index = folio->index + ++ folio_nr_pages(folio); + done = 1; + break; + } +@@ -3115,7 +3120,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, + if (need_readd) + goto readd; + } +- release_pages(pages, nr_pages); ++ folio_batch_release(&fbatch); + cond_resched(); + } + #ifdef CONFIG_F2FS_FS_COMPRESSION +diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h +index 3c7cdb70fe2e..dcb28240f724 100644 +--- a/fs/f2fs/f2fs.h ++++ b/fs/f2fs/f2fs.h +@@ -4196,8 +4196,9 @@ void f2fs_end_read_compressed_page(struct page *page, bool failed, + block_t blkaddr, bool in_task); + bool f2fs_cluster_is_empty(struct compress_ctx *cc); + bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index); +-bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, +- int index, int nr_pages, bool uptodate); ++bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, ++ struct folio_batch *fbatch, int index, int nr_folios, ++ bool uptodate); + bool f2fs_sanity_check_cluster(struct dnode_of_data *dn); + void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page); + int f2fs_write_multi_pages(struct compress_ctx *cc, +diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c +index e06a0c478b39..b993be76013e 100644 +--- a/fs/f2fs/node.c ++++ b/fs/f2fs/node.c +@@ -1513,23 +1513,24 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) + static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) + { + pgoff_t index; +- struct pagevec pvec; ++ struct folio_batch fbatch; + struct page *last_page = NULL; +- int nr_pages; ++ int nr_folios; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + index = 0; + +- while ((nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, +- PAGECACHE_TAG_DIRTY))) { ++ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, ++ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, ++ &fbatch))) { + int i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct page *page = &fbatch.folios[i]->page; + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_put_page(last_page, 0); +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + return ERR_PTR(-EIO); + } + +@@ -1560,7 +1561,7 @@ static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) + last_page = page; + unlock_page(page); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + return last_page; +@@ -1726,12 +1727,12 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, + unsigned int *seq_id) + { + pgoff_t index; +- struct pagevec pvec; ++ struct folio_batch fbatch; + int ret = 0; + struct page *last_page = NULL; + bool marked = false; + nid_t ino = inode->i_ino; +- int nr_pages; ++ int nr_folios; + int nwritten = 0; + + if (atomic) { +@@ -1740,20 +1741,21 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, + return PTR_ERR_OR_ZERO(last_page); + } + retry: +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + index = 0; + +- while ((nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, +- PAGECACHE_TAG_DIRTY))) { ++ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, ++ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, ++ &fbatch))) { + int i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct page *page = &fbatch.folios[i]->page; + bool submitted = false; + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_put_page(last_page, 0); +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + ret = -EIO; + goto out; + } +@@ -1819,7 +1821,7 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, + break; + } + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + + if (ret || marked) +@@ -1884,17 +1886,18 @@ static bool flush_dirty_inode(struct page *page) + void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) + { + pgoff_t index = 0; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ int nr_folios; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + +- while ((nr_pages = pagevec_lookup_tag(&pvec, +- NODE_MAPPING(sbi), &index, PAGECACHE_TAG_DIRTY))) { ++ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, ++ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, ++ &fbatch))) { + int i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct page *page = &fbatch.folios[i]->page; + + if (!IS_DNODE(page)) + continue; +@@ -1921,7 +1924,7 @@ void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) + } + unlock_page(page); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +@@ -1931,23 +1934,24 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, + bool do_balance, enum iostat_type io_type) + { + pgoff_t index; +- struct pagevec pvec; ++ struct folio_batch fbatch; + int step = 0; + int nwritten = 0; + int ret = 0; +- int nr_pages, done = 0; ++ int nr_folios, done = 0; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + + next_step: + index = 0; + +- while (!done && (nr_pages = pagevec_lookup_tag(&pvec, +- NODE_MAPPING(sbi), &index, PAGECACHE_TAG_DIRTY))) { ++ while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), ++ &index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY, ++ &fbatch))) { + int i; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct page *page = &fbatch.folios[i]->page; + bool submitted = false; + + /* give a priority to WB_SYNC threads */ +@@ -2022,7 +2026,7 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, + if (--wbc->nr_to_write == 0) + break; + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + + if (wbc->nr_to_write == 0) { +diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c +index 05bee80ac7de..8f87c2551a3d 100644 +--- a/fs/gfs2/aops.c ++++ b/fs/gfs2/aops.c +@@ -195,67 +195,71 @@ static int gfs2_writepages(struct address_space *mapping, + } + + /** +- * gfs2_write_jdata_pagevec - Write back a pagevec's worth of pages ++ * gfs2_write_jdata_batch - Write back a folio batch's worth of folios + * @mapping: The mapping + * @wbc: The writeback control +- * @pvec: The vector of pages +- * @nr_pages: The number of pages to write ++ * @fbatch: The batch of folios + * @done_index: Page index + * + * Returns: non-zero if loop should terminate, zero otherwise + */ + +-static int gfs2_write_jdata_pagevec(struct address_space *mapping, ++static int gfs2_write_jdata_batch(struct address_space *mapping, + struct writeback_control *wbc, +- struct pagevec *pvec, +- int nr_pages, ++ struct folio_batch *fbatch, + pgoff_t *done_index) + { + struct inode *inode = mapping->host; + struct gfs2_sbd *sdp = GFS2_SB(inode); +- unsigned nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); ++ unsigned nrblocks; + int i; + int ret; ++ int nr_pages = 0; ++ int nr_folios = folio_batch_count(fbatch); ++ ++ for (i = 0; i < nr_folios; i++) ++ nr_pages += folio_nr_pages(fbatch->folios[i]); ++ nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); + + ret = gfs2_trans_begin(sdp, nrblocks, nrblocks); + if (ret < 0) + return ret; + +- for(i = 0; i < nr_pages; i++) { +- struct page *page = pvec->pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch->folios[i]; + +- *done_index = page->index; ++ *done_index = folio->index; + +- lock_page(page); ++ folio_lock(folio); + +- if (unlikely(page->mapping != mapping)) { ++ if (unlikely(folio->mapping != mapping)) { + continue_unlock: +- unlock_page(page); ++ folio_unlock(folio); + continue; + } + +- if (!PageDirty(page)) { ++ if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + +- if (PageWriteback(page)) { ++ if (folio_test_writeback(folio)) { + if (wbc->sync_mode != WB_SYNC_NONE) +- wait_on_page_writeback(page); ++ folio_wait_writeback(folio); + else + goto continue_unlock; + } + +- BUG_ON(PageWriteback(page)); +- if (!clear_page_dirty_for_io(page)) ++ BUG_ON(folio_test_writeback(folio)); ++ if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + trace_wbc_writepage(wbc, inode_to_bdi(inode)); + +- ret = __gfs2_jdata_writepage(page, wbc); ++ ret = __gfs2_jdata_writepage(&folio->page, wbc); + if (unlikely(ret)) { + if (ret == AOP_WRITEPAGE_ACTIVATE) { +- unlock_page(page); ++ folio_unlock(folio); + ret = 0; + } else { + +@@ -268,7 +272,8 @@ static int gfs2_write_jdata_pagevec(struct address_space *mapping, + * not be suitable for data integrity + * writeout). + */ +- *done_index = page->index + 1; ++ *done_index = folio->index + ++ folio_nr_pages(folio); + ret = 1; + break; + } +@@ -305,8 +310,8 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + { + int ret = 0; + int done = 0; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ int nr_folios; + pgoff_t writeback_index; + pgoff_t index; + pgoff_t end; +@@ -315,7 +320,7 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + int range_whole = 0; + xa_mark_t tag; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + writeback_index = mapping->writeback_index; /* prev offset */ + index = writeback_index; +@@ -341,17 +346,18 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && (index <= end)) { +- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, +- tag); +- if (nr_pages == 0) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ tag, &fbatch); ++ if (nr_folios == 0) + break; + +- ret = gfs2_write_jdata_pagevec(mapping, wbc, &pvec, nr_pages, &done_index); ++ ret = gfs2_write_jdata_batch(mapping, wbc, &fbatch, ++ &done_index); + if (ret) + done = 1; + if (ret > 0) + ret = 0; +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + +diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c +index 9f4d9432d38a..1e26f32a4e36 100644 +--- a/fs/nilfs2/btree.c ++++ b/fs/nilfs2/btree.c +@@ -2143,7 +2143,7 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btcache = btnc_inode->i_mapping; + struct list_head lists[NILFS_BTREE_LEVEL_MAX]; +- struct pagevec pvec; ++ struct folio_batch fbatch; + struct buffer_head *bh, *head; + pgoff_t index = 0; + int level, i; +@@ -2153,19 +2153,19 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, + level++) + INIT_LIST_HEAD(&lists[level]); + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + +- while (pagevec_lookup_tag(&pvec, btcache, &index, +- PAGECACHE_TAG_DIRTY)) { +- for (i = 0; i < pagevec_count(&pvec); i++) { +- bh = head = page_buffers(pvec.pages[i]); ++ while (filemap_get_folios_tag(btcache, &index, (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch)) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ bh = head = folio_buffers(fbatch.folios[i]); + do { + if (buffer_dirty(bh)) + nilfs_btree_add_dirty_buffer(btree, + lists, bh); + } while ((bh = bh->b_this_page) != head); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + +diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c +index 3267e96c256c..b66f4e988016 100644 +--- a/fs/nilfs2/page.c ++++ b/fs/nilfs2/page.c +@@ -240,42 +240,43 @@ static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty) + int nilfs_copy_dirty_pages(struct address_space *dmap, + struct address_space *smap) + { +- struct pagevec pvec; ++ struct folio_batch fbatch; + unsigned int i; + pgoff_t index = 0; + int err = 0; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + repeat: +- if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY)) ++ if (!filemap_get_folios_tag(smap, &index, (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch)) + return 0; + +- for (i = 0; i < pagevec_count(&pvec); i++) { +- struct page *page = pvec.pages[i], *dpage; ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ struct folio *folio = fbatch.folios[i], *dfolio; + +- lock_page(page); +- if (unlikely(!PageDirty(page))) +- NILFS_PAGE_BUG(page, "inconsistent dirty state"); ++ folio_lock(folio); ++ if (unlikely(!folio_test_dirty(folio))) ++ NILFS_PAGE_BUG(&folio->page, "inconsistent dirty state"); + +- dpage = grab_cache_page(dmap, page->index); +- if (unlikely(!dpage)) { ++ dfolio = filemap_grab_folio(dmap, folio->index); ++ if (unlikely(!dfolio)) { + /* No empty page is added to the page cache */ + err = -ENOMEM; +- unlock_page(page); ++ folio_unlock(folio); + break; + } +- if (unlikely(!page_has_buffers(page))) +- NILFS_PAGE_BUG(page, ++ if (unlikely(!folio_buffers(folio))) ++ NILFS_PAGE_BUG(&folio->page, + "found empty page in dat page cache"); + +- nilfs_copy_page(dpage, page, 1); +- __set_page_dirty_nobuffers(dpage); ++ nilfs_copy_page(&dfolio->page, &folio->page, 1); ++ filemap_dirty_folio(folio_mapping(dfolio), dfolio); + +- unlock_page(dpage); +- put_page(dpage); +- unlock_page(page); ++ folio_unlock(dfolio); ++ folio_put(dfolio); ++ folio_unlock(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + + if (likely(!err)) +@@ -357,22 +358,22 @@ void nilfs_copy_back_pages(struct address_space *dmap, + */ + void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent) + { +- struct pagevec pvec; ++ struct folio_batch fbatch; + unsigned int i; + pgoff_t index = 0; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + +- while (pagevec_lookup_tag(&pvec, mapping, &index, +- PAGECACHE_TAG_DIRTY)) { +- for (i = 0; i < pagevec_count(&pvec); i++) { +- struct page *page = pvec.pages[i]; ++ while (filemap_get_folios_tag(mapping, &index, (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch)) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ struct folio *folio = fbatch.folios[i]; + +- lock_page(page); +- nilfs_clear_dirty_page(page, silent); +- unlock_page(page); ++ folio_lock(folio); ++ nilfs_clear_dirty_page(&folio->page, silent); ++ folio_unlock(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c +index 0afe0832c754..6f2ca279d230 100644 +--- a/fs/nilfs2/segment.c ++++ b/fs/nilfs2/segment.c +@@ -680,7 +680,7 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + loff_t start, loff_t end) + { + struct address_space *mapping = inode->i_mapping; +- struct pagevec pvec; ++ struct folio_batch fbatch; + pgoff_t index = 0, last = ULONG_MAX; + size_t ndirties = 0; + int i; +@@ -694,23 +694,26 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + index = start >> PAGE_SHIFT; + last = end >> PAGE_SHIFT; + } +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + repeat: + if (unlikely(index > last) || +- !pagevec_lookup_range_tag(&pvec, mapping, &index, last, +- PAGECACHE_TAG_DIRTY)) ++ !filemap_get_folios_tag(mapping, &index, last, ++ PAGECACHE_TAG_DIRTY, &fbatch)) + return ndirties; + +- for (i = 0; i < pagevec_count(&pvec); i++) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct buffer_head *bh, *head; +- struct page *page = pvec.pages[i]; ++ struct folio *folio = fbatch.folios[i]; + +- lock_page(page); +- if (!page_has_buffers(page)) +- create_empty_buffers(page, i_blocksize(inode), 0); +- unlock_page(page); ++ folio_lock(folio); ++ head = folio_buffers(folio); ++ if (!head) { ++ create_empty_buffers(&folio->page, i_blocksize(inode), 0); ++ head = folio_buffers(folio); ++ } ++ folio_unlock(folio); + +- bh = head = page_buffers(page); ++ bh = head; + do { + if (!buffer_dirty(bh) || buffer_async_write(bh)) + continue; +@@ -718,13 +721,13 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + list_add_tail(&bh->b_assoc_buffers, listp); + ndirties++; + if (unlikely(ndirties >= nlimit)) { +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + return ndirties; + } + } while (bh = bh->b_this_page, bh != head); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + goto repeat; + } +@@ -734,20 +737,19 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, + { + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode = ii->i_assoc_inode; +- struct pagevec pvec; ++ struct folio_batch fbatch; + struct buffer_head *bh, *head; + unsigned int i; + pgoff_t index = 0; + + if (!btnc_inode) + return; ++ folio_batch_init(&fbatch); + +- pagevec_init(&pvec); +- +- while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, +- PAGECACHE_TAG_DIRTY)) { +- for (i = 0; i < pagevec_count(&pvec); i++) { +- bh = head = page_buffers(pvec.pages[i]); ++ while (filemap_get_folios_tag(btnc_inode->i_mapping, &index, ++ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, &fbatch)) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ bh = head = folio_buffers(fbatch.folios[i]); + do { + if (buffer_dirty(bh) && + !buffer_async_write(bh)) { +@@ -758,7 +760,7 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, + bh = bh->b_this_page; + } while (bh != head); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h +index 39f05e00dac4..9fa4c4e0809b 100644 +--- a/include/linux/pagemap.h ++++ b/include/linux/pagemap.h +@@ -547,6 +547,26 @@ static inline struct folio *filemap_lock_folio(struct address_space *mapping, + return __filemap_get_folio(mapping, index, FGP_LOCK, 0); + } + ++/** ++ * filemap_grab_folio - grab a folio from the page cache ++ * @mapping: The address space to search ++ * @index: The page index ++ * ++ * Looks up the page cache entry at @mapping & @index. If no folio is found, ++ * a new folio is created. The folio is locked, marked as accessed, and ++ * returned. ++ * ++ * Return: A found or created folio. NULL if no folio is found and failed to ++ * create a folio. ++ */ ++static inline struct folio *filemap_grab_folio(struct address_space *mapping, ++ pgoff_t index) ++{ ++ return __filemap_get_folio(mapping, index, ++ FGP_LOCK | FGP_ACCESSED | FGP_CREAT, ++ mapping_gfp_mask(mapping)); ++} ++ + /** + * find_get_page - find and get a page reference + * @mapping: the address_space to search +@@ -720,16 +740,8 @@ unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, + pgoff_t end, struct folio_batch *fbatch); + unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, + unsigned int nr_pages, struct page **pages); +-unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, +- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, +- struct page **pages); +-static inline unsigned find_get_pages_tag(struct address_space *mapping, +- pgoff_t *index, xa_mark_t tag, unsigned int nr_pages, +- struct page **pages) +-{ +- return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag, +- nr_pages, pages); +-} ++unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, ++ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch); + + struct page *grab_cache_page_write_begin(struct address_space *mapping, + pgoff_t index); +diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h +index 215eb6c3bdc9..a520632297ac 100644 +--- a/include/linux/pagevec.h ++++ b/include/linux/pagevec.h +@@ -26,14 +26,6 @@ struct pagevec { + }; + + void __pagevec_release(struct pagevec *pvec); +-unsigned pagevec_lookup_range_tag(struct pagevec *pvec, +- struct address_space *mapping, pgoff_t *index, pgoff_t end, +- xa_mark_t tag); +-static inline unsigned pagevec_lookup_tag(struct pagevec *pvec, +- struct address_space *mapping, pgoff_t *index, xa_mark_t tag) +-{ +- return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag); +-} + + static inline void pagevec_init(struct pagevec *pvec) + { +diff --git a/mm/filemap.c b/mm/filemap.c +index 15800334147b..b986f246a6ae 100644 +--- a/mm/filemap.c ++++ b/mm/filemap.c +@@ -503,28 +503,30 @@ static void __filemap_fdatawait_range(struct address_space *mapping, + { + pgoff_t index = start_byte >> PAGE_SHIFT; + pgoff_t end = end_byte >> PAGE_SHIFT; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ unsigned nr_folios; + + if (end_byte < start_byte) + return; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); ++ + while (index <= end) { + unsigned i; + +- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, +- end, PAGECACHE_TAG_WRITEBACK); +- if (!nr_pages) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ PAGECACHE_TAG_WRITEBACK, &fbatch); ++ ++ if (!nr_folios) + break; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- wait_on_page_writeback(page); +- ClearPageError(page); ++ folio_wait_writeback(folio); ++ folio_clear_error(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +@@ -2255,64 +2257,57 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, + EXPORT_SYMBOL(find_get_pages_contig); + + /** +- * find_get_pages_range_tag - Find and return head pages matching @tag. +- * @mapping: the address_space to search +- * @index: the starting page index +- * @end: The final page index (inclusive) +- * @tag: the tag index +- * @nr_pages: the maximum number of pages +- * @pages: where the resulting pages are placed ++ * filemap_get_folios_tag - Get a batch of folios matching @tag. ++ * @mapping: The address_space to search ++ * @start: The starting page index ++ * @end: The final page index (inclusive) ++ * @tag: The tag index ++ * @fbatch: The batch to fill + * +- * Like find_get_pages_range(), except we only return head pages which are +- * tagged with @tag. @index is updated to the index immediately after the +- * last page we return, ready for the next iteration. ++ * Same as filemap_get_folios, but only returning folios tagged with @tag + * +- * Return: the number of pages which were found. ++ * Return: The number of folios found ++ * Also update @start to index the next folio for traversal + */ +-unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, +- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, +- struct page **pages) ++unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, ++ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) + { +- XA_STATE(xas, &mapping->i_pages, *index); ++ XA_STATE(xas, &mapping->i_pages, *start); + struct folio *folio; +- unsigned ret = 0; +- +- if (unlikely(!nr_pages)) +- return 0; + + rcu_read_lock(); +- while ((folio = find_get_entry(&xas, end, tag))) { +- /* +- * Shadow entries should never be tagged, but this iteration ++ while ((folio = find_get_entry(&xas, end, tag)) != NULL) { ++ /* Shadow entries should never be tagged, but this iteration + * is lockless so there is a window for page reclaim to evict +- * a page we saw tagged. Skip over it. ++ * a page we saw tagged. Skip over it. + */ + if (xa_is_value(folio)) + continue; ++ if (!folio_batch_add(fbatch, folio)) { ++ unsigned long nr = folio_nr_pages(folio); + +- pages[ret] = &folio->page; +- if (++ret == nr_pages) { +- *index = folio->index + folio_nr_pages(folio); ++ if (folio_test_hugetlb(folio)) ++ nr = 1; ++ *start = folio->index + nr; + goto out; + } + } +- + /* +- * We come here when we got to @end. We take care to not overflow the +- * index @index as it confuses some of the callers. This breaks the +- * iteration when there is a page at index -1 but that is already +- * broken anyway. ++ * We come here when there is no page beyond @end. We take care to not ++ * overflow the index @start as it confuses some of the callers. This ++ * breaks the iteration when there is a page at index -1 but that is ++ * already broke anyway. + */ + if (end == (pgoff_t)-1) +- *index = (pgoff_t)-1; ++ *start = (pgoff_t)-1; + else +- *index = end + 1; ++ *start = end + 1; + out: + rcu_read_unlock(); + +- return ret; ++ return folio_batch_count(fbatch); + } +-EXPORT_SYMBOL(find_get_pages_range_tag); ++EXPORT_SYMBOL(filemap_get_folios_tag); + + /* + * CD/DVDs are error prone. When a medium error occurs, the driver may fail +diff --git a/mm/page-writeback.c b/mm/page-writeback.c +index cbcecd565c16..8f20c037a87a 100644 +--- a/mm/page-writeback.c ++++ b/mm/page-writeback.c +@@ -2293,15 +2293,15 @@ int write_cache_pages(struct address_space *mapping, + int ret = 0; + int done = 0; + int error; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ int nr_folios; + pgoff_t index; + pgoff_t end; /* Inclusive */ + pgoff_t done_index; + int range_whole = 0; + xa_mark_t tag; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + index = mapping->writeback_index; /* prev offset */ + end = -1; +@@ -2321,17 +2321,18 @@ int write_cache_pages(struct address_space *mapping, + while (!done && (index <= end)) { + int i; + +- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, +- tag); +- if (nr_pages == 0) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ tag, &fbatch); ++ ++ if (nr_folios == 0) + break; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- done_index = page->index; ++ done_index = folio->index; + +- lock_page(page); ++ folio_lock(folio); + + /* + * Page truncated or invalidated. We can freely skip it +@@ -2341,30 +2342,30 @@ int write_cache_pages(struct address_space *mapping, + * even if there is now a new, dirty page at the same + * pagecache address. + */ +- if (unlikely(page->mapping != mapping)) { ++ if (unlikely(folio->mapping != mapping)) { + continue_unlock: +- unlock_page(page); ++ folio_unlock(folio); + continue; + } + +- if (!PageDirty(page)) { ++ if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + +- if (PageWriteback(page)) { ++ if (folio_test_writeback(folio)) { + if (wbc->sync_mode != WB_SYNC_NONE) +- wait_on_page_writeback(page); ++ folio_wait_writeback(folio); + else + goto continue_unlock; + } + +- BUG_ON(PageWriteback(page)); +- if (!clear_page_dirty_for_io(page)) ++ BUG_ON(folio_test_writeback(folio)); ++ if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); +- error = (*writepage)(page, wbc, data); ++ error = writepage(&folio->page, wbc, data); + if (unlikely(error)) { + /* + * Handle errors according to the type of +@@ -2379,11 +2380,12 @@ int write_cache_pages(struct address_space *mapping, + * the first error. + */ + if (error == AOP_WRITEPAGE_ACTIVATE) { +- unlock_page(page); ++ folio_unlock(folio); + error = 0; + } else if (wbc->sync_mode != WB_SYNC_ALL) { + ret = error; +- done_index = page->index + 1; ++ done_index = folio->index + ++ folio_nr_pages(folio); + done = 1; + break; + } +@@ -2403,7 +2405,7 @@ int write_cache_pages(struct address_space *mapping, + break; + } + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + +diff --git a/mm/swap.c b/mm/swap.c +index 027943b341a0..29bf97746da0 100644 +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -1099,16 +1099,6 @@ void folio_batch_remove_exceptionals(struct folio_batch *fbatch) + fbatch->nr = j; + } + +-unsigned pagevec_lookup_range_tag(struct pagevec *pvec, +- struct address_space *mapping, pgoff_t *index, pgoff_t end, +- xa_mark_t tag) +-{ +- pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, +- PAGEVEC_SIZE, pvec->pages); +- return pagevec_count(pvec); +-} +-EXPORT_SYMBOL(pagevec_lookup_range_tag); +- + /* + * Perform any setup for the swap system + */ +-- +2.38.0 + +From d33721e72a6bd6579920c553cb1cd11acb809ede Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 5 Oct 2022 19:09:43 +0200 +Subject: [PATCH 11/17] fixes + +Signed-off-by: Peter Jung +--- + Documentation/ABI/stable/sysfs-block | 10 + + .../testing/sysfs-class-led-trigger-blkdev | 78 ++ + Documentation/leds/index.rst | 1 + + Documentation/leds/ledtrig-blkdev.rst | 158 +++ + arch/x86/Kconfig | 10 + + arch/x86/boot/compressed/Makefile | 5 +- + arch/x86/boot/compressed/head_64.S | 8 +- + arch/x86/boot/compressed/misc.c | 35 +- + arch/x86/events/amd/uncore.c | 1 + + arch/x86/kernel/alternative.c | 9 + + arch/x86/net/bpf_jit_comp.c | 4 +- + arch/x86/purgatory/purgatory.c | 7 + + drivers/leds/trigger/Kconfig | 9 + + drivers/leds/trigger/Makefile | 1 + + drivers/leds/trigger/ledtrig-blkdev.c | 1220 +++++++++++++++++ + include/linux/kexec.h | 9 +- + include/linux/pageblock-flags.h | 2 +- + include/net/bluetooth/rfcomm.h | 3 + + include/uapi/linux/kexec.h | 2 + + kernel/kexec.c | 19 +- + kernel/kexec_core.c | 16 +- + kernel/kexec_file.c | 20 +- + lib/vdso/Makefile | 2 +- + mm/huge_memory.c | 3 +- + net/bluetooth/rfcomm/core.c | 2 + + net/bluetooth/rfcomm/sock.c | 34 +- + scripts/Makefile.lib | 5 + + tools/objtool/check.c | 3 - + 28 files changed, 1635 insertions(+), 41 deletions(-) + create mode 100644 Documentation/ABI/testing/sysfs-class-led-trigger-blkdev + create mode 100644 Documentation/leds/ledtrig-blkdev.rst + create mode 100644 drivers/leds/trigger/ledtrig-blkdev.c + +diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block +index cd14ecb3c9a5..853cb2601242 100644 +--- a/Documentation/ABI/stable/sysfs-block ++++ b/Documentation/ABI/stable/sysfs-block +@@ -101,6 +101,16 @@ Description: + devices that support receiving integrity metadata. + + ++What: /sys/block//linked_leds ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Directory that contains symbolic links to all LEDs that ++ are associated with (linked to) this block device by the ++ blkdev LED trigger. Only present when at least one LED ++ is linked. (See Documentation/leds/ledtrig-blkdev.rst.) ++ ++ + What: /sys/block///alignment_offset + Date: April 2009 + Contact: Martin K. Petersen +diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev +new file mode 100644 +index 000000000000..45275eb0bad3 +--- /dev/null ++++ b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev +@@ -0,0 +1,78 @@ ++What: /sys/class/leds//blink_time ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Time (in milliseconds) that the LED will be on during a single ++ "blink". ++ ++What: /sys/class/leds//check_interval ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Interval (in milliseconds) between checks of the block devices ++ linked to this LED. The LED will be blinked if the correct type ++ of activity (see blink_on_{read,write,discard,flush} attributes) ++ has occurred on any of the linked devices since the previous ++ check. ++ ++What: /sys/class/leds//blink_on_read ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to read activity on any of its linked block devices. ++ ++What: /sys/class/leds//blink_on_write ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to write activity on any of its linked block devices. ++ ++What: /sys/class/leds//blink_on_discard ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to discard activity on any of its linked block devices. ++ ++What: /sys/class/leds//blink_on_flush ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to cache flush activity on any of its linked block devices. ++ ++What: /sys/class/leds//link_dev_by_path ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Associate a block device with this LED by writing the path to ++ the device special file (e.g. /dev/sda) to this attribute. ++ Symbolic links are followed. ++ ++What: /sys/class/leds//unlink_dev_by_path ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Remove the association between this LED and a block device by ++ writing the path to the device special file (e.g. /dev/sda) to ++ this attribute. Symbolic links are followed. ++ ++What: /sys/class/leds//unlink_dev_by_name ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Remove the association between this LED and a block device by ++ writing the kernel name of the device (e.g. sda) to this ++ attribute. ++ ++What: /sys/class/leds//linked_devices ++Date: October 2022 ++Contact: Ian Pilcher ++Description: ++ Directory containing links to all block devices that are ++ associated with this LED. (Note that the names of the ++ symbolic links in this directory are *kernel* names, which ++ may not match the device special file paths written to ++ link_device and unlink_device.) +diff --git a/Documentation/leds/index.rst b/Documentation/leds/index.rst +index e5d63b940045..e3c24e468cbc 100644 +--- a/Documentation/leds/index.rst ++++ b/Documentation/leds/index.rst +@@ -10,6 +10,7 @@ LEDs + leds-class + leds-class-flash + leds-class-multicolor ++ ledtrig-blkdev + ledtrig-oneshot + ledtrig-transient + ledtrig-usbport +diff --git a/Documentation/leds/ledtrig-blkdev.rst b/Documentation/leds/ledtrig-blkdev.rst +new file mode 100644 +index 000000000000..9ff5b99de451 +--- /dev/null ++++ b/Documentation/leds/ledtrig-blkdev.rst +@@ -0,0 +1,158 @@ ++.. SPDX-License-Identifier: GPL-2.0 ++ ++================================= ++Block Device (blkdev) LED Trigger ++================================= ++ ++Available when ``CONFIG_LEDS_TRIGGER_BLKDEV=y`` or ++``CONFIG_LEDS_TRIGGER_BLKDEV=m``. ++ ++See also: ++ ++* ``Documentation/ABI/testing/sysfs-class-led-trigger-blkdev`` ++* ``Documentation/ABI/stable/sysfs-block`` (``/sys/block//linked_leds``) ++ ++Overview ++======== ++ ++.. note:: ++ The examples below use ```` to refer to the name of a ++ system-specific LED. If no suitable LED is available on a test ++ system (in a virtual machine, for example), it is possible to ++ use a userspace LED. (See ``Documentation/leds/uleds.rst``.) ++ ++Verify that the ``blkdev`` LED trigger is available:: ++ ++ # grep blkdev /sys/class/leds//trigger ++ ... rfkill-none blkdev ++ ++(If the previous command produces no output, you may need to load the trigger ++module - ``modprobe ledtrig_blkdev``. If the module is not available, check ++the value of ``CONFIG_LEDS_TRIGGER_BLKDEV`` in your kernel configuration.) ++ ++Associate the LED with the ``blkdev`` LED trigger:: ++ ++ # echo blkdev > /sys/class/leds//trigger ++ ++ # cat /sys/class/leds//trigger ++ ... rfkill-none [blkdev] ++ ++Note that several new device attributes are available in the ++``/sys/class/leds/`` directory. ++ ++* ``link_dev_by_path``, ``unlink_dev_by_path``, and ``unlink_dev_by_name`` are ++ used to manage the set of block devices associated with this LED. The LED ++ will blink when activity occurs on any of its linked devices. ++ ++* ``blink_on_read``, ``blink_on_write``, ``blink_on_discard``, and ++ ``blink_on_flush`` are boolean values that determine whether the LED will ++ blink when a particular type of activity is detected on one of its linked ++ block devices. ++ ++* ``blink_time`` is the duration (in milliseconds) of each blink of this LED. ++ (The minimum value is 10 milliseconds.) ++ ++* ``check_interval`` is the frequency (in milliseconds) with which block devices ++ linked to this LED will be checked for activity and the LED blinked (if the ++ correct type of activity has occurred). ++ ++* The ``linked_devices`` directory will contain a symbolic link to every device ++ that is associated with this LED. ++ ++Link a block device to the LED:: ++ ++ # echo /dev/sda > /sys/class/leds//link_dev_by_path ++ ++ # ls /sys/class/leds//linked_devices ++ sda ++ ++(The value written to ``link_dev_by_path`` must be the path of the device ++special file, such as ``/dev/sda``, that represents the block device - or the ++path of a symbolic link to such a device special file.) ++ ++Activity on the device will now cause the LED to blink. The duration of each ++blink (in milliseconds) can be adjusted by setting ++``/sys/class/leds//blink_time``. (But see **check_interval and ++blink_time** below.) ++ ++Associate a second device with the LED:: ++ ++ # echo /dev/sdb > /sys/class/leds//link_dev_by_path ++ ++ # ls /sys/class/leds//linked_devices ++ sda sdb ++ ++When a block device is linked to one or more LEDs, the LEDs are linked from ++the device's ``linked_leds`` directory:: ++ ++ # ls /sys/class/block/sd{a,b}/linked_leds ++ /sys/class/block/sda/linked_leds: ++ ++ ++ /sys/class/block/sdb/linked_leds: ++ ++ ++(The ``linked_leds`` directory only exists when the block device is linked to ++at least one LED.) ++ ++``check_interval`` and ``blink_time`` ++===================================== ++ ++* By default, linked block devices are checked for activity every 100 ++ milliseconds. This frequency can be changed for an LED via the ++ ``/sys/class/leds//check_interval`` attribute. (The minimum value is 25 ++ milliseconds.) ++ ++* All block devices associated with an LED are checked for activity every ++ ``check_interval`` milliseconds, and a blink is triggered if the correct type ++ of activity (as determined by the LED's ``blink_on_*`` attributes) is ++ detected. The duration of an LED's blink is determined by its ``blink_time`` ++ attribute. Thus (when the correct type of activity is detected), the LED will ++ be on for ``blink_time`` milliseconds and off for ++ ``check_interval - blink_time`` milliseconds. ++ ++* The LED subsystem ignores new blink requests for an LED that is already in ++ in the process of blinking, so setting a ``blink_time`` greater than or equal ++ to ``check_interval`` will cause some blinks to be missed. ++ ++* Because of processing times, scheduling latencies, etc., avoiding missed ++ blinks actually requires a difference of at least a few milliseconds between ++ the ``blink_time`` and ``check_interval``. The required difference is likely ++ to vary from system to system. As a reference, a Thecus N5550 NAS requires a ++ difference of 7 milliseconds (e.g. ``check_interval == 100``, ++ ``blink_time == 93``). ++ ++* The default values (``check_interval == 100``, ``blink_time == 75``) cause the ++ LED associated with a continuously active device to blink rapidly. For a more ++ "always on" effect, increase the ``blink_time`` (but not too much; see the ++ previous bullet). ++ ++Other Notes ++=========== ++ ++* Many (possibly all) types of block devices work with this trigger, including: ++ ++ * SCSI (including SATA and USB) hard disk drives and SSDs ++ * SCSI (including SATA and USB) optical drives ++ * NVMe SSDs ++ * SD cards ++ * loopback block devices (``/dev/loop*``) ++ * device mapper devices, such as LVM logical volumes ++ * MD RAID devices ++ * zRAM compressed RAM-disks ++ * partitions on block devices that support them ++ ++* The names of the symbolic links in ``/sys/class/leds//linked_devices`` ++ are **kernel** names, which may not match the paths used for ++ ``link_dev_by_path`` and ``unlink_dev_by_path``. This is most likely when a ++ symbolic link is used to refer to the device (as is common with logical ++ volumes), but it can be true for any device, because nothing prevents the ++ creation of device special files with arbitrary names (e.g. ++ ``sudo mknod /foo b 8 0``). ++ ++ Kernel names can be used to unlink block devices from LEDs by writing them to ++ the LED's ``unlink_dev_by_name`` attribute. ++ ++* The ``blkdev`` LED trigger supports many-to-many device/LED associations. ++ A device can be associated with multiple LEDs, and an LED can be associated ++ with multiple devices. +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index 674d694a665e..c8e85485b104 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -221,6 +221,7 @@ config X86 + select HAVE_KERNEL_LZO + select HAVE_KERNEL_XZ + select HAVE_KERNEL_ZSTD ++ select HAVE_KERNEL_UNCOMPRESSED + select HAVE_KPROBES + select HAVE_KPROBES_ON_FTRACE + select HAVE_FUNCTION_ERROR_INJECTION +@@ -2043,6 +2044,15 @@ config KEXEC_BZIMAGE_VERIFY_SIG + help + Enable bzImage signature verification support. + ++config KEXEC_PURGATORY_SKIP_SIG ++ bool "skip kexec purgatory signature verification" ++ depends on ARCH_HAS_KEXEC_PURGATORY ++ help ++ this options makes the kexec purgatory do not signature verification ++ which would get hundreds of milliseconds saved during kexec boot. If we can ++ confirm that the data of each segment loaded by kexec will not change we may ++ enable this option ++ + config CRASH_DUMP + bool "kernel crash dumps" + depends on X86_64 || (X86_32 && HIGHMEM) +diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile +index eba7709d75ae..b52eec53fa88 100644 +--- a/arch/x86/boot/compressed/Makefile ++++ b/arch/x86/boot/compressed/Makefile +@@ -26,7 +26,7 @@ OBJECT_FILES_NON_STANDARD := y + KCOV_INSTRUMENT := n + + targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ +- vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst ++ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst vmlinux.bin.none + + # CLANG_FLAGS must come before any cc-disable-warning or cc-option calls in + # case of cross compiling, as it has the '--target=' flag, which is needed to +@@ -141,6 +141,8 @@ $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE + $(call if_changed,lz4_with_size) + $(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE + $(call if_changed,zstd22_with_size) ++$(obj)/vmlinux.bin.none: $(vmlinux.bin.all-y) FORCE ++ $(call if_changed,none) + + suffix-$(CONFIG_KERNEL_GZIP) := gz + suffix-$(CONFIG_KERNEL_BZIP2) := bz2 +@@ -149,6 +151,7 @@ suffix-$(CONFIG_KERNEL_XZ) := xz + suffix-$(CONFIG_KERNEL_LZO) := lzo + suffix-$(CONFIG_KERNEL_LZ4) := lz4 + suffix-$(CONFIG_KERNEL_ZSTD) := zst ++suffix-$(CONFIG_KERNEL_UNCOMPRESSED) := none + + quiet_cmd_mkpiggy = MKPIGGY $@ + cmd_mkpiggy = $(obj)/mkpiggy $< > $@ +diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S +index d33f060900d2..9e7770c7047b 100644 +--- a/arch/x86/boot/compressed/head_64.S ++++ b/arch/x86/boot/compressed/head_64.S +@@ -398,10 +398,13 @@ SYM_CODE_START(startup_64) + 1: + + /* Target address to relocate to for decompression */ ++#ifdef CONFIG_KERNEL_UNCOMPRESSED ++ movq %rbp, %rbx ++#else + movl BP_init_size(%rsi), %ebx + subl $ rva(_end), %ebx + addq %rbp, %rbx +- ++#endif + /* Set up the stack */ + leaq rva(boot_stack_end)(%rbx), %rsp + +@@ -522,6 +525,7 @@ trampoline_return: + * Copy the compressed kernel to the end of our buffer + * where decompression in place becomes safe. + */ ++#ifndef CONFIG_KERNEL_UNCOMPRESSED + pushq %rsi + leaq (_bss-8)(%rip), %rsi + leaq rva(_bss-8)(%rbx), %rdi +@@ -531,7 +535,7 @@ trampoline_return: + rep movsq + cld + popq %rsi +- ++#endif + /* + * The GDT may get overwritten either during the copy we just did or + * during extract_kernel below. To avoid any issues, repoint the GDTR +diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c +index cf690d8712f4..d8445562d4e9 100644 +--- a/arch/x86/boot/compressed/misc.c ++++ b/arch/x86/boot/compressed/misc.c +@@ -181,6 +181,19 @@ void __puthex(unsigned long value) + } + } + ++#ifdef CONFIG_KERNEL_UNCOMPRESSED ++#include ++static int __decompress(unsigned char *buf, long len, ++ long (*fill)(void*, unsigned long), ++ long (*flush)(void*, unsigned long), ++ unsigned char *outbuf, long olen, ++ long *pos, void (*error)(char *x)) ++{ ++ memcpy(outbuf, buf, olen); ++ return 0; ++} ++#endif ++ + #ifdef CONFIG_X86_NEED_RELOCS + static void handle_relocations(void *output, unsigned long output_len, + unsigned long virt_addr) +@@ -277,7 +290,7 @@ static inline void handle_relocations(void *output, unsigned long output_len, + { } + #endif + +-static void parse_elf(void *output) ++static void parse_elf(void *output, void *input) + { + #ifdef CONFIG_X86_64 + Elf64_Ehdr ehdr; +@@ -289,7 +302,7 @@ static void parse_elf(void *output) + void *dest; + int i; + +- memcpy(&ehdr, output, sizeof(ehdr)); ++ memcpy(&ehdr, input, sizeof(ehdr)); + if (ehdr.e_ident[EI_MAG0] != ELFMAG0 || + ehdr.e_ident[EI_MAG1] != ELFMAG1 || + ehdr.e_ident[EI_MAG2] != ELFMAG2 || +@@ -304,7 +317,7 @@ static void parse_elf(void *output) + if (!phdrs) + error("Failed to allocate space for phdrs"); + +- memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum); ++ memcpy(phdrs, input + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum); + + for (i = 0; i < ehdr.e_phnum; i++) { + phdr = &phdrs[i]; +@@ -321,7 +334,7 @@ static void parse_elf(void *output) + #else + dest = (void *)(phdr->p_paddr); + #endif +- memmove(dest, output + phdr->p_offset, phdr->p_filesz); ++ memmove(dest, input + phdr->p_offset, phdr->p_filesz); + break; + default: /* Ignore other PT_* */ break; + } +@@ -454,9 +467,21 @@ asmlinkage __visible void *extract_kernel(void *rmode, memptr heap, + #endif + + debug_putstr("\nDecompressing Linux... "); ++ ++#ifdef CONFIG_KERNEL_UNCOMPRESSED ++ if (cmdline_find_option_bool("nokaslr")) { ++ parse_elf(output, input_data); ++ } else { ++ __decompress(input_data, input_len, NULL, NULL, output, output_len, ++ NULL, error); ++ parse_elf(output, output); ++ } ++#else + __decompress(input_data, input_len, NULL, NULL, output, output_len, + NULL, error); +- parse_elf(output); ++ parse_elf(output, output); ++#endif ++ + handle_relocations(output, output_len, virt_addr); + debug_putstr("done.\nBooting the kernel.\n"); + +diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c +index d568afc705d2..83f15fe411b3 100644 +--- a/arch/x86/events/amd/uncore.c ++++ b/arch/x86/events/amd/uncore.c +@@ -553,6 +553,7 @@ static void uncore_clean_online(void) + + hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { + hlist_del(&uncore->node); ++ kfree(uncore->events); + kfree(uncore); + } + } +diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c +index 4f3204364caa..5cadcea035e0 100644 +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) + return ret; + i += ret; + ++ /* ++ * The compiler is supposed to EMIT an INT3 after every unconditional ++ * JMP instruction due to AMD BTC. However, if the compiler is too old ++ * or SLS isn't enabled, we still need an INT3 after indirect JMPs ++ * even on Intel. ++ */ ++ if (op == JMP32_INSN_OPCODE && i < insn->length) ++ bytes[i++] = INT3_INSN_OPCODE; ++ + for (; i < insn->length;) + bytes[i++] = BYTES_NOP1; + +diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c +index c1f6c1c51d99..4922517ddb0d 100644 +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) + OPTIMIZER_HIDE_VAR(reg); + emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); + } else { +- EMIT2(0xFF, 0xE0 + reg); ++ EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ ++ if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS)) ++ EMIT1(0xCC); /* int3 */ + } + + *pprog = prog; +diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c +index 7558139920f8..b3f15774d86d 100644 +--- a/arch/x86/purgatory/purgatory.c ++++ b/arch/x86/purgatory/purgatory.c +@@ -20,6 +20,12 @@ u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE] __section(".kexec-purgatory"); + + struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(".kexec-purgatory"); + ++#ifdef CONFIG_KEXEC_PURGATORY_SKIP_SIG ++static int verify_sha256_digest(void) ++{ ++ return 0; ++} ++#else + static int verify_sha256_digest(void) + { + struct kexec_sha_region *ptr, *end; +@@ -39,6 +45,7 @@ static int verify_sha256_digest(void) + + return 0; + } ++#endif + + void purgatory(void) + { +diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig +index dc6816d36d06..bda249068182 100644 +--- a/drivers/leds/trigger/Kconfig ++++ b/drivers/leds/trigger/Kconfig +@@ -154,4 +154,13 @@ config LEDS_TRIGGER_TTY + + When build as a module this driver will be called ledtrig-tty. + ++config LEDS_TRIGGER_BLKDEV ++ tristate "LED Trigger for block devices" ++ depends on BLOCK ++ help ++ The blkdev LED trigger allows LEDs to be controlled by block device ++ activity (reads and writes). ++ ++ See Documentation/leds/ledtrig-blkdev.rst. ++ + endif # LEDS_TRIGGERS +diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile +index 25c4db97cdd4..d53bab5d93f1 100644 +--- a/drivers/leds/trigger/Makefile ++++ b/drivers/leds/trigger/Makefile +@@ -16,3 +16,4 @@ obj-$(CONFIG_LEDS_TRIGGER_NETDEV) += ledtrig-netdev.o + obj-$(CONFIG_LEDS_TRIGGER_PATTERN) += ledtrig-pattern.o + obj-$(CONFIG_LEDS_TRIGGER_AUDIO) += ledtrig-audio.o + obj-$(CONFIG_LEDS_TRIGGER_TTY) += ledtrig-tty.o ++obj-$(CONFIG_LEDS_TRIGGER_BLKDEV) += ledtrig-blkdev.o +diff --git a/drivers/leds/trigger/ledtrig-blkdev.c b/drivers/leds/trigger/ledtrig-blkdev.c +new file mode 100644 +index 000000000000..8614e308fadc +--- /dev/null ++++ b/drivers/leds/trigger/ledtrig-blkdev.c +@@ -0,0 +1,1220 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++ ++/* ++ * Block device LED trigger ++ * ++ * Copyright 2021-2022 Ian Pilcher ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * DOC: Overview ++ * ++ * The ``blkdev`` LED trigger works by periodically checking the activity ++ * counters of block devices that have been linked to one or more LEDs and ++ * blinking those LED(s) if the correct type of activity has occurred. The ++ * periodic check is scheduled with the Linux kernel's deferred work facility. ++ * ++ * Trigger-specific data about block devices and LEDs is stored in two data ++ * structures --- &struct blkdev_trig_bdev (a "BTB") and &struct blkdev_trig_led ++ * (a "BTL"). Each structure contains a &struct xarray that holds links to any ++ * linked devices of the other type. I.e. &blkdev_trig_bdev.linked_btls ++ * contains links to all BTLs whose LEDs have been linked to the BTB's block ++ * device, and &blkdev_trig_led.linked_btbs contains links to all BTBs whose ++ * block devices have been linked to the BTL's LED. Thus, a block device can ++ * be linked to more than one LED, and an LED can be linked to more than one ++ * block device. ++ */ ++ ++/* Default, minimum & maximum blink duration (milliseconds) */ ++#define BLKDEV_TRIG_BLINK_DEF 75 ++#define BLKDEV_TRIG_BLINK_MIN 10 ++#define BLKDEV_TRIG_BLINK_MAX 86400000 /* 24 hours */ ++ ++/* Default, minimum & maximum activity check interval (milliseconds) */ ++#define BLKDEV_TRIG_CHECK_DEF 100 ++#define BLKDEV_TRIG_CHECK_MIN 25 ++#define BLKDEV_TRIG_CHECK_MAX 86400000 /* 24 hours */ ++ ++/* ++ * If blkdev_trig_check() can't lock the mutex, how long to wait before trying ++ * again (milliseconds) ++ */ ++#define BLKDEV_TRIG_CHECK_RETRY 5 ++ ++/* Mode argument for calls to blkdev_get_by_path() and blkdev_put() */ ++#define BLKDEV_TRIG_FMODE 0 ++ ++/** ++ * struct blkdev_trig_bdev - Trigger-specific data about a block device. ++ * @last_checked: Time (in jiffies) at which the trigger last checked this ++ * block device for activity. ++ * @last_activity: Time (in jiffies) at which the trigger last detected ++ * activity of each type. ++ * @ios: Activity counter values for each type, corresponding to ++ * the timestamps in &last_activity. ++ * @index: &xarray index, so the BTB can be included in one or more ++ * &blkdev_trig_led.linked_btbs. ++ * @bdev: The block device. ++ * @linked_btls: The BTLs that represent the LEDs linked to the BTB's ++ * block device. ++ * ++ * Every block device linked to at least one LED gets a "BTB." A BTB is created ++ * when a block device that is not currently linked to any LEDs is linked to an ++ * LED. ++ * ++ * A BTB is freed when one of the following occurs: ++ * ++ * * The number of LEDs linked to the block device becomes zero, because it has ++ * been unlinked from its last LED using the trigger's &sysfs interface. ++ * ++ * * The number of LEDs linked to the block device becomes zero, because the ++ * last LED to which it was linked has been disassociated from the trigger ++ * (which happens automatically if the LED device is removed from the system). ++ * ++ * * The BTB's block device is removed from the system. To accomodate this ++ * scenario, BTB's are created as device resources, so that the release ++ * function will be called by the driver core when the device is removed. ++ */ ++struct blkdev_trig_bdev { ++ unsigned long last_checked; ++ unsigned long last_activity[NR_STAT_GROUPS]; ++ unsigned long ios[NR_STAT_GROUPS]; ++ unsigned long index; ++ struct block_device *bdev; ++ struct xarray linked_btls; ++}; ++ ++/** ++ * struct blkdev_trig_led - Trigger-specific data about an LED. ++ * @last_checked: Time (in jiffies) at which the trigger last checked the ++ * the block devices linked to this LED for activity. ++ * @index: &xarray index, so the BTL can be included in one or more ++ * &blkdev_trig_bdev.linked_btls. ++ * @mode: Bitmask for types of block device activity that will ++ * cause this LED to blink --- reads, writes, discards, ++ * etc. ++ * @led: The LED device. ++ * @blink_msec: Duration of a blink (milliseconds). ++ * @check_jiffies: Frequency with which block devices linked to this LED ++ * should be checked for activity (jiffies). ++ * @linked_btbs: The BTBs that represent the block devices linked to the ++ * BTL's LED. ++ * @all_btls_node: The BTL's node in the module's list of all BTLs. ++ * ++ * Every LED associated with the block device trigger gets a "BTL." A BTL is ++ * created when the trigger is "activated" on an LED (usually by writing ++ * ``blkdev`` to the LED's &sysfs &trigger attribute). A BTL is freed wnen its ++ * LED is disassociated from the trigger, either through the trigger's &sysfs ++ * interface or because the LED device is removed from the system. ++ */ ++struct blkdev_trig_led { ++ unsigned long last_checked; ++ unsigned long index; ++ unsigned long mode; /* must be ulong for atomic bit ops */ ++ struct led_classdev *led; ++ unsigned int blink_msec; ++ unsigned int check_jiffies; ++ struct xarray linked_btbs; ++ struct hlist_node all_btls_node; ++}; ++ ++/* Protects everything except atomic LED attributes */ ++static DEFINE_MUTEX(blkdev_trig_mutex); ++ ++/* BTB device resource release function */ ++static void blkdev_trig_btb_release(struct device *dev, void *res); ++ ++/* Index for next BTB or BTL */ ++static unsigned long blkdev_trig_next_index; ++ ++/* All LEDs associated with the trigger */ ++static HLIST_HEAD(blkdev_trig_all_btls); ++ ++/* Delayed work to periodically check for activity & blink LEDs */ ++static void blkdev_trig_check(struct work_struct *work); ++static DECLARE_DELAYED_WORK(blkdev_trig_work, blkdev_trig_check); ++ ++/* When is the delayed work scheduled to run next (jiffies) */ ++static unsigned long blkdev_trig_next_check; ++ ++/* Total number of BTB-to-BTL links */ ++static unsigned int blkdev_trig_link_count; ++ ++/* Empty sysfs attribute list for next 2 declarations */ ++static struct attribute *blkdev_trig_attrs_empty[] = { NULL }; ++ ++/* linked_leds sysfs directory for block devs linked to 1 or more LEDs */ ++static const struct attribute_group blkdev_trig_linked_leds = { ++ .name = "linked_leds", ++ .attrs = blkdev_trig_attrs_empty, ++}; ++ ++/* linked_devices sysfs directory for each LED associated with the trigger */ ++static const struct attribute_group blkdev_trig_linked_devs = { ++ .name = "linked_devices", ++ .attrs = blkdev_trig_attrs_empty, ++}; ++ ++ ++/* ++ * ++ * Delayed work to check for activity & blink LEDs ++ * ++ */ ++ ++/** ++ * blkdev_trig_blink() - Blink an LED, if the correct type of activity has ++ * occurred on the block device. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Return: &true if the LED is blinked, &false if not. ++ */ ++static bool blkdev_trig_blink(const struct blkdev_trig_led *btl, ++ const struct blkdev_trig_bdev *btb) ++{ ++ unsigned long mode, mask, delay_on, delay_off; ++ enum stat_group i; ++ ++ mode = READ_ONCE(btl->mode); ++ ++ for (i = STAT_READ, mask = 1; i <= STAT_FLUSH; ++i, mask <<= 1) { ++ ++ if (!(mode & mask)) ++ continue; ++ ++ if (time_before_eq(btb->last_activity[i], btl->last_checked)) ++ continue; ++ ++ delay_on = READ_ONCE(btl->blink_msec); ++ delay_off = 1; /* 0 leaves LED turned on */ ++ ++ led_blink_set_oneshot(btl->led, &delay_on, &delay_off, 0); ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * blkdev_trig_update_btb() - Update a BTB's activity counters and timestamps. ++ * @btb: The BTB ++ * @now: Timestamp (in jiffies) ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_update_btb(struct blkdev_trig_bdev *btb, ++ unsigned long now) ++{ ++ unsigned long new_ios; ++ enum stat_group i; ++ ++ for (i = STAT_READ; i <= STAT_FLUSH; ++i) { ++ ++ new_ios = part_stat_read(btb->bdev, ios[i]); ++ ++ if (new_ios != btb->ios[i]) { ++ btb->ios[i] = new_ios; ++ btb->last_activity[i] = now; ++ } ++ } ++ ++ btb->last_checked = now; ++} ++ ++/** ++ * blkdev_trig_check() - Check linked devices for activity and blink LEDs. ++ * @work: Delayed work (&blkdev_trig_work) ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_check(struct work_struct *work) ++{ ++ struct blkdev_trig_led *btl; ++ struct blkdev_trig_bdev *btb; ++ unsigned long index, delay, now, led_check, led_delay; ++ bool blinked; ++ ++ if (!mutex_trylock(&blkdev_trig_mutex)) { ++ delay = msecs_to_jiffies(BLKDEV_TRIG_CHECK_RETRY); ++ goto exit_reschedule; ++ } ++ ++ now = jiffies; ++ delay = ULONG_MAX; ++ ++ hlist_for_each_entry (btl, &blkdev_trig_all_btls, all_btls_node) { ++ ++ led_check = btl->last_checked + btl->check_jiffies; ++ ++ if (time_before_eq(led_check, now)) { ++ ++ blinked = false; ++ ++ xa_for_each (&btl->linked_btbs, index, btb) { ++ ++ if (btb->last_checked != now) ++ blkdev_trig_update_btb(btb, now); ++ if (!blinked) ++ blinked = blkdev_trig_blink(btl, btb); ++ } ++ ++ btl->last_checked = now; ++ led_delay = btl->check_jiffies; ++ ++ } else { ++ led_delay = led_check - now; ++ } ++ ++ if (led_delay < delay) ++ delay = led_delay; ++ } ++ ++ mutex_unlock(&blkdev_trig_mutex); ++ ++exit_reschedule: ++ WARN_ON_ONCE(delay == ULONG_MAX); ++ WARN_ON_ONCE(!schedule_delayed_work(&blkdev_trig_work, delay)); ++} ++ ++/** ++ * blkdev_trig_sched_led() - Set the schedule of the delayed work when a new ++ * LED is added to the schedule. ++ * @btl: The BTL that represents the LED ++ * ++ * Called when the number of block devices to which an LED is linked becomes ++ * non-zero. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_sched_led(const struct blkdev_trig_led *btl) ++{ ++ unsigned long delay = READ_ONCE(btl->check_jiffies); ++ unsigned long check_by = jiffies + delay; ++ ++ /* ++ * If no other LED-to-block device links exist, simply schedule the ++ * delayed work according to this LED's check_interval attribute ++ * (check_jiffies). ++ */ ++ if (blkdev_trig_link_count == 0) { ++ WARN_ON(!schedule_delayed_work(&blkdev_trig_work, delay)); ++ blkdev_trig_next_check = check_by; ++ return; ++ } ++ ++ /* ++ * If the next check is already scheduled to occur soon enough to ++ * accomodate this LED's check_interval, the schedule doesn't need ++ * to be changed. ++ */ ++ if (time_after_eq(check_by, blkdev_trig_next_check)) ++ return; ++ ++ /* ++ * Modify the schedule, so that the delayed work runs soon enough for ++ * this LED. ++ */ ++ WARN_ON(!mod_delayed_work(system_wq, &blkdev_trig_work, delay)); ++ blkdev_trig_next_check = check_by; ++} ++ ++ ++/* ++ * ++ * Linking and unlinking LEDs and block devices ++ * ++ */ ++ ++/** ++ * blkdev_trig_link() - Link a block device to an LED. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Return: &0 on success, negative &errno on error. ++ */ ++static int blkdev_trig_link(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ bool led_first_link; ++ int err; ++ ++ led_first_link = xa_empty(&btl->linked_btbs); ++ ++ err = xa_insert(&btb->linked_btls, btl->index, btl, GFP_KERNEL); ++ if (err) ++ return err; ++ ++ err = xa_insert(&btl->linked_btbs, btb->index, btb, GFP_KERNEL); ++ if (err) ++ goto error_erase_btl; ++ ++ /* Create /sys/class/block//linked_leds/ symlink */ ++ err = sysfs_add_link_to_group(bdev_kobj(btb->bdev), ++ blkdev_trig_linked_leds.name, ++ &btl->led->dev->kobj, btl->led->name); ++ if (err) ++ goto error_erase_btb; ++ ++ /* Create /sys/class/leds//linked_devices/ symlink */ ++ err = sysfs_add_link_to_group(&btl->led->dev->kobj, ++ blkdev_trig_linked_devs.name, ++ bdev_kobj(btb->bdev), ++ dev_name(&btb->bdev->bd_device)); ++ if (err) ++ goto error_remove_symlink; ++ ++ /* ++ * If this is the first block device linked to this LED, the delayed ++ * work schedule may need to be changed. ++ */ ++ if (led_first_link) ++ blkdev_trig_sched_led(btl); ++ ++ ++blkdev_trig_link_count; ++ ++ return 0; ++ ++error_remove_symlink: ++ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), ++ blkdev_trig_linked_leds.name, ++ btl->led->name); ++error_erase_btb: ++ xa_erase(&btl->linked_btbs, btb->index); ++error_erase_btl: ++ xa_erase(&btb->linked_btls, btl->index); ++ return err; ++} ++ ++/** ++ * blkdev_trig_put_btb() - Remove and free a BTB, if it is no longer needed. ++ * @btb: The BTB ++ * ++ * Does nothing if the BTB (block device) is still linked to at least one LED. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_put_btb(struct blkdev_trig_bdev *btb) ++{ ++ struct block_device *bdev = btb->bdev; ++ int err; ++ ++ if (xa_empty(&btb->linked_btls)) { ++ ++ sysfs_remove_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); ++ err = devres_destroy(&bdev->bd_device, blkdev_trig_btb_release, ++ NULL, NULL); ++ WARN_ON(err); ++ } ++} ++ ++/** ++ * _blkdev_trig_unlink_always() - Perform the unconditionally required steps of ++ * unlinking a block device from an LED. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * When a block device is unlinked from an LED, certain steps must be performed ++ * only if the block device is **not** being released. This function performs ++ * those steps that are **always** required, whether or not the block device is ++ * being released. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void _blkdev_trig_unlink_always(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ --blkdev_trig_link_count; ++ ++ if (blkdev_trig_link_count == 0) ++ WARN_ON(!cancel_delayed_work_sync(&blkdev_trig_work)); ++ ++ xa_erase(&btb->linked_btls, btl->index); ++ xa_erase(&btl->linked_btbs, btb->index); ++ ++ /* Remove /sys/class/leds//linked_devices/ symlink */ ++ sysfs_remove_link_from_group(&btl->led->dev->kobj, ++ blkdev_trig_linked_devs.name, ++ dev_name(&btb->bdev->bd_device)); ++} ++ ++/** ++ * blkdev_trig_unlink_norelease() - Unlink an LED from a block device that is ++ * **not** being released. ++ * @btl: The BTL that represents the LED. ++ * @btb: The BTB that represents the block device. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_unlink_norelease(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ _blkdev_trig_unlink_always(btl, btb); ++ ++ /* Remove /sys/class/block//linked_leds/ symlink */ ++ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), ++ blkdev_trig_linked_leds.name, ++ btl->led->name); ++ ++ blkdev_trig_put_btb(btb); ++} ++ ++/** ++ * blkdev_trig_unlink_release() - Unlink an LED from a block device that is ++ * being released. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_unlink_release(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ _blkdev_trig_unlink_always(btl, btb); ++ ++ /* ++ * If the BTB is being released, the driver core has already removed the ++ * device's attribute groups, and the BTB will be freed automatically, ++ * so there's nothing else to do. ++ */ ++} ++ ++ ++/* ++ * ++ * BTB creation ++ * ++ */ ++ ++/** ++ * blkdev_trig_btb_release() - BTB device resource release function. ++ * @dev: The block device ++ * @res: The BTB ++ * ++ * Called by the driver core when a block device with a BTB is removed. ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_btb_release(struct device *dev, void *res) ++{ ++ struct blkdev_trig_bdev *btb = res; ++ struct blkdev_trig_led *btl; ++ unsigned long index; ++ ++ mutex_lock(&blkdev_trig_mutex); ++ ++ xa_for_each (&btb->linked_btls, index, btl) ++ blkdev_trig_unlink_release(btl, btb); ++ ++ mutex_unlock(&blkdev_trig_mutex); ++} ++ ++/** ++ * blkdev_trig_get_bdev() - Get a block device by path. ++ * @path: The value written to an LED's &link_dev_by_path or ++ * &unlink_dev_by_path attribute, which should be the path to a ++ * special file that represents a block device ++ * @len: The number of characters in &path (not including its ++ * terminating null) ++ * ++ * The caller must call blkdev_put() when finished with the device. ++ * ++ * Context: Process context. ++ * Return: The block device, or an error pointer. ++ */ ++static struct block_device *blkdev_trig_get_bdev(const char *path, size_t len) ++{ ++ struct block_device *bdev; ++ char *buf; ++ ++ buf = kmemdup(path, len + 1, GFP_KERNEL); /* +1 to include null */ ++ if (buf == NULL) ++ return ERR_PTR(-ENOMEM); ++ ++ bdev = blkdev_get_by_path(strim(buf), BLKDEV_TRIG_FMODE, THIS_MODULE); ++ kfree(buf); ++ return bdev; ++} ++ ++/** ++ * blkdev_trig_get_btb() - Find or create the BTB for a block device. ++ * @path: The value written to an LED's &link_dev_by_path attribute, ++ * which should be the path to a special file that represents a ++ * block device ++ * @len: The number of characters in &path ++ * ++ * If a new BTB is created, because the block device was not previously linked ++ * to any LEDs, the block device's &linked_leds &sysfs directory is created. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Return: Pointer to the BTB, error pointer on error. ++ */ ++static struct blkdev_trig_bdev *blkdev_trig_get_btb(const char *path, ++ size_t len) ++{ ++ struct block_device *bdev; ++ struct blkdev_trig_bdev *btb; ++ int err; ++ ++ bdev = blkdev_trig_get_bdev(path, len); ++ if (IS_ERR(bdev)) ++ return ERR_CAST(bdev); ++ ++ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, ++ NULL, NULL); ++ if (btb != NULL) { ++ err = 0; ++ goto exit_put_bdev; ++ } ++ ++ if (blkdev_trig_next_index == ULONG_MAX) { ++ err = -EOVERFLOW; ++ goto exit_put_bdev; ++ } ++ ++ btb = devres_alloc(blkdev_trig_btb_release, sizeof(*btb), GFP_KERNEL); ++ if (btb == NULL) { ++ err = -ENOMEM; ++ goto exit_put_bdev; ++ } ++ ++ err = sysfs_create_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); ++ if (err) ++ goto exit_free_btb; ++ ++ btb->index = blkdev_trig_next_index++; ++ btb->bdev = bdev; ++ xa_init(&btb->linked_btls); ++ ++ /* Populate BTB activity counters */ ++ blkdev_trig_update_btb(btb, jiffies); ++ ++ devres_add(&bdev->bd_device, btb); ++ ++exit_free_btb: ++ if (err) ++ devres_free(btb); ++exit_put_bdev: ++ blkdev_put(bdev, BLKDEV_TRIG_FMODE); ++ return err ? ERR_PTR(err) : btb; ++} ++ ++ ++/* ++ * ++ * Activating and deactivating the trigger on an LED ++ * ++ */ ++ ++/** ++ * blkdev_trig_activate() - Called by the LEDs subsystem when an LED is ++ * associated with the trigger. ++ * @led: The LED ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &0 on success, negative &errno on error. ++ */ ++static int blkdev_trig_activate(struct led_classdev *led) ++{ ++ struct blkdev_trig_led *btl; ++ int err; ++ ++ btl = kzalloc(sizeof(*btl), GFP_KERNEL); ++ if (btl == NULL) ++ return -ENOMEM; ++ ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ goto exit_free; ++ ++ if (blkdev_trig_next_index == ULONG_MAX) { ++ err = -EOVERFLOW; ++ goto exit_unlock; ++ } ++ ++ btl->index = blkdev_trig_next_index++; ++ btl->last_checked = jiffies; ++ btl->mode = -1; /* set all bits */ ++ btl->led = led; ++ btl->blink_msec = BLKDEV_TRIG_BLINK_DEF; ++ btl->check_jiffies = msecs_to_jiffies(BLKDEV_TRIG_CHECK_DEF); ++ xa_init(&btl->linked_btbs); ++ ++ hlist_add_head(&btl->all_btls_node, &blkdev_trig_all_btls); ++ led_set_trigger_data(led, btl); ++ ++exit_unlock: ++ mutex_unlock(&blkdev_trig_mutex); ++exit_free: ++ if (err) ++ kfree(btl); ++ return err; ++} ++ ++/** ++ * blkdev_trig_deactivate() - Called by the the LEDs subsystem when an LED is ++ * disassociated from the trigger. ++ * @led: The LED ++ * ++ * The LEDs subsystem also calls this function when an LED associated with the ++ * trigger is removed or when the trigger is unregistered (if the module is ++ * unloaded). ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_deactivate(struct led_classdev *led) ++{ ++ struct blkdev_trig_led *btl = led_get_trigger_data(led); ++ struct blkdev_trig_bdev *btb; ++ unsigned long index; ++ ++ mutex_lock(&blkdev_trig_mutex); ++ ++ xa_for_each (&btl->linked_btbs, index, btb) ++ blkdev_trig_unlink_norelease(btl, btb); ++ ++ hlist_del(&btl->all_btls_node); ++ kfree(btl); ++ ++ mutex_unlock(&blkdev_trig_mutex); ++} ++ ++ ++/* ++ * ++ * Link-related attribute store functions ++ * ++ */ ++ ++/** ++ * link_dev_by_path_store() - &link_dev_by_path device attribute store function. ++ * @dev: The LED device ++ * @attr: The &link_dev_by_path attribute (&dev_attr_link_dev_by_path) ++ * @buf: The value written to the attribute, which should be the path to ++ * a special file that represents a block device to be linked to ++ * the LED (e.g. ``/dev/sda``) ++ * @count: The number of characters in &buf ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t link_dev_by_path_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ struct blkdev_trig_bdev *btb; ++ int err; ++ ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ return err; ++ ++ btb = blkdev_trig_get_btb(buf, count); ++ if (IS_ERR(btb)) { ++ err = PTR_ERR(btb); ++ goto exit_unlock; ++ } ++ ++ if (xa_load(&btb->linked_btls, btl->index) != NULL) { ++ err = -EEXIST; ++ goto exit_put_btb; ++ } ++ ++ err = blkdev_trig_link(btl, btb); ++ ++exit_put_btb: ++ if (err) ++ blkdev_trig_put_btb(btb); ++exit_unlock: ++ mutex_unlock(&blkdev_trig_mutex); ++ return err ? : count; ++} ++ ++/** ++ * unlink_dev_by_path_store() - &unlink_dev_by_path device attribute store ++ * function. ++ * @dev: The LED device ++ * @attr: The &unlink_dev_by_path attribute (&dev_attr_unlink_dev_by_path) ++ * @buf: The value written to the attribute, which should be the path to ++ * a special file that represents a block device to be unlinked ++ * from the LED (e.g. ``/dev/sda``) ++ * @count: The number of characters in &buf ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t unlink_dev_by_path_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ struct block_device *bdev; ++ struct blkdev_trig_bdev *btb; ++ int err; ++ ++ bdev = blkdev_trig_get_bdev(buf, count); ++ if (IS_ERR(bdev)) ++ return PTR_ERR(bdev); ++ ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ goto exit_put_bdev; ++ ++ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, ++ NULL, NULL); ++ if (btb == NULL) { ++ err = -EUNATCH; /* bdev isn't linked to any LED */ ++ goto exit_unlock; ++ } ++ ++ if (xa_load(&btb->linked_btls, btl->index) == NULL) { ++ err = -EUNATCH; /* bdev isn't linked to this LED */ ++ goto exit_unlock; ++ } ++ ++ blkdev_trig_unlink_norelease(btl, btb); ++ ++exit_unlock: ++ mutex_unlock(&blkdev_trig_mutex); ++exit_put_bdev: ++ blkdev_put(bdev, BLKDEV_TRIG_FMODE); ++ return err ? : count; ++} ++ ++/** ++ * unlink_dev_by_name_store() - &unlink_dev_by_name device attribute store ++ * function. ++ * @dev: The LED device ++ * @attr: The &unlink_dev_by_name attribute (&dev_attr_unlink_dev_by_name) ++ * @buf: The value written to the attribute, which should be the kernel ++ * name of a block device to be unlinked from the LED (e.g. ++ * ``sda``) ++ * @count: The number of characters in &buf ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t unlink_dev_by_name_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ struct blkdev_trig_bdev *btb; ++ unsigned long index; ++ int err; ++ ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ return err; ++ ++ err = -EUNATCH; ++ ++ xa_for_each (&btl->linked_btbs, index, btb) { ++ ++ if (sysfs_streq(dev_name(&btb->bdev->bd_device), buf)) { ++ blkdev_trig_unlink_norelease(btl, btb); ++ err = 0; ++ break; ++ } ++ } ++ ++ mutex_unlock(&blkdev_trig_mutex); ++ return err ? : count; ++} ++ ++ ++/* ++ * ++ * Atomic attribute show & store functions ++ * ++ */ ++ ++/** ++ * blink_time_show() - &blink_time device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_time attribute (&dev_attr_blink_time) ++ * @buf: Output buffer ++ * ++ * Writes the value of &blkdev_trig_led.blink_msec to &buf. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_time_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ ++ return sprintf(buf, "%u\n", READ_ONCE(btl->blink_msec)); ++} ++ ++/** ++ * blink_time_store() - &blink_time device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_time attribute (&dev_attr_blink_time) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets &blkdev_trig_led.blink_msec to the value in &buf. ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_time_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ unsigned int value; ++ int err; ++ ++ err = kstrtouint(buf, 0, &value); ++ if (err) ++ return err; ++ ++ if (value < BLKDEV_TRIG_BLINK_MIN || value > BLKDEV_TRIG_BLINK_MAX) ++ return -ERANGE; ++ ++ WRITE_ONCE(btl->blink_msec, value); ++ return count; ++} ++ ++/** ++ * check_interval_show() - &check_interval device attribute show function. ++ * @dev: The LED device ++ * @attr: The &check_interval attribute (&dev_attr_check_interval) ++ * @buf: Output buffer ++ * ++ * Writes the value of &blkdev_trig_led.check_jiffies (converted to ++ * milliseconds) to &buf. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t check_interval_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ ++ return sprintf(buf, "%u\n", ++ jiffies_to_msecs(READ_ONCE(btl->check_jiffies))); ++} ++ ++/** ++ * check_interval_store() - &check_interval device attribute store function ++ * @dev: The LED device ++ * @attr: The &check_interval attribute (&dev_attr_check_interval) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets &blkdev_trig_led.check_jiffies to the value in &buf (after converting ++ * from milliseconds). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t check_interval_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *led = led_trigger_get_drvdata(dev); ++ unsigned int value; ++ int err; ++ ++ err = kstrtouint(buf, 0, &value); ++ if (err) ++ return err; ++ ++ if (value < BLKDEV_TRIG_CHECK_MIN || value > BLKDEV_TRIG_CHECK_MAX) ++ return -ERANGE; ++ ++ WRITE_ONCE(led->check_jiffies, msecs_to_jiffies(value)); ++ ++ return count; ++} ++ ++/** ++ * blkdev_trig_mode_show() - Helper for boolean attribute show functions. ++ * @led: The LED ++ * @buf: Output buffer ++ * @bit: Which bit to show ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static int blkdev_trig_mode_show(const struct blkdev_trig_led *led, char *buf, ++ enum stat_group bit) ++{ ++ return sprintf(buf, READ_ONCE(led->mode) & (1 << bit) ? "Y\n" : "N\n"); ++} ++ ++/** ++ * blkdev_trig_mode_store() - Helper for boolean attribute store functions. ++ * @led: The LED ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * @bit: Which bit to set ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static int blkdev_trig_mode_store(struct blkdev_trig_led *led, ++ const char *buf, size_t count, ++ enum stat_group bit) ++{ ++ bool set; ++ int err; ++ ++ err = kstrtobool(buf, &set); ++ if (err) ++ return err; ++ ++ if (set) ++ set_bit(bit, &led->mode); ++ else ++ clear_bit(bit, &led->mode); ++ ++ return count; ++} ++ ++/** ++ * blink_on_read_show() - &blink_on_read device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) ++ * @buf: Output buffer ++ * ++ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_READ bit in ++ * &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_on_read_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_READ); ++} ++ ++/** ++ * blink_on_read_store() - &blink_on_read device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets the &STAT_READ bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_on_read_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_READ); ++} ++ ++/** ++ * blink_on_write_show() - &blink_on_write device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) ++ * @buf: Output buffer ++ * ++ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_WRITE bit in ++ * in &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_on_write_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_WRITE); ++} ++ ++/** ++ * blink_on_write_store() - &blink_on_write device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets the &STAT_WRITE bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_on_write_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_WRITE); ++} ++ ++/** ++ * blink_on_flush_show() - &blink_on_flush device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) ++ * @buf: Output buffer ++ * ++ * Writes ``Y`` or ``N`` to &buf, depending whether the &STAT_FLUSH bit in ++ * &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_on_flush_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_FLUSH); ++} ++ ++/** ++ * blink_on_flush_store() - &blink_on_flush device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets the &STAT_FLUSH bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_on_flush_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_FLUSH); ++} ++ ++/** ++ * blink_on_discard_show() - &blink_on_discard device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) ++ * @buf: Output buffer ++ * ++ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_DISCARD bit in ++ * &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_on_discard_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_DISCARD); ++} ++ ++/** ++ * blink_on_discard_store() - &blink_on_discard device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets the &STAT_DISCARD bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_on_discard_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_DISCARD); ++} ++ ++/* Device attributes */ ++static DEVICE_ATTR_WO(link_dev_by_path); ++static DEVICE_ATTR_WO(unlink_dev_by_path); ++static DEVICE_ATTR_WO(unlink_dev_by_name); ++static DEVICE_ATTR_RW(blink_time); ++static DEVICE_ATTR_RW(check_interval); ++static DEVICE_ATTR_RW(blink_on_read); ++static DEVICE_ATTR_RW(blink_on_write); ++static DEVICE_ATTR_RW(blink_on_flush); ++static DEVICE_ATTR_RW(blink_on_discard); ++ ++/* Device attributes in LED directory (/sys/class/leds//...) */ ++static struct attribute *blkdev_trig_attrs[] = { ++ &dev_attr_link_dev_by_path.attr, ++ &dev_attr_unlink_dev_by_path.attr, ++ &dev_attr_unlink_dev_by_name.attr, ++ &dev_attr_blink_time.attr, ++ &dev_attr_check_interval.attr, ++ &dev_attr_blink_on_read.attr, ++ &dev_attr_blink_on_write.attr, ++ &dev_attr_blink_on_flush.attr, ++ &dev_attr_blink_on_discard.attr, ++ NULL ++}; ++ ++/* Unnamed attribute group == no subdirectory */ ++static const struct attribute_group blkdev_trig_attr_group = { ++ .attrs = blkdev_trig_attrs, ++}; ++ ++/* Attribute groups for the trigger */ ++static const struct attribute_group *blkdev_trig_attr_groups[] = { ++ &blkdev_trig_attr_group, /* /sys/class/leds//... */ ++ &blkdev_trig_linked_devs, /* /sys/class/leds//linked_devices/ */ ++ NULL ++}; ++ ++/* Trigger registration data */ ++static struct led_trigger blkdev_trig_trigger = { ++ .name = "blkdev", ++ .activate = blkdev_trig_activate, ++ .deactivate = blkdev_trig_deactivate, ++ .groups = blkdev_trig_attr_groups, ++}; ++ ++/** ++ * blkdev_trig_init() - Block device LED trigger initialization. ++ * ++ * Registers the ``blkdev`` LED trigger. ++ * ++ * Return: &0 on success, negative &errno on failure. ++ */ ++static int __init blkdev_trig_init(void) ++{ ++ return led_trigger_register(&blkdev_trig_trigger); ++} ++module_init(blkdev_trig_init); ++ ++/** ++ * blkdev_trig_exit() - Block device LED trigger module exit. ++ * ++ * Unregisters the ``blkdev`` LED trigger. ++ */ ++static void __exit blkdev_trig_exit(void) ++{ ++ led_trigger_unregister(&blkdev_trig_trigger); ++} ++module_exit(blkdev_trig_exit); ++ ++MODULE_DESCRIPTION("Block device LED trigger"); ++MODULE_AUTHOR("Ian Pilcher "); ++MODULE_LICENSE("GPL v2"); +diff --git a/include/linux/kexec.h b/include/linux/kexec.h +index 13e6c4b58f07..53b147c0bdda 100644 +--- a/include/linux/kexec.h ++++ b/include/linux/kexec.h +@@ -339,9 +339,10 @@ struct kimage { + unsigned long control_page; + + /* Flags to indicate special processing */ +- unsigned int type : 1; ++ unsigned int type : 2; + #define KEXEC_TYPE_DEFAULT 0 + #define KEXEC_TYPE_CRASH 1 ++#define KEXEC_TYPE_RESERVED_MEM 2 + unsigned int preserve_context : 1; + /* If set, we are using file mode kexec syscall */ + unsigned int file_mode:1; +@@ -414,14 +415,14 @@ extern int kexec_load_disabled; + + /* List of defined/legal kexec flags */ + #ifndef CONFIG_KEXEC_JUMP +-#define KEXEC_FLAGS KEXEC_ON_CRASH ++#define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_RESERVED_MEM) + #else +-#define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_PRESERVE_CONTEXT) ++#define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_PRESERVE_CONTEXT | KEXEC_RESERVED_MEM) + #endif + + /* List of defined/legal kexec file flags */ + #define KEXEC_FILE_FLAGS (KEXEC_FILE_UNLOAD | KEXEC_FILE_ON_CRASH | \ +- KEXEC_FILE_NO_INITRAMFS) ++ KEXEC_FILE_NO_INITRAMFS | KEXEC_FILE_RESERVED_MEM) + + /* flag to track if kexec reboot is in progress */ + extern bool kexec_in_progress; +diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h +index 83c7248053a1..d2b8741eabf7 100644 +--- a/include/linux/pageblock-flags.h ++++ b/include/linux/pageblock-flags.h +@@ -48,7 +48,7 @@ extern unsigned int pageblock_order; + #else /* CONFIG_HUGETLB_PAGE */ + + /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ +-#define pageblock_order (MAX_ORDER-1) ++#define pageblock_order PAGE_ALLOC_COSTLY_ORDER + + #endif /* CONFIG_HUGETLB_PAGE */ + +diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h +index 99d26879b02a..a92799fc5e74 100644 +--- a/include/net/bluetooth/rfcomm.h ++++ b/include/net/bluetooth/rfcomm.h +@@ -171,6 +171,7 @@ struct rfcomm_dlc { + struct rfcomm_session *session; + struct sk_buff_head tx_queue; + struct timer_list timer; ++ struct work_struct state_change_work; + + struct mutex lock; + unsigned long state; +@@ -186,6 +187,7 @@ struct rfcomm_dlc { + u8 sec_level; + u8 role_switch; + u32 defer_setup; ++ int err; + + uint mtu; + uint cfc; +@@ -310,6 +312,7 @@ struct rfcomm_pinfo { + u8 role_switch; + }; + ++void __rfcomm_sk_state_change(struct work_struct *work); + int rfcomm_init_sockets(void); + void rfcomm_cleanup_sockets(void); + +diff --git a/include/uapi/linux/kexec.h b/include/uapi/linux/kexec.h +index 981016e05cfa..c29011eb7fc2 100644 +--- a/include/uapi/linux/kexec.h ++++ b/include/uapi/linux/kexec.h +@@ -12,6 +12,7 @@ + /* kexec flags for different usage scenarios */ + #define KEXEC_ON_CRASH 0x00000001 + #define KEXEC_PRESERVE_CONTEXT 0x00000002 ++#define KEXEC_RESERVED_MEM 0x00000004 + #define KEXEC_ARCH_MASK 0xffff0000 + + /* +@@ -24,6 +25,7 @@ + #define KEXEC_FILE_UNLOAD 0x00000001 + #define KEXEC_FILE_ON_CRASH 0x00000002 + #define KEXEC_FILE_NO_INITRAMFS 0x00000004 ++#define KEXEC_FILE_RESERVED_MEM 0x00000008 + + /* These values match the ELF architecture values. + * Unless there is a good reason that should continue to be the case. +diff --git a/kernel/kexec.c b/kernel/kexec.c +index b5e40f069768..0d9ea52c81c1 100644 +--- a/kernel/kexec.c ++++ b/kernel/kexec.c +@@ -27,8 +27,14 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, + int ret; + struct kimage *image; + bool kexec_on_panic = flags & KEXEC_ON_CRASH; ++ bool kexec_on_reserved = flags & KEXEC_RESERVED_MEM; + +- if (kexec_on_panic) { ++ if (kexec_on_panic && kexec_on_reserved) { ++ pr_err("both kexec_on_panic and kexec_on_reserved is true, they can not coexist"); ++ return -EINVAL; ++ } ++ ++ if (kexec_on_panic || kexec_on_reserved) { + /* Verify we have a valid entry point */ + if ((entry < phys_to_boot_phys(crashk_res.start)) || + (entry > phys_to_boot_phys(crashk_res.end))) +@@ -50,6 +56,12 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, + image->type = KEXEC_TYPE_CRASH; + } + ++ if (kexec_on_reserved) { ++ /* Enable special reserved kernel control page alloc policy. */ ++ image->control_page = crashk_res.start; ++ image->type = KEXEC_TYPE_RESERVED_MEM; ++ } ++ + ret = sanity_check_segment_list(image); + if (ret) + goto out_free_image; +@@ -110,6 +122,11 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, + dest_image = &kexec_image; + } + ++ if (flags & KEXEC_RESERVED_MEM) { ++ if (kexec_crash_image) ++ arch_kexec_unprotect_crashkres(); ++ } ++ + if (nr_segments == 0) { + /* Uninstall image */ + kimage_free(xchg(dest_image, NULL)); +diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c +index acd029b307e4..68b3b7e7003a 100644 +--- a/kernel/kexec_core.c ++++ b/kernel/kexec_core.c +@@ -230,13 +230,13 @@ int sanity_check_segment_list(struct kimage *image) + * Verify we have good destination addresses. Normally + * the caller is responsible for making certain we don't + * attempt to load the new image into invalid or reserved +- * areas of RAM. But crash kernels are preloaded into a ++ * areas of RAM. But crash kernels (or we specify to load ++ * the new image into reserved areas) are preloaded into a + * reserved area of ram. We must ensure the addresses + * are in the reserved area otherwise preloading the + * kernel could corrupt things. + */ +- +- if (image->type == KEXEC_TYPE_CRASH) { ++ if (image->type == KEXEC_TYPE_CRASH || image->type == KEXEC_TYPE_RESERVED_MEM) { + for (i = 0; i < nr_segments; i++) { + unsigned long mstart, mend; + +@@ -414,7 +414,7 @@ static struct page *kimage_alloc_normal_control_pages(struct kimage *image, + return pages; + } + +-static struct page *kimage_alloc_crash_control_pages(struct kimage *image, ++static struct page *kimage_alloc_reserverd_control_pages(struct kimage *image, + unsigned int order) + { + /* Control pages are special, they are the intermediaries +@@ -491,7 +491,8 @@ struct page *kimage_alloc_control_pages(struct kimage *image, + pages = kimage_alloc_normal_control_pages(image, order); + break; + case KEXEC_TYPE_CRASH: +- pages = kimage_alloc_crash_control_pages(image, order); ++ case KEXEC_TYPE_RESERVED_MEM: ++ pages = kimage_alloc_reserverd_control_pages(image, order); + break; + } + +@@ -841,7 +842,7 @@ static int kimage_load_normal_segment(struct kimage *image, + return result; + } + +-static int kimage_load_crash_segment(struct kimage *image, ++static int kimage_load_reserved_segment(struct kimage *image, + struct kexec_segment *segment) + { + /* For crash dumps kernels we simply copy the data from +@@ -919,7 +920,8 @@ int kimage_load_segment(struct kimage *image, + result = kimage_load_normal_segment(image, segment); + break; + case KEXEC_TYPE_CRASH: +- result = kimage_load_crash_segment(image, segment); ++ case KEXEC_TYPE_RESERVED_MEM: ++ result = kimage_load_reserved_segment(image, segment); + break; + } + +diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c +index 1d546dc97c50..63c3d71753b7 100644 +--- a/kernel/kexec_file.c ++++ b/kernel/kexec_file.c +@@ -273,7 +273,7 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, + int ret; + struct kimage *image; + bool kexec_on_panic = flags & KEXEC_FILE_ON_CRASH; +- ++ bool kexec_on_reserved = flags & KEXEC_FILE_RESERVED_MEM; + image = do_kimage_alloc_init(); + if (!image) + return -ENOMEM; +@@ -286,6 +286,12 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, + image->type = KEXEC_TYPE_CRASH; + } + ++ if (kexec_on_reserved) { ++ /* Enable special crash kernel control page alloc policy. */ ++ image->control_page = crashk_res.start; ++ image->type = KEXEC_TYPE_RESERVED_MEM; ++ } ++ + ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd, + cmdline_ptr, cmdline_len, flags); + if (ret) +@@ -342,6 +348,11 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, + if (!mutex_trylock(&kexec_mutex)) + return -EBUSY; + ++ if ((flags & KEXEC_FILE_ON_CRASH) && (flags & KEXEC_FILE_RESERVED_MEM)) { ++ pr_err("both kexec_on_panic and kexec_on_reserved is true, they can not coexist"); ++ return -EINVAL; ++ } ++ + dest_image = &kexec_image; + if (flags & KEXEC_FILE_ON_CRASH) { + dest_image = &kexec_crash_image; +@@ -349,6 +360,11 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, + arch_kexec_unprotect_crashkres(); + } + ++ if (flags & KEXEC_FILE_RESERVED_MEM) { ++ if (kexec_crash_image) ++ arch_kexec_unprotect_crashkres(); ++ } ++ + if (flags & KEXEC_FILE_UNLOAD) + goto exchange; + +@@ -584,7 +600,7 @@ static int kexec_walk_memblock(struct kexec_buf *kbuf, + static int kexec_walk_resources(struct kexec_buf *kbuf, + int (*func)(struct resource *, void *)) + { +- if (kbuf->image->type == KEXEC_TYPE_CRASH) ++ if (kbuf->image->type == KEXEC_TYPE_CRASH || kbuf->image->type == KEXEC_TYPE_RESERVED_MEM) + return walk_iomem_res_desc(crashk_res.desc, + IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, + crashk_res.start, crashk_res.end, +diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile +index c415a685d61b..e814061d6aa0 100644 +--- a/lib/vdso/Makefile ++++ b/lib/vdso/Makefile +@@ -17,6 +17,6 @@ $(error ARCH_REL_TYPE_ABS is not set) + endif + + quiet_cmd_vdso_check = VDSOCHK $@ +- cmd_vdso_check = if $(OBJDUMP) -R $@ | egrep -h "$(ARCH_REL_TYPE_ABS)"; \ ++ cmd_vdso_check = if $(OBJDUMP) -R $@ | grep -E -h "$(ARCH_REL_TYPE_ABS)"; \ + then (echo >&2 "$@: dynamic relocations are not supported"; \ + rm -f $@; /bin/false); fi +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 2a5c7f420098..2a98e4125f24 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2447,7 +2447,8 @@ static void __split_huge_page_tail(struct page *head, int tail, + page_tail); + page_tail->mapping = head->mapping; + page_tail->index = head->index + tail; +- page_tail->private = 0; ++ if (!PageSwapCache(page_tail)) ++ page_tail->private = 0; + + /* Page flags must be visible before we make the page non-compound. */ + smp_wmb(); +diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c +index 7324764384b6..c6494e85cd68 100644 +--- a/net/bluetooth/rfcomm/core.c ++++ b/net/bluetooth/rfcomm/core.c +@@ -289,6 +289,7 @@ static void rfcomm_dlc_clear_state(struct rfcomm_dlc *d) + d->flags = 0; + d->mscex = 0; + d->sec_level = BT_SECURITY_LOW; ++ d->err = 0; + d->mtu = RFCOMM_DEFAULT_MTU; + d->v24_sig = RFCOMM_V24_RTC | RFCOMM_V24_RTR | RFCOMM_V24_DV; + +@@ -306,6 +307,7 @@ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio) + timer_setup(&d->timer, rfcomm_dlc_timeout, 0); + + skb_queue_head_init(&d->tx_queue); ++ INIT_WORK(&d->state_change_work, __rfcomm_sk_state_change); + mutex_init(&d->lock); + refcount_set(&d->refcnt, 1); + +diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c +index 4bf4ea6cbb5e..4850dafbaa05 100644 +--- a/net/bluetooth/rfcomm/sock.c ++++ b/net/bluetooth/rfcomm/sock.c +@@ -61,19 +61,22 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) + rfcomm_dlc_throttle(d); + } + +-static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) ++void __rfcomm_sk_state_change(struct work_struct *work) + { ++ struct rfcomm_dlc *d = container_of(work, struct rfcomm_dlc, ++ state_change_work); + struct sock *sk = d->owner, *parent; + + if (!sk) + return; + +- BT_DBG("dlc %p state %ld err %d", d, d->state, err); +- + lock_sock(sk); ++ rfcomm_dlc_lock(d); + +- if (err) +- sk->sk_err = err; ++ BT_DBG("dlc %p state %ld err %d", d, d->state, d->err); ++ ++ if (d->err) ++ sk->sk_err = d->err; + + sk->sk_state = d->state; + +@@ -91,15 +94,22 @@ static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) + sk->sk_state_change(sk); + } + ++ rfcomm_dlc_unlock(d); + release_sock(sk); ++ sock_put(sk); ++} + +- if (parent && sock_flag(sk, SOCK_ZAPPED)) { +- /* We have to drop DLC lock here, otherwise +- * rfcomm_sock_destruct() will dead lock. */ +- rfcomm_dlc_unlock(d); +- rfcomm_sock_kill(sk); +- rfcomm_dlc_lock(d); +- } ++static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) ++{ ++ struct sock *sk = d->owner; ++ ++ if (!sk) ++ return; ++ ++ d->err = err; ++ sock_hold(sk); ++ if (!schedule_work(&d->state_change_work)) ++ sock_put(sk); + } + + /* ---- Socket functions ---- */ +diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib +index f62770a0a84f..2dda88a30554 100644 +--- a/scripts/Makefile.lib ++++ b/scripts/Makefile.lib +@@ -438,6 +438,11 @@ quiet_cmd_lz4 = LZ4 $@ + quiet_cmd_lz4_with_size = LZ4 $@ + cmd_lz4_with_size = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \ + $(size_append); } > $@ ++# none ++quiet_cmd_none = NONE $@ ++ cmd_none = (cat $(filter-out FORCE,$^) && \ ++ $(call size_append, $(filter-out FORCE,$^))) > $@ || \ ++ (rm -f $@ ; false) + + # U-Boot mkimage + # --------------------------------------------------------------------------- +diff --git a/tools/objtool/check.c b/tools/objtool/check.c +index e55fdf952a3a..445741f36d6d 100644 +--- a/tools/objtool/check.c ++++ b/tools/objtool/check.c +@@ -2104,9 +2104,6 @@ static int read_noendbr_hints(struct objtool_file *file) + return -1; + } + +- if (insn->type == INSN_ENDBR) +- WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); +- + insn->noendbr = 1; + } + +-- +2.38.0 + +From 852ccfcccb332e9ce5767a50c98711cfa1855dd1 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 26 Sep 2022 00:19:51 +0200 +Subject: [PATCH 12/17] kallsyms + +Signed-off-by: Peter Jung +--- + include/linux/kallsyms.h | 12 +- + include/linux/module.h | 4 +- + init/Kconfig | 13 ++ + kernel/Makefile | 1 + + kernel/kallsyms.c | 195 +++++++++++++++-- + kernel/kallsyms_internal.h | 1 + + kernel/kallsyms_selftest.c | 421 +++++++++++++++++++++++++++++++++++++ + kernel/livepatch/core.c | 31 ++- + kernel/module/kallsyms.c | 15 +- + kernel/trace/ftrace.c | 3 +- + scripts/kallsyms.c | 88 +++++--- + 11 files changed, 717 insertions(+), 67 deletions(-) + create mode 100644 kernel/kallsyms_selftest.c + +diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h +index ad39636e0c3f..c7219d74e290 100644 +--- a/include/linux/kallsyms.h ++++ b/include/linux/kallsyms.h +@@ -66,9 +66,11 @@ static inline void *dereference_symbol_descriptor(void *ptr) + } + + #ifdef CONFIG_KALLSYMS +-int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, +- unsigned long), ++unsigned long kallsyms_sym_address(int idx); ++int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), + void *data); ++int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), ++ const char *name, void *data); + + /* Lookup the address for a symbol. Returns 0 if not found. */ + unsigned long kallsyms_lookup_name(const char *name); +@@ -168,6 +170,12 @@ static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct + { + return -EOPNOTSUPP; + } ++ ++static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), ++ const char *name, void *data) ++{ ++ return -EOPNOTSUPP; ++} + #endif /*CONFIG_KALLSYMS*/ + + static inline void print_ip_sym(const char *loglvl, unsigned long ip) +diff --git a/include/linux/module.h b/include/linux/module.h +index 518296ea7f73..6e1a531d78e7 100644 +--- a/include/linux/module.h ++++ b/include/linux/module.h +@@ -879,8 +879,8 @@ static inline bool module_sig_ok(struct module *module) + } + #endif /* CONFIG_MODULE_SIG */ + +-int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, +- struct module *, unsigned long), ++int module_kallsyms_on_each_symbol(const char *modname, ++ int (*fn)(void *, const char *, unsigned long), + void *data); + + #endif /* _LINUX_MODULE_H */ +diff --git a/init/Kconfig b/init/Kconfig +index f5bd72b39352..274cabde40ab 100644 +--- a/init/Kconfig ++++ b/init/Kconfig +@@ -1755,6 +1755,19 @@ config KALLSYMS + symbolic stack backtraces. This increases the size of the kernel + somewhat, as all symbols have to be loaded into the kernel image. + ++config KALLSYMS_SELFTEST ++ bool "Test the basic functions and performance of kallsyms" ++ depends on KALLSYMS ++ default n ++ help ++ Test the basic functions and performance of some interfaces, such as ++ kallsyms_lookup_name. It also calculates the compression rate of the ++ kallsyms compression algorithm for the current symbol set. ++ ++ Start self-test automatically after system startup. Suggest executing ++ "dmesg | grep kallsyms_selftest" to collect test results. "finish" is ++ displayed in the last line, indicating that the test is complete. ++ + config KALLSYMS_ALL + bool "Include all symbols in kallsyms" + depends on DEBUG_KERNEL && KALLSYMS +diff --git a/kernel/Makefile b/kernel/Makefile +index 318789c728d3..122a5fed457b 100644 +--- a/kernel/Makefile ++++ b/kernel/Makefile +@@ -68,6 +68,7 @@ endif + obj-$(CONFIG_UID16) += uid16.o + obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o + obj-$(CONFIG_KALLSYMS) += kallsyms.o ++obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o + obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o + obj-$(CONFIG_CRASH_CORE) += crash_core.o + obj-$(CONFIG_KEXEC_CORE) += kexec_core.o +diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c +index 3e7e2c2ad2f7..1512db69aa0a 100644 +--- a/kernel/kallsyms.c ++++ b/kernel/kallsyms.c +@@ -87,6 +87,86 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, + return off; + } + ++static unsigned char *find_token(unsigned char *str, int len, ++ const unsigned char *token) ++{ ++ int i; ++ ++ for (i = 0; i < len - 1; i++) { ++ if (str[i] == token[0] && str[i+1] == token[1]) ++ return &str[i]; ++ } ++ return NULL; ++} ++ ++static int kallsyms_compress_symbol_name(const char *name, char *buf, size_t size) ++{ ++ int i, j, n, len; ++ unsigned char *p1, *p2; ++ const unsigned char *token; ++ ++ len = strscpy(buf, name, size); ++ if (WARN_ON_ONCE(len <= 0)) ++ return 0; ++ ++ /* ++ * For each entry in kallsyms_best_token_table[], the storage ++ * format is: ++ * 1. For tokens that cannot be used to compress characters, the value ++ * at [j] is 0, and the value at [j+1] is the number of consecutive ++ * tokens with this feature. ++ * 2. For each token: the larger the token value, the higher the ++ * frequency, and the lower the index. ++ * ++ * ------------------------------- ++ * | j | [j] [j+1] | token | ++ * -----|---------------|---------| ++ * | 0 | ?? ?? | 255 | ++ * | 2 | ?? ?? | 254 | ++ * | ... | ?? ?? | ... | ++ * | n-2 | ?? ?? | x | ++ * | n | 00 len | x-1 | ++ * | n+2 | ?? ?? | x-1-len | ++ * above '??' is non-zero ++ */ ++ for (i = 255, j = 0; i >= 0; i--, j += 2) { ++ if (!kallsyms_best_token_table[j]) { ++ i -= kallsyms_best_token_table[j + 1]; ++ if (i < 0) ++ break; ++ j += 2; ++ } ++ token = &kallsyms_best_token_table[j]; ++ ++ p1 = buf; ++ ++ /* find the token on the symbol */ ++ p2 = find_token(p1, len, token); ++ if (!p2) ++ continue; ++ ++ n = len; ++ ++ do { ++ *p2 = i; ++ p2++; ++ n -= (p2 - p1); ++ memmove(p2, p2 + 1, n); ++ p1 = p2; ++ len--; ++ ++ if (n < 2) ++ break; ++ ++ /* find the token on the symbol */ ++ p2 = find_token(p1, n, token); ++ ++ } while (p2); ++ } ++ ++ return len; ++} ++ + /* + * Get symbol type information. This is encoded as a single char at the + * beginning of the symbol name. +@@ -128,7 +208,7 @@ static unsigned int get_symbol_offset(unsigned long pos) + return name - kallsyms_names; + } + +-static unsigned long kallsyms_sym_address(int idx) ++unsigned long kallsyms_sym_address(int idx) + { + if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE)) + return kallsyms_addresses[idx]; +@@ -186,26 +266,95 @@ static bool cleanup_symbol_name(char *s) + return false; + } + ++static int kallsyms_lookup_clang_name(unsigned char *namebuf, int len, ++ const char *name, ++ unsigned long *addr) ++{ ++ unsigned long i; ++ unsigned int off; ++ ++ if (!IS_ENABLED(CONFIG_LTO_CLANG)) ++ return -ENOENT; ++ ++ for (i = 0, off = 0; i < kallsyms_num_syms; i++) { ++ off = kallsyms_expand_symbol(off, namebuf, len); ++ ++ if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) { ++ *addr = kallsyms_sym_address(i); ++ return 0; ++ } ++ } ++ ++ return -ENOENT; ++} ++ ++static int __kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, ++ unsigned int *index, ++ unsigned int *offset, ++ unsigned long *addr) ++{ ++ unsigned int i = *index; ++ unsigned int off = *offset; ++ unsigned int name_len; ++ const unsigned char *name; ++ ++ for (; len && i < kallsyms_num_syms; i++) { ++ /* ++ * For each entry in kallsyms_names[], the storage format is: ++ * ---------------------------- ++ * | len(1) | type(1) | name(x) | ++ * ---------------------------- ++ * ++ * Number of bytes in parentheses, and: len = 1 + x ++ */ ++ name_len = kallsyms_names[off] - 1; ++ name = &kallsyms_names[off + 2]; ++ off += name_len + 2; ++ ++ if (name_len != len) ++ continue; ++ ++ if (!memcmp(name, namebuf, len)) { ++ /* Prepare for the next iteration */ ++ *index = i + 1; ++ *offset = off; ++ ++ *addr = kallsyms_sym_address(i); ++ return 0; ++ } ++ } ++ ++ return -ENOENT; ++} ++ ++static int kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, ++ unsigned long *addr) ++{ ++ unsigned int i = 0, off = 0; ++ ++ return __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, addr); ++} ++ + /* Lookup the address for this symbol. Returns 0 if not found. */ + unsigned long kallsyms_lookup_name(const char *name) + { + char namebuf[KSYM_NAME_LEN]; +- unsigned long i; +- unsigned int off; ++ unsigned long addr; ++ int ret, len; + + /* Skip the search for empty string. */ + if (!*name) + return 0; + +- for (i = 0, off = 0; i < kallsyms_num_syms; i++) { +- off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); ++ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); ++ ret = kallsyms_lookup_compressed_name(namebuf, len, &addr); ++ if (!ret) ++ return addr; + +- if (strcmp(namebuf, name) == 0) +- return kallsyms_sym_address(i); ++ ret = kallsyms_lookup_clang_name(namebuf, len, name, &addr); ++ if (!ret) ++ return addr; + +- if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) +- return kallsyms_sym_address(i); +- } + return module_kallsyms_lookup_name(name); + } + +@@ -213,8 +362,7 @@ unsigned long kallsyms_lookup_name(const char *name) + * Iterate over all symbols in vmlinux. For symbols from modules use + * module_kallsyms_on_each_symbol instead. + */ +-int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, +- unsigned long), ++int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), + void *data) + { + char namebuf[KSYM_NAME_LEN]; +@@ -224,7 +372,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, + + for (i = 0, off = 0; i < kallsyms_num_syms; i++) { + off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); +- ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); ++ ret = fn(data, namebuf, kallsyms_sym_address(i)); + if (ret != 0) + return ret; + cond_resched(); +@@ -232,6 +380,27 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, + return 0; + } + ++int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), ++ const char *name, void *data) ++{ ++ int ret, len; ++ unsigned long addr; ++ unsigned int i = 0, off = 0; ++ char namebuf[KSYM_NAME_LEN]; ++ ++ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); ++ do { ++ ret = __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, &addr); ++ if (ret) ++ return 0; /* end of lookup */ ++ ++ ret = fn(data, addr); ++ cond_resched(); ++ } while (!ret); ++ ++ return ret; ++} ++ + static unsigned long get_symbol_pos(unsigned long addr, + unsigned long *symbolsize, + unsigned long *offset) +diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h +index 2d0c6f2f0243..d9672ede8cfc 100644 +--- a/kernel/kallsyms_internal.h ++++ b/kernel/kallsyms_internal.h +@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; + extern const u16 kallsyms_token_index[] __weak; + + extern const unsigned int kallsyms_markers[] __weak; ++extern const unsigned char kallsyms_best_token_table[] __weak; + + #endif // LINUX_KALLSYMS_INTERNAL_H_ +diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c +new file mode 100644 +index 000000000000..f7538a70d36c +--- /dev/null ++++ b/kernel/kallsyms_selftest.c +@@ -0,0 +1,421 @@ ++// SPDX-License-Identifier: GPL-2.0-or-later ++/* ++ * Test the function and performance of kallsyms ++ * ++ * Copyright (C) Huawei Technologies Co., Ltd., 2022 ++ * ++ * Authors: Zhen Lei Huawei ++ */ ++ ++#define pr_fmt(fmt) "kallsyms_selftest: " fmt ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "kallsyms_internal.h" ++ ++ ++#define MAX_NUM_OF_RECORDS 64 ++ ++struct test_stat { ++ int min; ++ int max; ++ int save_cnt; ++ int real_cnt; ++ u64 sum; ++ char *name; ++ unsigned long addr; ++ unsigned long addrs[MAX_NUM_OF_RECORDS]; ++}; ++ ++struct test_item { ++ char *name; ++ unsigned long addr; ++}; ++ ++#define ITEM_FUNC(s) \ ++ { \ ++ .name = #s, \ ++ .addr = (unsigned long)s, \ ++ } ++ ++#define ITEM_DATA(s) \ ++ { \ ++ .name = #s, \ ++ .addr = (unsigned long)&s, \ ++ } ++ ++static int test_var_bss_static; ++static int test_var_data_static = 1; ++int test_var_bss; ++int test_var_data = 1; ++ ++static int test_func_static(void) ++{ ++ test_var_bss_static++; ++ test_var_data_static++; ++ ++ return 0; ++} ++ ++int test_func(void) ++{ ++ return test_func_static(); ++} ++ ++__weak int test_func_weak(void) ++{ ++ test_var_bss++; ++ test_var_data++; ++ return 0; ++} ++ ++static struct test_item test_items[] = { ++ ITEM_FUNC(test_func_static), ++ ITEM_FUNC(test_func), ++ ITEM_FUNC(test_func_weak), ++ ITEM_FUNC(vmalloc), ++ ITEM_FUNC(vfree), ++#ifdef CONFIG_KALLSYMS_ALL ++ ITEM_DATA(test_var_bss_static), ++ ITEM_DATA(test_var_data_static), ++ ITEM_DATA(test_var_bss), ++ ITEM_DATA(test_var_data), ++ ITEM_DATA(vmap_area_list), ++#endif ++}; ++ ++static char stub_name[KSYM_NAME_LEN]; ++ ++static int stat_symbol_len(void *data, const char *name, unsigned long addr) ++{ ++ *(u32 *)data += strlen(name); ++ ++ return 0; ++} ++ ++static void test_kallsyms_compression_ratio(void) ++{ ++ int i; ++ const u8 *name; ++ u32 pos; ++ u32 ratio, total_size, total_len = 0; ++ ++ kallsyms_on_each_symbol(stat_symbol_len, &total_len); ++ ++ /* ++ * A symbol name cannot start with a number. This stub name helps us ++ * traverse the entire symbol table without finding a match. It's used ++ * for subsequent performance tests, and its length is the average ++ * length of all symbol names. ++ */ ++ memset(stub_name, '4', sizeof(stub_name)); ++ pos = total_len / kallsyms_num_syms; ++ stub_name[pos] = 0; ++ ++ pos = kallsyms_num_syms - 1; ++ name = &kallsyms_names[kallsyms_markers[pos >> 8]]; ++ for (i = 0; i <= (pos & 0xff); i++) ++ name = name + (*name) + 1; ++ ++ /* ++ * 1. The length fields is not counted ++ * 2. The memory occupied by array kallsyms_token_table[] and ++ * kallsyms_token_index[] needs to be counted. ++ */ ++ total_size = (name - kallsyms_names) - kallsyms_num_syms; ++ pos = kallsyms_token_index[0xff]; ++ total_size += pos + strlen(&kallsyms_token_table[pos]) + 1; ++ total_size += 0x100 * sizeof(u16); ++ ++ pr_info(" ---------------------------------------------------------\n"); ++ pr_info("| nr_symbols | compressed size | original size | ratio(%%) |\n"); ++ pr_info("|---------------------------------------------------------|\n"); ++ ratio = 10000ULL * total_size / total_len; ++ pr_info("| %10d | %10d | %10d | %2d.%-2d |\n", ++ kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100); ++ pr_info(" ---------------------------------------------------------\n"); ++} ++ ++static int lookup_name(void *data, const char *name, unsigned long addr) ++{ ++ u64 t0, t1, t; ++ unsigned long flags; ++ struct test_stat *stat = (struct test_stat *)data; ++ ++ local_irq_save(flags); ++ t0 = sched_clock(); ++ (void)kallsyms_lookup_name(name); ++ t1 = sched_clock(); ++ local_irq_restore(flags); ++ ++ t = t1 - t0; ++ if (t < stat->min) ++ stat->min = t; ++ ++ if (t > stat->max) ++ stat->max = t; ++ ++ stat->real_cnt++; ++ stat->sum += t; ++ ++ return 0; ++} ++ ++static void test_perf_kallsyms_lookup_name(void) ++{ ++ struct test_stat stat; ++ ++ memset(&stat, 0, sizeof(stat)); ++ stat.min = INT_MAX; ++ kallsyms_on_each_symbol(lookup_name, &stat); ++ pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt); ++ pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n", ++ stat.min, stat.max, stat.sum / stat.real_cnt); ++} ++ ++static int find_symbol(void *data, const char *name, unsigned long addr) ++{ ++ struct test_stat *stat = (struct test_stat *)data; ++ ++ if (strcmp(name, stat->name) == 0) { ++ stat->real_cnt++; ++ stat->addr = addr; ++ ++ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { ++ stat->addrs[stat->save_cnt] = addr; ++ stat->save_cnt++; ++ } ++ ++ if (stat->real_cnt == stat->max) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static void test_perf_kallsyms_on_each_symbol(void) ++{ ++ u64 t0, t1; ++ unsigned long flags; ++ struct test_stat stat; ++ ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = stub_name; ++ local_irq_save(flags); ++ t0 = sched_clock(); ++ kallsyms_on_each_symbol(find_symbol, &stat); ++ t1 = sched_clock(); ++ local_irq_restore(flags); ++ pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0); ++} ++ ++static int match_symbol(void *data, unsigned long addr) ++{ ++ struct test_stat *stat = (struct test_stat *)data; ++ ++ stat->real_cnt++; ++ stat->addr = addr; ++ ++ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { ++ stat->addrs[stat->save_cnt] = addr; ++ stat->save_cnt++; ++ } ++ ++ if (stat->real_cnt == stat->max) ++ return 1; ++ ++ return 0; ++} ++ ++static void test_perf_kallsyms_on_each_match_symbol(void) ++{ ++ u64 t0, t1; ++ unsigned long flags; ++ struct test_stat stat; ++ ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = stub_name; ++ local_irq_save(flags); ++ t0 = sched_clock(); ++ kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat); ++ t1 = sched_clock(); ++ local_irq_restore(flags); ++ pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0); ++} ++ ++static int test_kallsyms_basic_function(void) ++{ ++ int i, j, ret; ++ int next = 0, nr_failed = 0; ++ char *prefix; ++ unsigned short rand; ++ unsigned long addr; ++ char namebuf[KSYM_NAME_LEN]; ++ struct test_stat stat, stat1, stat2; ++ ++ prefix = "kallsyms_lookup_name() for"; ++ for (i = 0; i < ARRAY_SIZE(test_items); i++) { ++ addr = kallsyms_lookup_name(test_items[i].name); ++ if (addr != test_items[i].addr) { ++ nr_failed++; ++ pr_info("%s %s failed: addr=%lx, expect %lx\n", ++ prefix, test_items[i].name, addr, test_items[i].addr); ++ } ++ } ++ ++ prefix = "kallsyms_on_each_symbol() for"; ++ for (i = 0; i < ARRAY_SIZE(test_items); i++) { ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = test_items[i].name; ++ kallsyms_on_each_symbol(find_symbol, &stat); ++ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { ++ nr_failed++; ++ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", ++ prefix, test_items[i].name, ++ stat.real_cnt, stat.addr, test_items[i].addr); ++ } ++ } ++ ++ prefix = "kallsyms_on_each_match_symbol() for"; ++ for (i = 0; i < ARRAY_SIZE(test_items); i++) { ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = test_items[i].name; ++ kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, &stat); ++ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { ++ nr_failed++; ++ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", ++ prefix, test_items[i].name, ++ stat.real_cnt, stat.addr, test_items[i].addr); ++ } ++ } ++ ++ if (nr_failed) ++ return -ESRCH; ++ ++ for (i = 0; i < kallsyms_num_syms; i++) { ++ addr = kallsyms_sym_address(i); ++ if (!is_ksym_addr(addr)) ++ continue; ++ ++ ret = lookup_symbol_name(addr, namebuf); ++ if (unlikely(ret)) { ++ namebuf[0] = 0; ++ goto failed; ++ } ++ ++ stat.addr = kallsyms_lookup_name(namebuf); ++ ++ memset(&stat1, 0, sizeof(stat1)); ++ stat1.max = INT_MAX; ++ kallsyms_on_each_match_symbol(match_symbol, namebuf, &stat1); ++ ++ /* ++ * kallsyms_on_each_symbol() is too slow, randomly select some ++ * symbols for test. ++ */ ++ if (i >= next) { ++ memset(&stat2, 0, sizeof(stat2)); ++ stat2.max = INT_MAX; ++ stat2.name = namebuf; ++ kallsyms_on_each_symbol(find_symbol, &stat2); ++ ++ /* ++ * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol() ++ * need to get the same traversal result. ++ */ ++ if (stat1.addr != stat2.addr || ++ stat1.real_cnt != stat2.real_cnt || ++ memcmp(stat1.addrs, stat2.addrs, ++ stat1.save_cnt * sizeof(stat1.addrs[0]))) ++ goto failed; ++ ++ /* ++ * The average of random increments is 128, that is, one of ++ * them is tested every 128 symbols. ++ */ ++ get_random_bytes(&rand, sizeof(rand)); ++ next = i + (rand & 0xff) + 1; ++ } ++ ++ /* Need to be found at least once */ ++ if (!stat1.real_cnt) ++ goto failed; ++ ++ /* ++ * kallsyms_lookup_name() returns the address of the first ++ * symbol found and cannot be NULL. ++ */ ++ if (!stat.addr || stat.addr != stat1.addrs[0]) ++ goto failed; ++ ++ /* ++ * If the addresses of all matching symbols are recorded, the ++ * target address needs to be exist. ++ */ ++ if (stat1.real_cnt <= MAX_NUM_OF_RECORDS) { ++ for (j = 0; j < stat1.save_cnt; j++) { ++ if (stat1.addrs[j] == addr) ++ break; ++ } ++ ++ if (j == stat1.save_cnt) ++ goto failed; ++ } ++ } ++ ++ return 0; ++ ++failed: ++ pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr); ++ return -ESRCH; ++} ++ ++static int test_entry(void *p) ++{ ++ int ret; ++ ++ do { ++ schedule_timeout(5 * HZ); ++ } while (system_state != SYSTEM_RUNNING); ++ ++ pr_info("start\n"); ++ ret = test_kallsyms_basic_function(); ++ if (ret) { ++ pr_info("abort\n"); ++ return 0; ++ } ++ ++ test_kallsyms_compression_ratio(); ++ test_perf_kallsyms_lookup_name(); ++ test_perf_kallsyms_on_each_symbol(); ++ test_perf_kallsyms_on_each_match_symbol(); ++ pr_info("finish\n"); ++ ++ return 0; ++} ++ ++static int __init kallsyms_test_init(void) ++{ ++ struct task_struct *t; ++ ++ t = kthread_create(test_entry, NULL, "kallsyms_test"); ++ if (IS_ERR(t)) { ++ pr_info("Create kallsyms selftest task failed\n"); ++ return PTR_ERR(t); ++ } ++ kthread_bind(t, 0); ++ wake_up_process(t); ++ ++ return 0; ++} ++late_initcall(kallsyms_test_init); +diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c +index bc475e62279d..074a91e2cfd5 100644 +--- a/kernel/livepatch/core.c ++++ b/kernel/livepatch/core.c +@@ -118,26 +118,36 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, + } + + struct klp_find_arg { +- const char *objname; + const char *name; + unsigned long addr; + unsigned long count; + unsigned long pos; + }; + +-static int klp_find_callback(void *data, const char *name, +- struct module *mod, unsigned long addr) ++static int klp_find_callback(void *data, const char *name, unsigned long addr) + { + struct klp_find_arg *args = data; + +- if ((mod && !args->objname) || (!mod && args->objname)) +- return 0; +- + if (strcmp(args->name, name)) + return 0; + +- if (args->objname && strcmp(args->objname, mod->name)) +- return 0; ++ args->addr = addr; ++ args->count++; ++ ++ /* ++ * Finish the search when the symbol is found for the desired position ++ * or the position is not defined for a non-unique symbol. ++ */ ++ if ((args->pos && (args->count == args->pos)) || ++ (!args->pos && (args->count > 1))) ++ return 1; ++ ++ return 0; ++} ++ ++static int klp_match_callback(void *data, unsigned long addr) ++{ ++ struct klp_find_arg *args = data; + + args->addr = addr; + args->count++; +@@ -157,7 +167,6 @@ static int klp_find_object_symbol(const char *objname, const char *name, + unsigned long sympos, unsigned long *addr) + { + struct klp_find_arg args = { +- .objname = objname, + .name = name, + .addr = 0, + .count = 0, +@@ -165,9 +174,9 @@ static int klp_find_object_symbol(const char *objname, const char *name, + }; + + if (objname) +- module_kallsyms_on_each_symbol(klp_find_callback, &args); ++ module_kallsyms_on_each_symbol(objname, klp_find_callback, &args); + else +- kallsyms_on_each_symbol(klp_find_callback, &args); ++ kallsyms_on_each_match_symbol(klp_match_callback, name, &args); + + /* + * Ensure an address was found. If sympos is 0, ensure symbol is unique; +diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c +index f5c5c9175333..329cef573675 100644 +--- a/kernel/module/kallsyms.c ++++ b/kernel/module/kallsyms.c +@@ -495,8 +495,8 @@ unsigned long module_kallsyms_lookup_name(const char *name) + } + + #ifdef CONFIG_LIVEPATCH +-int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, +- struct module *, unsigned long), ++int module_kallsyms_on_each_symbol(const char *modname, ++ int (*fn)(void *, const char *, unsigned long), + void *data) + { + struct module *mod; +@@ -510,6 +510,9 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, + if (mod->state == MODULE_STATE_UNFORMED) + continue; + ++ if (strcmp(modname, mod->name)) ++ continue; ++ + /* Use rcu_dereference_sched() to remain compliant with the sparse tool */ + preempt_disable(); + kallsyms = rcu_dereference_sched(mod->kallsyms); +@@ -522,10 +525,16 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, + continue; + + ret = fn(data, kallsyms_symbol_name(kallsyms, i), +- mod, kallsyms_symbol_value(sym)); ++ kallsyms_symbol_value(sym)); + if (ret != 0) + goto out; + } ++ ++ /* ++ * The given module is found, the subsequent modules do not ++ * need to be compared. ++ */ ++ break; + } + out: + mutex_unlock(&module_mutex); +diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c +index 439e2ab6905e..f135a0a334a3 100644 +--- a/kernel/trace/ftrace.c ++++ b/kernel/trace/ftrace.c +@@ -8250,8 +8250,7 @@ struct kallsyms_data { + size_t found; + }; + +-static int kallsyms_callback(void *data, const char *name, +- struct module *mod, unsigned long addr) ++static int kallsyms_callback(void *data, const char *name, unsigned long addr) + { + struct kallsyms_data *args = data; + const char **sym; +diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c +index f18e6dfc68c5..40a6fe6d14ef 100644 +--- a/scripts/kallsyms.c ++++ b/scripts/kallsyms.c +@@ -34,7 +34,8 @@ struct sym_entry { + unsigned int len; + unsigned int start_pos; + unsigned int percpu_absolute; +- unsigned char sym[]; ++ unsigned char type; ++ unsigned char name[]; + }; + + struct addr_range { +@@ -75,11 +76,6 @@ static void usage(void) + exit(1); + } + +-static char *sym_name(const struct sym_entry *s) +-{ +- return (char *)s->sym + 1; +-} +- + static bool is_ignored_symbol(const char *name, char type) + { + /* Symbol names that exactly match to the following are ignored.*/ +@@ -227,11 +223,7 @@ static struct sym_entry *read_symbol(FILE *in) + check_symbol_range(name, addr, text_ranges, ARRAY_SIZE(text_ranges)); + check_symbol_range(name, addr, &percpu_range, 1); + +- /* include the type field in the symbol name, so that it gets +- * compressed together */ +- +- len = strlen(name) + 1; +- ++ len = strlen(name); + sym = malloc(sizeof(*sym) + len + 1); + if (!sym) { + fprintf(stderr, "kallsyms failure: " +@@ -240,8 +232,8 @@ static struct sym_entry *read_symbol(FILE *in) + } + sym->addr = addr; + sym->len = len; +- sym->sym[0] = type; +- strcpy(sym_name(sym), name); ++ sym->type = type; ++ strcpy((char *)sym->name, name); + sym->percpu_absolute = 0; + + return sym; +@@ -265,7 +257,7 @@ static int symbol_in_range(const struct sym_entry *s, + + static int symbol_valid(const struct sym_entry *s) + { +- const char *name = sym_name(s); ++ const char *name = (char *)s->name; + + /* if --all-symbols is not specified, then symbols outside the text + * and inittext sections are discarded */ +@@ -471,12 +463,18 @@ static void write_src(void) + if ((i & 0xFF) == 0) + markers[i >> 8] = off; + +- printf("\t.byte 0x%02x", table[i]->len); ++ /* ++ * Store the symbol type togerher with symbol name. ++ * It helps to reduce the size. ++ */ ++ printf("\t.byte 0x%02x", table[i]->len + 1); ++ printf(", 0x%02x", table[i]->type); + for (k = 0; k < table[i]->len; k++) +- printf(", 0x%02x", table[i]->sym[k]); ++ printf(", 0x%02x", table[i]->name[k]); + printf("\n"); + +- off += table[i]->len + 1; ++ /* fields 'len' and 'type' occupy one byte each */ ++ off += table[i]->len + 1 + 1; + } + printf("\n"); + +@@ -501,6 +499,24 @@ static void write_src(void) + for (i = 0; i < 256; i++) + printf("\t.short\t%d\n", best_idx[i]); + printf("\n"); ++ ++ output_label("kallsyms_best_token_table"); ++ for (i = 255, k = 0; (int)i >= 0; i--) { ++ if (best_table_len[i] <= 1) { ++ k++; ++ continue; ++ } ++ ++ if (k) { ++ printf("\t.byte 0x00, 0x%02x\n", k); ++ k = 0; ++ } ++ ++ printf("\t.byte 0x%02x, 0x%02x\n", best_table[i][0], best_table[i][1]); ++ } ++ if (k) ++ printf("\t.byte 0x00, 0x%02x\n", k); ++ printf("\n"); + } + + +@@ -525,12 +541,12 @@ static void forget_symbol(const unsigned char *symbol, int len) + } + + /* do the initial token count */ +-static void build_initial_tok_table(void) ++static void build_initial_token_table(void) + { + unsigned int i; + + for (i = 0; i < table_cnt; i++) +- learn_symbol(table[i]->sym, table[i]->len); ++ learn_symbol(table[i]->name, table[i]->len); + } + + static unsigned char *find_token(unsigned char *str, int len, +@@ -555,14 +571,14 @@ static void compress_symbols(const unsigned char *str, int idx) + for (i = 0; i < table_cnt; i++) { + + len = table[i]->len; +- p1 = table[i]->sym; ++ p1 = table[i]->name; + + /* find the token on the symbol */ + p2 = find_token(p1, len, str); + if (!p2) continue; + + /* decrease the counts for this symbol's tokens */ +- forget_symbol(table[i]->sym, len); ++ forget_symbol(table[i]->name, len); + + size = len; + +@@ -584,7 +600,7 @@ static void compress_symbols(const unsigned char *str, int idx) + table[i]->len = len; + + /* increase the counts for this symbol's new tokens */ +- learn_symbol(table[i]->sym, len); ++ learn_symbol(table[i]->name, len); + } + } + +@@ -637,20 +653,24 @@ static void optimize_result(void) + /* start by placing the symbols that are actually used on the table */ + static void insert_real_symbols_in_table(void) + { +- unsigned int i, j, c; ++ unsigned int i, j; ++ unsigned char c; + + for (i = 0; i < table_cnt; i++) { + for (j = 0; j < table[i]->len; j++) { +- c = table[i]->sym[j]; +- best_table[c][0]=c; +- best_table_len[c]=1; ++ c = table[i]->name[j]; ++ best_table[c][0] = c; ++ best_table_len[c] = 1; + } ++ c = table[i]->type; ++ best_table[c][0] = c; ++ best_table_len[c] = 1; + } + } + + static void optimize_token_table(void) + { +- build_initial_tok_table(); ++ build_initial_token_table(); + + insert_real_symbols_in_table(); + +@@ -660,8 +680,8 @@ static void optimize_token_table(void) + /* guess for "linker script provide" symbol */ + static int may_be_linker_script_provide_symbol(const struct sym_entry *se) + { +- const char *symbol = sym_name(se); +- int len = se->len - 1; ++ const char *symbol = (char *)se->name; ++ int len = se->len; + + if (len < 8) + return 0; +@@ -705,8 +725,8 @@ static int compare_symbols(const void *a, const void *b) + return -1; + + /* sort by "weakness" type */ +- wa = (sa->sym[0] == 'w') || (sa->sym[0] == 'W'); +- wb = (sb->sym[0] == 'w') || (sb->sym[0] == 'W'); ++ wa = (sa->type == 'w') || (sa->type == 'W'); ++ wb = (sb->type == 'w') || (sb->type == 'W'); + if (wa != wb) + return wa - wb; + +@@ -717,8 +737,8 @@ static int compare_symbols(const void *a, const void *b) + return wa - wb; + + /* sort by the number of prefix underscores */ +- wa = strspn(sym_name(sa), "_"); +- wb = strspn(sym_name(sb), "_"); ++ wa = strspn((char *)sa->name, "_"); ++ wb = strspn((char *)sb->name, "_"); + if (wa != wb) + return wa - wb; + +@@ -742,7 +762,7 @@ static void make_percpus_absolute(void) + * ensure consistent behavior compared to older + * versions of this tool. + */ +- table[i]->sym[0] = 'A'; ++ table[i]->type = 'A'; + table[i]->percpu_absolute = 1; + } + } +-- +2.38.0 + +From 318ae604f16ece7680af7a75b1ae427ec2a35b88 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Sun, 2 Oct 2022 19:11:33 +0200 +Subject: [PATCH 13/17] rcu + +Signed-off-by: Peter Jung +--- + Documentation/RCU/checklist.rst | 15 +- + Documentation/RCU/rcu_dereference.rst | 14 +- + Documentation/RCU/whatisRCU.rst | 47 ++-- + include/linux/rcupdate.h | 42 +++- + include/linux/rcutiny.h | 50 ++++ + include/linux/rcutree.h | 40 ++++ + include/linux/srcutiny.h | 10 +- + kernel/rcu/rcutorture.c | 290 ++++++++++++++++++---- + kernel/rcu/srcutiny.c | 14 +- + kernel/rcu/tasks.h | 5 +- + kernel/rcu/tiny.c | 27 ++- + kernel/rcu/tree.c | 330 ++++++++++++++++++++------ + kernel/rcu/tree_exp.h | 57 ++++- + kernel/rcu/tree_nocb.h | 10 +- + kernel/rcu/tree_plugin.h | 26 +- + kernel/rcu/tree_stall.h | 5 +- + kernel/sched/core.c | 14 ++ + kernel/smp.c | 3 +- + 18 files changed, 813 insertions(+), 186 deletions(-) + +diff --git a/Documentation/RCU/checklist.rst b/Documentation/RCU/checklist.rst +index 42cc5d891bd2..178ca7547b98 100644 +--- a/Documentation/RCU/checklist.rst ++++ b/Documentation/RCU/checklist.rst +@@ -66,8 +66,13 @@ over a rather long period of time, but improvements are always welcome! + As a rough rule of thumb, any dereference of an RCU-protected + pointer must be covered by rcu_read_lock(), rcu_read_lock_bh(), + rcu_read_lock_sched(), or by the appropriate update-side lock. +- Disabling of preemption can serve as rcu_read_lock_sched(), but +- is less readable and prevents lockdep from detecting locking issues. ++ Explicit disabling of preemption (preempt_disable(), for example) ++ can serve as rcu_read_lock_sched(), but is less readable and ++ prevents lockdep from detecting locking issues. ++ ++ Please not that you *cannot* rely on code known to be built ++ only in non-preemptible kernels. Such code can and will break, ++ especially in kernels built with CONFIG_PREEMPT_COUNT=y. + + Letting RCU-protected pointers "leak" out of an RCU read-side + critical section is every bit as bad as letting them leak out +@@ -185,6 +190,9 @@ over a rather long period of time, but improvements are always welcome! + + 5. If call_rcu() or call_srcu() is used, the callback function will + be called from softirq context. In particular, it cannot block. ++ If you need the callback to block, run that code in a workqueue ++ handler scheduled from the callback. The queue_rcu_work() ++ function does this for you in the case of call_rcu(). + + 6. Since synchronize_rcu() can block, it cannot be called + from any sort of irq context. The same rule applies +@@ -297,7 +305,8 @@ over a rather long period of time, but improvements are always welcome! + the machine. + + d. Periodically invoke synchronize_rcu(), permitting a limited +- number of updates per grace period. ++ number of updates per grace period. Better yet, periodically ++ invoke rcu_barrier() to wait for all outstanding callbacks. + + The same cautions apply to call_srcu() and kfree_rcu(). + +diff --git a/Documentation/RCU/rcu_dereference.rst b/Documentation/RCU/rcu_dereference.rst +index 0b418a5b243c..81e828c8313b 100644 +--- a/Documentation/RCU/rcu_dereference.rst ++++ b/Documentation/RCU/rcu_dereference.rst +@@ -128,10 +128,16 @@ Follow these rules to keep your RCU code working properly: + This sort of comparison occurs frequently when scanning + RCU-protected circular linked lists. + +- Note that if checks for being within an RCU read-side +- critical section are not required and the pointer is never +- dereferenced, rcu_access_pointer() should be used in place +- of rcu_dereference(). ++ Note that if the pointer comparison is done outside ++ of an RCU read-side critical section, and the pointer ++ is never dereferenced, rcu_access_pointer() should be ++ used in place of rcu_dereference(). In most cases, ++ it is best to avoid accidental dereferences by testing ++ the rcu_access_pointer() return value directly, without ++ assigning it to a variable. ++ ++ Within an RCU read-side critical section, there is little ++ reason to use rcu_access_pointer(). + + - The comparison is against a pointer that references memory + that was initialized "a long time ago." The reason +diff --git a/Documentation/RCU/whatisRCU.rst b/Documentation/RCU/whatisRCU.rst +index 77ea260efd12..1c747ac3f2c8 100644 +--- a/Documentation/RCU/whatisRCU.rst ++++ b/Documentation/RCU/whatisRCU.rst +@@ -6,13 +6,15 @@ What is RCU? -- "Read, Copy, Update" + Please note that the "What is RCU?" LWN series is an excellent place + to start learning about RCU: + +-| 1. What is RCU, Fundamentally? http://lwn.net/Articles/262464/ +-| 2. What is RCU? Part 2: Usage http://lwn.net/Articles/263130/ +-| 3. RCU part 3: the RCU API http://lwn.net/Articles/264090/ +-| 4. The RCU API, 2010 Edition http://lwn.net/Articles/418853/ +-| 2010 Big API Table http://lwn.net/Articles/419086/ +-| 5. The RCU API, 2014 Edition http://lwn.net/Articles/609904/ +-| 2014 Big API Table http://lwn.net/Articles/609973/ ++| 1. What is RCU, Fundamentally? https://lwn.net/Articles/262464/ ++| 2. What is RCU? Part 2: Usage https://lwn.net/Articles/263130/ ++| 3. RCU part 3: the RCU API https://lwn.net/Articles/264090/ ++| 4. The RCU API, 2010 Edition https://lwn.net/Articles/418853/ ++| 2010 Big API Table https://lwn.net/Articles/419086/ ++| 5. The RCU API, 2014 Edition https://lwn.net/Articles/609904/ ++| 2014 Big API Table https://lwn.net/Articles/609973/ ++| 6. The RCU API, 2019 Edition https://lwn.net/Articles/777036/ ++| 2019 Big API Table https://lwn.net/Articles/777165/ + + + What is RCU? +@@ -915,13 +917,18 @@ which an RCU reference is held include: + The understanding that RCU provides a reference that only prevents a + change of type is particularly visible with objects allocated from a + slab cache marked ``SLAB_TYPESAFE_BY_RCU``. RCU operations may yield a +-reference to an object from such a cache that has been concurrently +-freed and the memory reallocated to a completely different object, +-though of the same type. In this case RCU doesn't even protect the +-identity of the object from changing, only its type. So the object +-found may not be the one expected, but it will be one where it is safe +-to take a reference or spinlock and then confirm that the identity +-matches the expectations. ++reference to an object from such a cache that has been concurrently freed ++and the memory reallocated to a completely different object, though of ++the same type. In this case RCU doesn't even protect the identity of the ++object from changing, only its type. So the object found may not be the ++one expected, but it will be one where it is safe to take a reference ++(and then potentially acquiring a spinlock), allowing subsequent code ++to check whether the identity matches expectations. It is tempting ++to simply acquire the spinlock without first taking the reference, but ++unfortunately any spinlock in a ``SLAB_TYPESAFE_BY_RCU`` object must be ++initialized after each and every call to kmem_cache_alloc(), which renders ++reference-free spinlock acquisition completely unsafe. Therefore, when ++using ``SLAB_TYPESAFE_BY_RCU``, make proper use of a reference counter. + + With traditional reference counting -- such as that implemented by the + kref library in Linux -- there is typically code that runs when the last +@@ -1057,14 +1064,20 @@ SRCU: Initialization/cleanup:: + init_srcu_struct + cleanup_srcu_struct + +-All: lockdep-checked RCU-protected pointer access:: ++All: lockdep-checked RCU utility APIs:: + +- rcu_access_pointer +- rcu_dereference_raw + RCU_LOCKDEP_WARN + rcu_sleep_check + RCU_NONIDLE + ++All: Unchecked RCU-protected pointer access:: ++ ++ rcu_dereference_raw ++ ++All: Unchecked RCU-protected pointer access with dereferencing prohibited:: ++ ++ rcu_access_pointer ++ + See the comment headers in the source code (or the docbook generated + from them) for more information. + +diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h +index f527f27e6438..08605ce7379d 100644 +--- a/include/linux/rcupdate.h ++++ b/include/linux/rcupdate.h +@@ -42,7 +42,31 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); + void rcu_barrier_tasks(void); + void rcu_barrier_tasks_rude(void); + void synchronize_rcu(void); ++ ++struct rcu_gp_oldstate; + unsigned long get_completed_synchronize_rcu(void); ++void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); ++ ++// Maximum number of unsigned long values corresponding to ++// not-yet-completed RCU grace periods. ++#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2 ++ ++/** ++ * same_state_synchronize_rcu - Are two old-state values identical? ++ * @oldstate1: First old-state value. ++ * @oldstate2: Second old-state value. ++ * ++ * The two old-state values must have been obtained from either ++ * get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or ++ * get_completed_synchronize_rcu(). Returns @true if the two values are ++ * identical and @false otherwise. This allows structures whose lifetimes ++ * are tracked by old-state values to push these values to a list header, ++ * allowing those structures to be slightly smaller. ++ */ ++static inline bool same_state_synchronize_rcu(unsigned long oldstate1, unsigned long oldstate2) ++{ ++ return oldstate1 == oldstate2; ++} + + #ifdef CONFIG_PREEMPT_RCU + +@@ -496,13 +520,21 @@ do { \ + * against NULL. Although rcu_access_pointer() may also be used in cases + * where update-side locks prevent the value of the pointer from changing, + * you should instead use rcu_dereference_protected() for this use case. ++ * Within an RCU read-side critical section, there is little reason to ++ * use rcu_access_pointer(). ++ * ++ * It is usually best to test the rcu_access_pointer() return value ++ * directly in order to avoid accidental dereferences being introduced ++ * by later inattentive changes. In other words, assigning the ++ * rcu_access_pointer() return value to a local variable results in an ++ * accident waiting to happen. + * + * It is also permissible to use rcu_access_pointer() when read-side +- * access to the pointer was removed at least one grace period ago, as +- * is the case in the context of the RCU callback that is freeing up +- * the data, or after a synchronize_rcu() returns. This can be useful +- * when tearing down multi-linked structures after a grace period +- * has elapsed. ++ * access to the pointer was removed at least one grace period ago, as is ++ * the case in the context of the RCU callback that is freeing up the data, ++ * or after a synchronize_rcu() returns. This can be useful when tearing ++ * down multi-linked structures after a grace period has elapsed. However, ++ * rcu_dereference_protected() is normally preferred for this use case. + */ + #define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu) + +diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h +index 62815c0a2dce..768196a5f39d 100644 +--- a/include/linux/rcutiny.h ++++ b/include/linux/rcutiny.h +@@ -14,25 +14,75 @@ + + #include /* for HZ */ + ++struct rcu_gp_oldstate { ++ unsigned long rgos_norm; ++}; ++ ++// Maximum number of rcu_gp_oldstate values corresponding to ++// not-yet-completed RCU grace periods. ++#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 ++ ++/* ++ * Are the two oldstate values the same? See the Tree RCU version for ++ * docbook header. ++ */ ++static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, ++ struct rcu_gp_oldstate *rgosp2) ++{ ++ return rgosp1->rgos_norm == rgosp2->rgos_norm; ++} ++ + unsigned long get_state_synchronize_rcu(void); ++ ++static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ rgosp->rgos_norm = get_state_synchronize_rcu(); ++} ++ + unsigned long start_poll_synchronize_rcu(void); ++ ++static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ rgosp->rgos_norm = start_poll_synchronize_rcu(); ++} ++ + bool poll_state_synchronize_rcu(unsigned long oldstate); + ++static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ return poll_state_synchronize_rcu(rgosp->rgos_norm); ++} ++ + static inline void cond_synchronize_rcu(unsigned long oldstate) + { + might_sleep(); + } + ++static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ cond_synchronize_rcu(rgosp->rgos_norm); ++} ++ + static inline unsigned long start_poll_synchronize_rcu_expedited(void) + { + return start_poll_synchronize_rcu(); + } + ++static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) ++{ ++ rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); ++} ++ + static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) + { + cond_synchronize_rcu(oldstate); + } + ++static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) ++{ ++ cond_synchronize_rcu_expedited(rgosp->rgos_norm); ++} ++ + extern void rcu_barrier(void); + + static inline void synchronize_rcu_expedited(void) +diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h +index 47eaa4cb0df7..5efb51486e8a 100644 +--- a/include/linux/rcutree.h ++++ b/include/linux/rcutree.h +@@ -40,12 +40,52 @@ bool rcu_eqs_special_set(int cpu); + void rcu_momentary_dyntick_idle(void); + void kfree_rcu_scheduler_running(void); + bool rcu_gp_might_be_stalled(void); ++ ++struct rcu_gp_oldstate { ++ unsigned long rgos_norm; ++ unsigned long rgos_exp; ++}; ++ ++// Maximum number of rcu_gp_oldstate values corresponding to ++// not-yet-completed RCU grace periods. ++#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 ++ ++/** ++ * same_state_synchronize_rcu_full - Are two old-state values identical? ++ * @rgosp1: First old-state value. ++ * @rgosp2: Second old-state value. ++ * ++ * The two old-state values must have been obtained from either ++ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), ++ * or get_completed_synchronize_rcu_full(). Returns @true if the two ++ * values are identical and @false otherwise. This allows structures ++ * whose lifetimes are tracked by old-state values to push these values ++ * to a list header, allowing those structures to be slightly smaller. ++ * ++ * Note that equality is judged on a bitwise basis, so that an ++ * @rcu_gp_oldstate structure with an already-completed state in one field ++ * will compare not-equal to a structure with an already-completed state ++ * in the other field. After all, the @rcu_gp_oldstate structure is opaque ++ * so how did such a situation come to pass in the first place? ++ */ ++static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, ++ struct rcu_gp_oldstate *rgosp2) ++{ ++ return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; ++} ++ + unsigned long start_poll_synchronize_rcu_expedited(void); ++void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); + void cond_synchronize_rcu_expedited(unsigned long oldstate); ++void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); + unsigned long get_state_synchronize_rcu(void); ++void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); + unsigned long start_poll_synchronize_rcu(void); ++void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); + bool poll_state_synchronize_rcu(unsigned long oldstate); ++bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); + void cond_synchronize_rcu(unsigned long oldstate); ++void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); + + bool rcu_is_idle_cpu(int cpu); + +diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h +index 6cfaa0a9a9b9..5aa5e0faf6a1 100644 +--- a/include/linux/srcutiny.h ++++ b/include/linux/srcutiny.h +@@ -15,10 +15,10 @@ + + struct srcu_struct { + short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ +- unsigned short srcu_idx; /* Current reader array element in bit 0x2. */ +- unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */ + u8 srcu_gp_running; /* GP workqueue running? */ + u8 srcu_gp_waiting; /* GP waiting for readers? */ ++ unsigned long srcu_idx; /* Current reader array element in bit 0x2. */ ++ unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */ + struct swait_queue_head srcu_wq; + /* Last srcu_read_unlock() wakes GP. */ + struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ +@@ -82,10 +82,12 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, + int idx; + + idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1; +- pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n", ++ pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n", + tt, tf, idx, + data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])), +- data_race(READ_ONCE(ssp->srcu_lock_nesting[idx]))); ++ data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])), ++ data_race(READ_ONCE(ssp->srcu_idx)), ++ data_race(READ_ONCE(ssp->srcu_idx_max))); + } + + #endif +diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c +index d8e1b270a065..503c2aa845a4 100644 +--- a/kernel/rcu/rcutorture.c ++++ b/kernel/rcu/rcutorture.c +@@ -84,10 +84,15 @@ torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress test + torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); + torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); + torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); ++torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); ++torture_param(bool, gp_cond_exp_full, false, ++ "Use conditional/async full-stateexpedited GP wait primitives"); + torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); + torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); + torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); + torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); ++torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); ++torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); + torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); + torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); + torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); +@@ -194,16 +199,24 @@ static int rcu_torture_writer_state; + #define RTWS_DEF_FREE 3 + #define RTWS_EXP_SYNC 4 + #define RTWS_COND_GET 5 +-#define RTWS_COND_GET_EXP 6 +-#define RTWS_COND_SYNC 7 +-#define RTWS_COND_SYNC_EXP 8 +-#define RTWS_POLL_GET 9 +-#define RTWS_POLL_GET_EXP 10 +-#define RTWS_POLL_WAIT 11 +-#define RTWS_POLL_WAIT_EXP 12 +-#define RTWS_SYNC 13 +-#define RTWS_STUTTER 14 +-#define RTWS_STOPPING 15 ++#define RTWS_COND_GET_FULL 6 ++#define RTWS_COND_GET_EXP 7 ++#define RTWS_COND_GET_EXP_FULL 8 ++#define RTWS_COND_SYNC 9 ++#define RTWS_COND_SYNC_FULL 10 ++#define RTWS_COND_SYNC_EXP 11 ++#define RTWS_COND_SYNC_EXP_FULL 12 ++#define RTWS_POLL_GET 13 ++#define RTWS_POLL_GET_FULL 14 ++#define RTWS_POLL_GET_EXP 15 ++#define RTWS_POLL_GET_EXP_FULL 16 ++#define RTWS_POLL_WAIT 17 ++#define RTWS_POLL_WAIT_FULL 18 ++#define RTWS_POLL_WAIT_EXP 19 ++#define RTWS_POLL_WAIT_EXP_FULL 20 ++#define RTWS_SYNC 21 ++#define RTWS_STUTTER 22 ++#define RTWS_STOPPING 23 + static const char * const rcu_torture_writer_state_names[] = { + "RTWS_FIXED_DELAY", + "RTWS_DELAY", +@@ -211,13 +224,21 @@ static const char * const rcu_torture_writer_state_names[] = { + "RTWS_DEF_FREE", + "RTWS_EXP_SYNC", + "RTWS_COND_GET", ++ "RTWS_COND_GET_FULL", + "RTWS_COND_GET_EXP", ++ "RTWS_COND_GET_EXP_FULL", + "RTWS_COND_SYNC", ++ "RTWS_COND_SYNC_FULL", + "RTWS_COND_SYNC_EXP", ++ "RTWS_COND_SYNC_EXP_FULL", + "RTWS_POLL_GET", ++ "RTWS_POLL_GET_FULL", + "RTWS_POLL_GET_EXP", ++ "RTWS_POLL_GET_EXP_FULL", + "RTWS_POLL_WAIT", ++ "RTWS_POLL_WAIT_FULL", + "RTWS_POLL_WAIT_EXP", ++ "RTWS_POLL_WAIT_EXP_FULL", + "RTWS_SYNC", + "RTWS_STUTTER", + "RTWS_STOPPING", +@@ -332,13 +353,21 @@ struct rcu_torture_ops { + void (*exp_sync)(void); + unsigned long (*get_gp_state_exp)(void); + unsigned long (*start_gp_poll_exp)(void); ++ void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); + bool (*poll_gp_state_exp)(unsigned long oldstate); + void (*cond_sync_exp)(unsigned long oldstate); ++ void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); + unsigned long (*get_gp_state)(void); ++ void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); + unsigned long (*get_gp_completed)(void); ++ void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp); + unsigned long (*start_gp_poll)(void); ++ void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); + bool (*poll_gp_state)(unsigned long oldstate); ++ bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); ++ bool (*poll_need_2gp)(bool poll, bool poll_full); + void (*cond_sync)(unsigned long oldstate); ++ void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); + call_rcu_func_t call; + void (*cb_barrier)(void); + void (*fqs)(void); +@@ -489,6 +518,11 @@ static void rcu_sync_torture_init(void) + INIT_LIST_HEAD(&rcu_torture_removed); + } + ++static bool rcu_poll_need_2gp(bool poll, bool poll_full) ++{ ++ return poll; ++} ++ + static struct rcu_torture_ops rcu_ops = { + .ttype = RCU_FLAVOR, + .init = rcu_sync_torture_init, +@@ -502,12 +536,19 @@ static struct rcu_torture_ops rcu_ops = { + .sync = synchronize_rcu, + .exp_sync = synchronize_rcu_expedited, + .get_gp_state = get_state_synchronize_rcu, ++ .get_gp_state_full = get_state_synchronize_rcu_full, + .get_gp_completed = get_completed_synchronize_rcu, ++ .get_gp_completed_full = get_completed_synchronize_rcu_full, + .start_gp_poll = start_poll_synchronize_rcu, ++ .start_gp_poll_full = start_poll_synchronize_rcu_full, + .poll_gp_state = poll_state_synchronize_rcu, ++ .poll_gp_state_full = poll_state_synchronize_rcu_full, ++ .poll_need_2gp = rcu_poll_need_2gp, + .cond_sync = cond_synchronize_rcu, ++ .cond_sync_full = cond_synchronize_rcu_full, + .get_gp_state_exp = get_state_synchronize_rcu, + .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, ++ .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, + .poll_gp_state_exp = poll_state_synchronize_rcu, + .cond_sync_exp = cond_synchronize_rcu_expedited, + .call = call_rcu, +@@ -709,6 +750,9 @@ static struct rcu_torture_ops srcud_ops = { + .deferred_free = srcu_torture_deferred_free, + .sync = srcu_torture_synchronize, + .exp_sync = srcu_torture_synchronize_expedited, ++ .get_gp_state = srcu_torture_get_gp_state, ++ .start_gp_poll = srcu_torture_start_gp_poll, ++ .poll_gp_state = srcu_torture_poll_gp_state, + .call = srcu_torture_call, + .cb_barrier = srcu_torture_barrier, + .stats = srcu_torture_stats, +@@ -1148,15 +1192,35 @@ static int nsynctypes; + */ + static void rcu_torture_write_types(void) + { +- bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp; +- bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll; +- bool gp_sync1 = gp_sync; ++ bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; ++ bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; ++ bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; ++ bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; + + /* Initialize synctype[] array. If none set, take default. */ +- if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp && +- !gp_normal1 && !gp_poll1 && !gp_sync1) +- gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 = +- gp_normal1 = gp_poll1 = gp_sync1 = true; ++ if (!gp_cond1 && ++ !gp_cond_exp1 && ++ !gp_cond_full1 && ++ !gp_cond_exp_full1 && ++ !gp_exp1 && ++ !gp_poll_exp1 && ++ !gp_poll_exp_full1 && ++ !gp_normal1 && ++ !gp_poll1 && ++ !gp_poll_full1 && ++ !gp_sync1) { ++ gp_cond1 = true; ++ gp_cond_exp1 = true; ++ gp_cond_full1 = true; ++ gp_cond_exp_full1 = true; ++ gp_exp1 = true; ++ gp_poll_exp1 = true; ++ gp_poll_exp_full1 = true; ++ gp_normal1 = true; ++ gp_poll1 = true; ++ gp_poll_full1 = true; ++ gp_sync1 = true; ++ } + if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { + synctype[nsynctypes++] = RTWS_COND_GET; + pr_info("%s: Testing conditional GPs.\n", __func__); +@@ -1169,6 +1233,19 @@ static void rcu_torture_write_types(void) + } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { + pr_alert("%s: gp_cond_exp without primitives.\n", __func__); + } ++ if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { ++ synctype[nsynctypes++] = RTWS_COND_GET_FULL; ++ pr_info("%s: Testing conditional full-state GPs.\n", __func__); ++ } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { ++ pr_alert("%s: gp_cond_full without primitives.\n", __func__); ++ } ++ if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { ++ synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; ++ pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); ++ } else if (gp_cond_exp_full && ++ (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { ++ pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); ++ } + if (gp_exp1 && cur_ops->exp_sync) { + synctype[nsynctypes++] = RTWS_EXP_SYNC; + pr_info("%s: Testing expedited GPs.\n", __func__); +@@ -1187,12 +1264,25 @@ static void rcu_torture_write_types(void) + } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { + pr_alert("%s: gp_poll without primitives.\n", __func__); + } ++ if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { ++ synctype[nsynctypes++] = RTWS_POLL_GET_FULL; ++ pr_info("%s: Testing polling full-state GPs.\n", __func__); ++ } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { ++ pr_alert("%s: gp_poll_full without primitives.\n", __func__); ++ } + if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { + synctype[nsynctypes++] = RTWS_POLL_GET_EXP; + pr_info("%s: Testing polling expedited GPs.\n", __func__); + } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { + pr_alert("%s: gp_poll_exp without primitives.\n", __func__); + } ++ if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { ++ synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; ++ pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); ++ } else if (gp_poll_exp_full && ++ (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { ++ pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); ++ } + if (gp_sync1 && cur_ops->sync) { + synctype[nsynctypes++] = RTWS_SYNC; + pr_info("%s: Testing normal GPs.\n", __func__); +@@ -1201,6 +1291,40 @@ static void rcu_torture_write_types(void) + } + } + ++/* ++ * Do the specified rcu_torture_writer() synchronous grace period, ++ * while also testing out the polled APIs. Note well that the single-CPU ++ * grace-period optimizations must be accounted for. ++ */ ++static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) ++{ ++ unsigned long cookie; ++ struct rcu_gp_oldstate cookie_full; ++ bool dopoll; ++ bool dopoll_full; ++ unsigned long r = torture_random(trsp); ++ ++ dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); ++ dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); ++ if (dopoll || dopoll_full) ++ cpus_read_lock(); ++ if (dopoll) ++ cookie = cur_ops->get_gp_state(); ++ if (dopoll_full) ++ cur_ops->get_gp_state_full(&cookie_full); ++ if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) ++ sync(); ++ sync(); ++ WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), ++ "%s: Cookie check 3 failed %pS() online %*pbl.", ++ __func__, sync, cpumask_pr_args(cpu_online_mask)); ++ WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), ++ "%s: Cookie check 4 failed %pS() online %*pbl", ++ __func__, sync, cpumask_pr_args(cpu_online_mask)); ++ if (dopoll || dopoll_full) ++ cpus_read_unlock(); ++} ++ + /* + * RCU torture writer kthread. Repeatedly substitutes a new structure + * for that pointed to by rcu_torture_current, freeing the old structure +@@ -1212,8 +1336,10 @@ rcu_torture_writer(void *arg) + bool boot_ended; + bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); + unsigned long cookie; ++ struct rcu_gp_oldstate cookie_full; + int expediting = 0; + unsigned long gp_snap; ++ struct rcu_gp_oldstate gp_snap_full; + int i; + int idx; + int oldnice = task_nice(current); +@@ -1261,11 +1387,12 @@ rcu_torture_writer(void *arg) + atomic_inc(&rcu_torture_wcount[i]); + WRITE_ONCE(old_rp->rtort_pipe_count, + old_rp->rtort_pipe_count + 1); ++ ++ // Make sure readers block polled grace periods. + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { + idx = cur_ops->readlock(); + cookie = cur_ops->get_gp_state(); +- WARN_ONCE(rcu_torture_writer_state != RTWS_DEF_FREE && +- cur_ops->poll_gp_state(cookie), ++ WARN_ONCE(cur_ops->poll_gp_state(cookie), + "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", + __func__, + rcu_torture_writer_state_getname(), +@@ -1277,6 +1404,21 @@ rcu_torture_writer(void *arg) + } + cur_ops->readunlock(idx); + } ++ if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { ++ idx = cur_ops->readlock(); ++ cur_ops->get_gp_state_full(&cookie_full); ++ WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), ++ "%s: Cookie check 5 failed %s(%d) online %*pbl\n", ++ __func__, ++ rcu_torture_writer_state_getname(), ++ rcu_torture_writer_state, ++ cpumask_pr_args(cpu_online_mask)); ++ if (cur_ops->get_gp_completed_full) { ++ cur_ops->get_gp_completed_full(&cookie_full); ++ WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); ++ } ++ cur_ops->readunlock(idx); ++ } + switch (synctype[torture_random(&rand) % nsynctypes]) { + case RTWS_DEF_FREE: + rcu_torture_writer_state = RTWS_DEF_FREE; +@@ -1284,12 +1426,7 @@ rcu_torture_writer(void *arg) + break; + case RTWS_EXP_SYNC: + rcu_torture_writer_state = RTWS_EXP_SYNC; +- if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +- cookie = cur_ops->get_gp_state(); +- cur_ops->exp_sync(); +- cur_ops->exp_sync(); +- if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +- WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); ++ do_rtws_sync(&rand, cur_ops->exp_sync); + rcu_torture_pipe_update(old_rp); + break; + case RTWS_COND_GET: +@@ -1308,6 +1445,22 @@ rcu_torture_writer(void *arg) + cur_ops->cond_sync_exp(gp_snap); + rcu_torture_pipe_update(old_rp); + break; ++ case RTWS_COND_GET_FULL: ++ rcu_torture_writer_state = RTWS_COND_GET_FULL; ++ cur_ops->get_gp_state_full(&gp_snap_full); ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); ++ rcu_torture_writer_state = RTWS_COND_SYNC_FULL; ++ cur_ops->cond_sync_full(&gp_snap_full); ++ rcu_torture_pipe_update(old_rp); ++ break; ++ case RTWS_COND_GET_EXP_FULL: ++ rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; ++ cur_ops->get_gp_state_full(&gp_snap_full); ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); ++ rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; ++ cur_ops->cond_sync_exp_full(&gp_snap_full); ++ rcu_torture_pipe_update(old_rp); ++ break; + case RTWS_POLL_GET: + rcu_torture_writer_state = RTWS_POLL_GET; + gp_snap = cur_ops->start_gp_poll(); +@@ -1317,6 +1470,15 @@ rcu_torture_writer(void *arg) + &rand); + rcu_torture_pipe_update(old_rp); + break; ++ case RTWS_POLL_GET_FULL: ++ rcu_torture_writer_state = RTWS_POLL_GET_FULL; ++ cur_ops->start_gp_poll_full(&gp_snap_full); ++ rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; ++ while (!cur_ops->poll_gp_state_full(&gp_snap_full)) ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, ++ &rand); ++ rcu_torture_pipe_update(old_rp); ++ break; + case RTWS_POLL_GET_EXP: + rcu_torture_writer_state = RTWS_POLL_GET_EXP; + gp_snap = cur_ops->start_gp_poll_exp(); +@@ -1326,14 +1488,18 @@ rcu_torture_writer(void *arg) + &rand); + rcu_torture_pipe_update(old_rp); + break; ++ case RTWS_POLL_GET_EXP_FULL: ++ rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; ++ cur_ops->start_gp_poll_exp_full(&gp_snap_full); ++ rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; ++ while (!cur_ops->poll_gp_state_full(&gp_snap_full)) ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, ++ &rand); ++ rcu_torture_pipe_update(old_rp); ++ break; + case RTWS_SYNC: + rcu_torture_writer_state = RTWS_SYNC; +- if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +- cookie = cur_ops->get_gp_state(); +- cur_ops->sync(); +- cur_ops->sync(); +- if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +- WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); ++ do_rtws_sync(&rand, cur_ops->sync); + rcu_torture_pipe_update(old_rp); + break; + default: +@@ -1400,6 +1566,7 @@ static int + rcu_torture_fakewriter(void *arg) + { + unsigned long gp_snap; ++ struct rcu_gp_oldstate gp_snap_full; + DEFINE_TORTURE_RANDOM(rand); + + VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); +@@ -1438,6 +1605,16 @@ rcu_torture_fakewriter(void *arg) + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + cur_ops->cond_sync_exp(gp_snap); + break; ++ case RTWS_COND_GET_FULL: ++ cur_ops->get_gp_state_full(&gp_snap_full); ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); ++ cur_ops->cond_sync_full(&gp_snap_full); ++ break; ++ case RTWS_COND_GET_EXP_FULL: ++ cur_ops->get_gp_state_full(&gp_snap_full); ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); ++ cur_ops->cond_sync_exp_full(&gp_snap_full); ++ break; + case RTWS_POLL_GET: + gp_snap = cur_ops->start_gp_poll(); + while (!cur_ops->poll_gp_state(gp_snap)) { +@@ -1445,6 +1622,13 @@ rcu_torture_fakewriter(void *arg) + &rand); + } + break; ++ case RTWS_POLL_GET_FULL: ++ cur_ops->start_gp_poll_full(&gp_snap_full); ++ while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, ++ &rand); ++ } ++ break; + case RTWS_POLL_GET_EXP: + gp_snap = cur_ops->start_gp_poll_exp(); + while (!cur_ops->poll_gp_state_exp(gp_snap)) { +@@ -1452,6 +1636,13 @@ rcu_torture_fakewriter(void *arg) + &rand); + } + break; ++ case RTWS_POLL_GET_EXP_FULL: ++ cur_ops->start_gp_poll_exp_full(&gp_snap_full); ++ while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { ++ torture_hrtimeout_jiffies(torture_random(&rand) % 16, ++ &rand); ++ } ++ break; + case RTWS_SYNC: + cur_ops->sync(); + break; +@@ -1715,7 +1906,9 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, + */ + static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) + { ++ bool checkpolling = !(torture_random(trsp) & 0xfff); + unsigned long cookie; ++ struct rcu_gp_oldstate cookie_full; + int i; + unsigned long started; + unsigned long completed; +@@ -1731,8 +1924,12 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) + WARN_ON_ONCE(!rcu_is_watching()); + newstate = rcutorture_extend_mask(readstate, trsp); + rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); +- if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +- cookie = cur_ops->get_gp_state(); ++ if (checkpolling) { ++ if (cur_ops->get_gp_state && cur_ops->poll_gp_state) ++ cookie = cur_ops->get_gp_state(); ++ if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) ++ cur_ops->get_gp_state_full(&cookie_full); ++ } + started = cur_ops->get_gp_seq(); + ts = rcu_trace_clock_local(); + p = rcu_dereference_check(rcu_torture_current, +@@ -1766,13 +1963,22 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) + } + __this_cpu_inc(rcu_torture_batch[completed]); + preempt_enable(); +- if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +- WARN_ONCE(cur_ops->poll_gp_state(cookie), +- "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", +- __func__, +- rcu_torture_writer_state_getname(), +- rcu_torture_writer_state, +- cookie, cur_ops->get_gp_state()); ++ if (checkpolling) { ++ if (cur_ops->get_gp_state && cur_ops->poll_gp_state) ++ WARN_ONCE(cur_ops->poll_gp_state(cookie), ++ "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", ++ __func__, ++ rcu_torture_writer_state_getname(), ++ rcu_torture_writer_state, ++ cookie, cur_ops->get_gp_state()); ++ if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) ++ WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), ++ "%s: Cookie check 6 failed %s(%d) online %*pbl\n", ++ __func__, ++ rcu_torture_writer_state_getname(), ++ rcu_torture_writer_state, ++ cpumask_pr_args(cpu_online_mask)); ++ } + rcutorture_one_extend(&readstate, 0, trsp, rtrsp); + WARN_ON_ONCE(readstate); + // This next splat is expected behavior if leakpointer, especially +@@ -2600,12 +2806,12 @@ static int rcutorture_oom_notify(struct notifier_block *self, + for (i = 0; i < fwd_progress; i++) + ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); + pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); +- rcu_barrier(); ++ cur_ops->cb_barrier(); + ncbs = 0; + for (i = 0; i < fwd_progress; i++) + ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); + pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); +- rcu_barrier(); ++ cur_ops->cb_barrier(); + ncbs = 0; + for (i = 0; i < fwd_progress; i++) + ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); +diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c +index 92c002d65482..33adafdad261 100644 +--- a/kernel/rcu/srcutiny.c ++++ b/kernel/rcu/srcutiny.c +@@ -117,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) + struct srcu_struct *ssp; + + ssp = container_of(wp, struct srcu_struct, srcu_work); +- if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) ++ if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) + return; /* Already running or nothing to do. */ + + /* Remove recently arrived callbacks and wait for readers. */ +@@ -150,17 +150,17 @@ void srcu_drive_gp(struct work_struct *wp) + * straighten that out. + */ + WRITE_ONCE(ssp->srcu_gp_running, false); +- if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) ++ if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) + schedule_work(&ssp->srcu_work); + } + EXPORT_SYMBOL_GPL(srcu_drive_gp); + + static void srcu_gp_start_if_needed(struct srcu_struct *ssp) + { +- unsigned short cookie; ++ unsigned long cookie; + + cookie = get_state_synchronize_srcu(ssp); +- if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) ++ if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) + return; + WRITE_ONCE(ssp->srcu_idx_max, cookie); + if (!READ_ONCE(ssp->srcu_gp_running)) { +@@ -215,7 +215,7 @@ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) + barrier(); + ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1; + barrier(); +- return ret & USHRT_MAX; ++ return ret; + } + EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); + +@@ -240,10 +240,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); + */ + bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) + { +- bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie); ++ unsigned long cur_s = READ_ONCE(ssp->srcu_idx); + + barrier(); +- return ret; ++ return ULONG_CMP_GE(cur_s, cookie) || ULONG_CMP_LT(cur_s, cookie - 3); + } + EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); + +diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h +index 83c7e6620d40..f5bf6fb430da 100644 +--- a/kernel/rcu/tasks.h ++++ b/kernel/rcu/tasks.h +@@ -560,7 +560,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) + static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) + { + /* Complain if the scheduler has not started. */ +- RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, ++ WARN_ONCE(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, + "synchronize_rcu_tasks called too soon"); + + // If the grace-period kthread is running, use it. +@@ -1500,6 +1500,7 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) + if (rcu_tasks_trace_pertask_prep(t, true)) + trc_add_holdout(t, hop); + rcu_read_unlock(); ++ cond_resched_tasks_rcu_qs(); + } + + // Only after all running tasks have been accounted for is it +@@ -1520,6 +1521,7 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); + } + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); ++ cond_resched_tasks_rcu_qs(); + } + + // Re-enable CPU hotplug now that the holdout list is populated. +@@ -1619,6 +1621,7 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, + trc_del_holdout(t); + else if (needreport) + show_stalled_task_trace(t, firstreport); ++ cond_resched_tasks_rcu_qs(); + } + + // Re-enable CPU hotplug now that the holdout list scan has completed. +diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c +index f0561ee16b9c..a33a8d4942c3 100644 +--- a/kernel/rcu/tiny.c ++++ b/kernel/rcu/tiny.c +@@ -158,6 +158,10 @@ void synchronize_rcu(void) + } + EXPORT_SYMBOL_GPL(synchronize_rcu); + ++static void tiny_rcu_leak_callback(struct rcu_head *rhp) ++{ ++} ++ + /* + * Post an RCU callback to be invoked after the end of an RCU grace + * period. But since we have but one CPU, that would be after any +@@ -165,9 +169,20 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); + */ + void call_rcu(struct rcu_head *head, rcu_callback_t func) + { ++ static atomic_t doublefrees; + unsigned long flags; + +- debug_rcu_head_queue(head); ++ if (debug_rcu_head_queue(head)) { ++ if (atomic_inc_return(&doublefrees) < 4) { ++ pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); ++ mem_dump_obj(head); ++ } ++ ++ if (!__is_kvfree_rcu_offset((unsigned long)head->func)) ++ WRITE_ONCE(head->func, tiny_rcu_leak_callback); ++ return; ++ } ++ + head->func = func; + head->next = NULL; + +@@ -183,6 +198,16 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) + } + EXPORT_SYMBOL_GPL(call_rcu); + ++/* ++ * Store a grace-period-counter "cookie". For more information, ++ * see the Tree RCU header comment. ++ */ ++void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; ++} ++EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); ++ + /* + * Return a grace-period-counter "cookie". For more information, + * see the Tree RCU header comment. +diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c +index 79aea7df4345..6bb8e72bc815 100644 +--- a/kernel/rcu/tree.c ++++ b/kernel/rcu/tree.c +@@ -76,6 +76,7 @@ + /* Data structures. */ + + static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { ++ .gpwrap = true, + #ifdef CONFIG_RCU_NOCB_CPU + .cblist.flags = SEGCBLIST_RCU_CORE, + #endif +@@ -1755,6 +1756,8 @@ static noinline void rcu_gp_cleanup(void) + dump_blkd_tasks(rnp, 10); + WARN_ON_ONCE(rnp->qsmask); + WRITE_ONCE(rnp->gp_seq, new_gp_seq); ++ if (!rnp->parent) ++ smp_mb(); // Order against failing poll_state_synchronize_rcu_full(). + rdp = this_cpu_ptr(&rcu_data); + if (rnp == rdp->mynode) + needgp = __note_gp_changes(rnp, rdp) || needgp; +@@ -2341,8 +2344,8 @@ void rcu_sched_clock_irq(int user) + rcu_flavor_sched_clock_irq(user); + if (rcu_pending(user)) + invoke_rcu_core(); +- if (user) +- rcu_tasks_classic_qs(current, false); ++ if (user || rcu_is_cpu_rrupt_from_idle()) ++ rcu_note_voluntary_context_switch(current); + lockdep_assert_irqs_disabled(); + + trace_rcu_utilization(TPS("End scheduler-tick")); +@@ -2832,7 +2835,7 @@ EXPORT_SYMBOL_GPL(call_rcu); + + + /* Maximum number of jiffies to wait before draining a batch. */ +-#define KFREE_DRAIN_JIFFIES (HZ / 50) ++#define KFREE_DRAIN_JIFFIES (5 * HZ) + #define KFREE_N_BATCHES 2 + #define FREE_N_CHANNELS 2 + +@@ -3093,6 +3096,21 @@ need_offload_krc(struct kfree_rcu_cpu *krcp) + return !!krcp->head; + } + ++static void ++schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp) ++{ ++ long delay, delay_left; ++ ++ delay = READ_ONCE(krcp->count) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES; ++ if (delayed_work_pending(&krcp->monitor_work)) { ++ delay_left = krcp->monitor_work.timer.expires - jiffies; ++ if (delay < delay_left) ++ mod_delayed_work(system_wq, &krcp->monitor_work, delay); ++ return; ++ } ++ queue_delayed_work(system_wq, &krcp->monitor_work, delay); ++} ++ + /* + * This function is invoked after the KFREE_DRAIN_JIFFIES timeout. + */ +@@ -3150,7 +3168,7 @@ static void kfree_rcu_monitor(struct work_struct *work) + // work to repeat an attempt. Because previous batches are + // still in progress. + if (need_offload_krc(krcp)) +- schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES); ++ schedule_delayed_monitor_work(krcp); + + raw_spin_unlock_irqrestore(&krcp->lock, flags); + } +@@ -3183,15 +3201,16 @@ static void fill_page_cache_func(struct work_struct *work) + bnode = (struct kvfree_rcu_bulk_data *) + __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); + +- if (bnode) { +- raw_spin_lock_irqsave(&krcp->lock, flags); +- pushed = put_cached_bnode(krcp, bnode); +- raw_spin_unlock_irqrestore(&krcp->lock, flags); ++ if (!bnode) ++ break; + +- if (!pushed) { +- free_page((unsigned long) bnode); +- break; +- } ++ raw_spin_lock_irqsave(&krcp->lock, flags); ++ pushed = put_cached_bnode(krcp, bnode); ++ raw_spin_unlock_irqrestore(&krcp->lock, flags); ++ ++ if (!pushed) { ++ free_page((unsigned long) bnode); ++ break; + } + } + +@@ -3338,7 +3357,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) + + // Set timer to drain after KFREE_DRAIN_JIFFIES. + if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING) +- schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES); ++ schedule_delayed_monitor_work(krcp); + + unlock_return: + krc_this_cpu_unlock(krcp, flags); +@@ -3371,7 +3390,7 @@ kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) + atomic_set(&krcp->backoff_page_cache_fill, 1); + } + +- return count; ++ return count == 0 ? SHRINK_EMPTY : count; + } + + static unsigned long +@@ -3414,49 +3433,27 @@ void __init kfree_rcu_scheduler_running(void) + + raw_spin_lock_irqsave(&krcp->lock, flags); + if (need_offload_krc(krcp)) +- schedule_delayed_work_on(cpu, &krcp->monitor_work, KFREE_DRAIN_JIFFIES); ++ schedule_delayed_monitor_work(krcp); + raw_spin_unlock_irqrestore(&krcp->lock, flags); + } + } + + /* + * During early boot, any blocking grace-period wait automatically +- * implies a grace period. Later on, this is never the case for PREEMPTION. ++ * implies a grace period. + * +- * However, because a context switch is a grace period for !PREEMPTION, any +- * blocking grace-period wait automatically implies a grace period if +- * there is only one CPU online at any point time during execution of +- * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to +- * occasionally incorrectly indicate that there are multiple CPUs online +- * when there was in fact only one the whole time, as this just adds some +- * overhead: RCU still operates correctly. ++ * Later on, this could in theory be the case for kernels built with ++ * CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this ++ * is not a common case. Furthermore, this optimization would cause ++ * the rcu_gp_oldstate structure to expand by 50%, so this potential ++ * grace-period optimization is ignored once the scheduler is running. + */ + static int rcu_blocking_is_gp(void) + { +- int ret; +- +- // Invoking preempt_model_*() too early gets a splat. +- if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE || +- preempt_model_full() || preempt_model_rt()) +- return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE; ++ if (rcu_scheduler_active != RCU_SCHEDULER_INACTIVE) ++ return false; + might_sleep(); /* Check for RCU read-side critical section. */ +- preempt_disable(); +- /* +- * If the rcu_state.n_online_cpus counter is equal to one, +- * there is only one CPU, and that CPU sees all prior accesses +- * made by any CPU that was online at the time of its access. +- * Furthermore, if this counter is equal to one, its value cannot +- * change until after the preempt_enable() below. +- * +- * Furthermore, if rcu_state.n_online_cpus is equal to one here, +- * all later CPUs (both this one and any that come online later +- * on) are guaranteed to see all accesses prior to this point +- * in the code, without the need for additional memory barriers. +- * Those memory barriers are provided by CPU-hotplug code. +- */ +- ret = READ_ONCE(rcu_state.n_online_cpus) <= 1; +- preempt_enable(); +- return ret; ++ return true; + } + + /** +@@ -3499,29 +3496,58 @@ static int rcu_blocking_is_gp(void) + */ + void synchronize_rcu(void) + { ++ unsigned long flags; ++ struct rcu_node *rnp; ++ + RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || + lock_is_held(&rcu_lock_map) || + lock_is_held(&rcu_sched_lock_map), + "Illegal synchronize_rcu() in RCU read-side critical section"); +- if (rcu_blocking_is_gp()) { +- // Note well that this code runs with !PREEMPT && !SMP. +- // In addition, all code that advances grace periods runs at +- // process level. Therefore, this normal GP overlaps with +- // other normal GPs only by being fully nested within them, +- // which allows reuse of ->gp_seq_polled_snap. +- rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); +- rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); +- if (rcu_init_invoked()) +- cond_resched_tasks_rcu_qs(); +- return; // Context allows vacuous grace periods. ++ if (!rcu_blocking_is_gp()) { ++ if (rcu_gp_is_expedited()) ++ synchronize_rcu_expedited(); ++ else ++ wait_rcu_gp(call_rcu); ++ return; + } +- if (rcu_gp_is_expedited()) +- synchronize_rcu_expedited(); +- else +- wait_rcu_gp(call_rcu); ++ ++ // Context allows vacuous grace periods. ++ // Note well that this code runs with !PREEMPT && !SMP. ++ // In addition, all code that advances grace periods runs at ++ // process level. Therefore, this normal GP overlaps with other ++ // normal GPs only by being fully nested within them, which allows ++ // reuse of ->gp_seq_polled_snap. ++ rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); ++ rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); ++ ++ // Update the normal grace-period counters to record ++ // this grace period, but only those used by the boot CPU. ++ // The rcu_scheduler_starting() will take care of the rest of ++ // these counters. ++ local_irq_save(flags); ++ WARN_ON_ONCE(num_online_cpus() > 1); ++ rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT); ++ for (rnp = this_cpu_ptr(&rcu_data)->mynode; rnp; rnp = rnp->parent) ++ rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; ++ local_irq_restore(flags); + } + EXPORT_SYMBOL_GPL(synchronize_rcu); + ++/** ++ * get_completed_synchronize_rcu_full - Return a full pre-completed polled state cookie ++ * @rgosp: Place to put state cookie ++ * ++ * Stores into @rgosp a value that will always be treated by functions ++ * like poll_state_synchronize_rcu_full() as a cookie whose grace period ++ * has already completed. ++ */ ++void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; ++ rgosp->rgos_exp = RCU_GET_STATE_COMPLETED; ++} ++EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); ++ + /** + * get_state_synchronize_rcu - Snapshot current RCU state + * +@@ -3541,21 +3567,42 @@ unsigned long get_state_synchronize_rcu(void) + EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); + + /** +- * start_poll_synchronize_rcu - Snapshot and start RCU grace period ++ * get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited ++ * @rgosp: location to place combined normal/expedited grace-period state + * +- * Returns a cookie that is used by a later call to cond_synchronize_rcu() +- * or poll_state_synchronize_rcu() to determine whether or not a full +- * grace period has elapsed in the meantime. If the needed grace period +- * is not already slated to start, notifies RCU core of the need for that +- * grace period. ++ * Places the normal and expedited grace-period states in @rgosp. This ++ * state value can be passed to a later call to cond_synchronize_rcu_full() ++ * or poll_state_synchronize_rcu_full() to determine whether or not a ++ * grace period (whether normal or expedited) has elapsed in the meantime. ++ * The rcu_gp_oldstate structure takes up twice the memory of an unsigned ++ * long, but is guaranteed to see all grace periods. In contrast, the ++ * combined state occupies less memory, but can sometimes fail to take ++ * grace periods into account. + * +- * Interrupts must be enabled for the case where it is necessary to awaken +- * the grace-period kthread. ++ * This does not guarantee that the needed grace period will actually ++ * start. + */ +-unsigned long start_poll_synchronize_rcu(void) ++void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ struct rcu_node *rnp = rcu_get_root(); ++ ++ /* ++ * Any prior manipulation of RCU-protected data must happen ++ * before the loads from ->gp_seq and ->expedited_sequence. ++ */ ++ smp_mb(); /* ^^^ */ ++ rgosp->rgos_norm = rcu_seq_snap(&rnp->gp_seq); ++ rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); ++} ++EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full); ++ ++/* ++ * Helper function for start_poll_synchronize_rcu() and ++ * start_poll_synchronize_rcu_full(). ++ */ ++static void start_poll_synchronize_rcu_common(void) + { + unsigned long flags; +- unsigned long gp_seq = get_state_synchronize_rcu(); + bool needwake; + struct rcu_data *rdp; + struct rcu_node *rnp; +@@ -3575,17 +3622,57 @@ unsigned long start_poll_synchronize_rcu(void) + raw_spin_unlock_irqrestore_rcu_node(rnp, flags); + if (needwake) + rcu_gp_kthread_wake(); ++} ++ ++/** ++ * start_poll_synchronize_rcu - Snapshot and start RCU grace period ++ * ++ * Returns a cookie that is used by a later call to cond_synchronize_rcu() ++ * or poll_state_synchronize_rcu() to determine whether or not a full ++ * grace period has elapsed in the meantime. If the needed grace period ++ * is not already slated to start, notifies RCU core of the need for that ++ * grace period. ++ * ++ * Interrupts must be enabled for the case where it is necessary to awaken ++ * the grace-period kthread. ++ */ ++unsigned long start_poll_synchronize_rcu(void) ++{ ++ unsigned long gp_seq = get_state_synchronize_rcu(); ++ ++ start_poll_synchronize_rcu_common(); + return gp_seq; + } + EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); + + /** +- * poll_state_synchronize_rcu - Conditionally wait for an RCU grace period ++ * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period ++ * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * ++ * Places the normal and expedited grace-period states in *@rgos. This ++ * state value can be passed to a later call to cond_synchronize_rcu_full() ++ * or poll_state_synchronize_rcu_full() to determine whether or not a ++ * grace period (whether normal or expedited) has elapsed in the meantime. ++ * If the needed grace period is not already slated to start, notifies ++ * RCU core of the need for that grace period. ++ * ++ * Interrupts must be enabled for the case where it is necessary to awaken ++ * the grace-period kthread. ++ */ ++void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ get_state_synchronize_rcu_full(rgosp); ++ ++ start_poll_synchronize_rcu_common(); ++} ++EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full); ++ ++/** ++ * poll_state_synchronize_rcu - Has the specified RCU grace period completed? + * @oldstate: value from get_state_synchronize_rcu() or start_poll_synchronize_rcu() + * + * If a full RCU grace period has elapsed since the earlier call from +- * which oldstate was obtained, return @true, otherwise return @false. ++ * which @oldstate was obtained, return @true, otherwise return @false. + * If @false is returned, it is the caller's responsibility to invoke this + * function later on until it does return @true. Alternatively, the caller + * can explicitly wait for a grace period, for example, by passing @oldstate +@@ -3594,10 +3681,11 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for + * more than a billion grace periods (and way more on a 64-bit system!). +- * Those needing to keep oldstate values for very long time periods +- * (many hours even on 32-bit systems) should check them occasionally +- * and either refresh them or set a flag indicating that the grace period +- * has completed. ++ * Those needing to keep old state values for very long time periods ++ * (many hours even on 32-bit systems) should check them occasionally and ++ * either refresh them or set a flag indicating that the grace period has ++ * completed. Alternatively, they can use get_completed_synchronize_rcu() ++ * to get a guaranteed-completed grace-period state. + * + * This function provides the same memory-ordering guarantees that + * would be provided by a synchronize_rcu() that was invoked at the call +@@ -3616,8 +3704,56 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) + EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); + + /** +- * cond_synchronize_rcu - Conditionally wait for an RCU grace period ++ * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed? ++ * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * ++ * If a full RCU grace period has elapsed since the earlier call from ++ * which *rgosp was obtained, return @true, otherwise return @false. ++ * If @false is returned, it is the caller's responsibility to invoke this ++ * function later on until it does return @true. Alternatively, the caller ++ * can explicitly wait for a grace period, for example, by passing @rgosp ++ * to cond_synchronize_rcu() or by directly invoking synchronize_rcu(). ++ * ++ * Yes, this function does not take counter wrap into account. ++ * But counter wrap is harmless. If the counter wraps, we have waited ++ * for more than a billion grace periods (and way more on a 64-bit ++ * system!). Those needing to keep rcu_gp_oldstate values for very ++ * long time periods (many hours even on 32-bit systems) should check ++ * them occasionally and either refresh them or set a flag indicating ++ * that the grace period has completed. Alternatively, they can use ++ * get_completed_synchronize_rcu_full() to get a guaranteed-completed ++ * grace-period state. ++ * ++ * This function provides the same memory-ordering guarantees that would ++ * be provided by a synchronize_rcu() that was invoked at the call to ++ * the function that provided @rgosp, and that returned at the end of this ++ * function. And this guarantee requires that the root rcu_node structure's ++ * ->gp_seq field be checked instead of that of the rcu_state structure. ++ * The problem is that the just-ending grace-period's callbacks can be ++ * invoked between the time that the root rcu_node structure's ->gp_seq ++ * field is updated and the time that the rcu_state structure's ->gp_seq ++ * field is updated. Therefore, if a single synchronize_rcu() is to ++ * cause a subsequent poll_state_synchronize_rcu_full() to return @true, ++ * then the root rcu_node structure is the one that needs to be polled. ++ */ ++bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ struct rcu_node *rnp = rcu_get_root(); ++ ++ smp_mb(); // Order against root rcu_node structure grace-period cleanup. ++ if (rgosp->rgos_norm == RCU_GET_STATE_COMPLETED || ++ rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || ++ rgosp->rgos_exp == RCU_GET_STATE_COMPLETED || ++ rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { ++ smp_mb(); /* Ensure GP ends before subsequent accesses. */ ++ return true; ++ } ++ return false; ++} ++EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu_full); ++ ++/** ++ * cond_synchronize_rcu - Conditionally wait for an RCU grace period + * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() + * + * If a full RCU grace period has elapsed since the earlier call to +@@ -3641,6 +3777,33 @@ void cond_synchronize_rcu(unsigned long oldstate) + } + EXPORT_SYMBOL_GPL(cond_synchronize_rcu); + ++/** ++ * cond_synchronize_rcu_full - Conditionally wait for an RCU grace period ++ * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() ++ * ++ * If a full RCU grace period has elapsed since the call to ++ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), ++ * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was ++ * obtained, just return. Otherwise, invoke synchronize_rcu() to wait ++ * for a full grace period. ++ * ++ * Yes, this function does not take counter wrap into account. ++ * But counter wrap is harmless. If the counter wraps, we have waited for ++ * more than 2 billion grace periods (and way more on a 64-bit system!), ++ * so waiting for a couple of additional grace periods should be just fine. ++ * ++ * This function provides the same memory-ordering guarantees that ++ * would be provided by a synchronize_rcu() that was invoked at the call ++ * to the function that provided @rgosp and that returned at the end of ++ * this function. ++ */ ++void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) ++{ ++ if (!poll_state_synchronize_rcu_full(rgosp)) ++ synchronize_rcu(); ++} ++EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); ++ + /* + * Check to see if there is any immediate RCU-related work to be done by + * the current CPU, returning 1 if so and zero otherwise. The checks are +@@ -4312,9 +4475,20 @@ early_initcall(rcu_spawn_gp_kthread); + */ + void rcu_scheduler_starting(void) + { ++ unsigned long flags; ++ struct rcu_node *rnp; ++ + WARN_ON(num_online_cpus() != 1); + WARN_ON(nr_context_switches() > 0); + rcu_test_sync_prims(); ++ ++ // Fix up the ->gp_seq counters. ++ local_irq_save(flags); ++ rcu_for_each_node_breadth_first(rnp) ++ rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; ++ local_irq_restore(flags); ++ ++ // Switch out of early boot mode. + rcu_scheduler_active = RCU_SCHEDULER_INIT; + rcu_test_sync_prims(); + } +diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h +index be667583a554..18e9b4cd78ef 100644 +--- a/kernel/rcu/tree_exp.h ++++ b/kernel/rcu/tree_exp.h +@@ -828,11 +828,13 @@ static void rcu_exp_handler(void *unused) + { + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + struct rcu_node *rnp = rdp->mynode; ++ bool preempt_bh_enabled = !(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)); + + if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || + __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + return; +- if (rcu_is_cpu_rrupt_from_idle()) { ++ if (rcu_is_cpu_rrupt_from_idle() || ++ (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) { + rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); + return; + } +@@ -906,6 +908,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp) + void synchronize_rcu_expedited(void) + { + bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT); ++ unsigned long flags; + struct rcu_exp_work rew; + struct rcu_node *rnp; + unsigned long s; +@@ -924,8 +927,11 @@ void synchronize_rcu_expedited(void) + // them, which allows reuse of ->gp_seq_polled_exp_snap. + rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); + rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); +- if (rcu_init_invoked()) +- cond_resched(); ++ ++ local_irq_save(flags); ++ WARN_ON_ONCE(num_online_cpus() > 1); ++ rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT); ++ local_irq_restore(flags); + return; // Context allows vacuous grace periods. + } + +@@ -1027,6 +1033,24 @@ unsigned long start_poll_synchronize_rcu_expedited(void) + } + EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); + ++/** ++ * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period ++ * @rgosp: Place to put snapshot of grace-period state ++ * ++ * Places the normal and expedited grace-period states in rgosp. This ++ * state value can be passed to a later call to cond_synchronize_rcu_full() ++ * or poll_state_synchronize_rcu_full() to determine whether or not a ++ * grace period (whether normal or expedited) has elapsed in the meantime. ++ * If the needed expedited grace period is not already slated to start, ++ * initiates that grace period. ++ */ ++void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) ++{ ++ get_state_synchronize_rcu_full(rgosp); ++ (void)start_poll_synchronize_rcu_expedited(); ++} ++EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full); ++ + /** + * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period + * +@@ -1053,3 +1077,30 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate) + synchronize_rcu_expedited(); + } + EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); ++ ++/** ++ * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period ++ * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() ++ * ++ * If a full RCU grace period has elapsed since the call to ++ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), ++ * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was ++ * obtained, just return. Otherwise, invoke synchronize_rcu_expedited() ++ * to wait for a full grace period. ++ * ++ * Yes, this function does not take counter wrap into account. ++ * But counter wrap is harmless. If the counter wraps, we have waited for ++ * more than 2 billion grace periods (and way more on a 64-bit system!), ++ * so waiting for a couple of additional grace periods should be just fine. ++ * ++ * This function provides the same memory-ordering guarantees that ++ * would be provided by a synchronize_rcu() that was invoked at the call ++ * to the function that provided @rgosp and that returned at the end of ++ * this function. ++ */ ++void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) ++{ ++ if (!poll_state_synchronize_rcu_full(rgosp)) ++ synchronize_rcu_expedited(); ++} ++EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full); +diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h +index a8f574d8850d..0a5f0ef41484 100644 +--- a/kernel/rcu/tree_nocb.h ++++ b/kernel/rcu/tree_nocb.h +@@ -1111,7 +1111,7 @@ int rcu_nocb_cpu_deoffload(int cpu) + if (!ret) + cpumask_clear_cpu(cpu, rcu_nocb_mask); + } else { +- pr_info("NOCB: Can't CB-deoffload an offline CPU\n"); ++ pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu); + ret = -EINVAL; + } + } +@@ -1196,7 +1196,7 @@ int rcu_nocb_cpu_offload(int cpu) + if (!ret) + cpumask_set_cpu(cpu, rcu_nocb_mask); + } else { +- pr_info("NOCB: Can't CB-offload an offline CPU\n"); ++ pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu); + ret = -EINVAL; + } + } +@@ -1452,8 +1452,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) + (long)rdp->nocb_gp_seq, + rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), + rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', +- rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, +- show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); ++ rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, ++ show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread)); + } + + /* Dump out nocb kthread state for the specified rcu_data structure. */ +@@ -1497,7 +1497,7 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) + ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], + rcu_segcblist_n_cbs(&rdp->cblist), + rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.', +- rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, ++ rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1, + show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); + + /* It is OK for GP kthreads to have GP state. */ +diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h +index 438ecae6bd7e..e3142ee35fc6 100644 +--- a/kernel/rcu/tree_plugin.h ++++ b/kernel/rcu/tree_plugin.h +@@ -641,7 +641,8 @@ static void rcu_read_unlock_special(struct task_struct *t) + + expboost = (t->rcu_blocked_node && READ_ONCE(t->rcu_blocked_node->exp_tasks)) || + (rdp->grpmask & READ_ONCE(rnp->expmask)) || +- IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) || ++ (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && ++ ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node)) || + (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled && + t->rcu_blocked_node); + // Need to defer quiescent state until everything is enabled. +@@ -718,9 +719,6 @@ static void rcu_flavor_sched_clock_irq(int user) + struct task_struct *t = current; + + lockdep_assert_irqs_disabled(); +- if (user || rcu_is_cpu_rrupt_from_idle()) { +- rcu_note_voluntary_context_switch(current); +- } + if (rcu_preempt_depth() > 0 || + (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) { + /* No QS, force context switch if deferred. */ +@@ -824,6 +822,7 @@ void rcu_read_unlock_strict(void) + if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) + return; + rdp = this_cpu_ptr(&rcu_data); ++ rdp->cpu_no_qs.b.norm = false; + rcu_report_qs_rdp(rdp); + udelay(rcu_unlock_delay); + } +@@ -869,7 +868,7 @@ void rcu_all_qs(void) + + if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + return; +- preempt_disable(); ++ preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels + /* Load rcu_urgent_qs before other flags. */ + if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { + preempt_enable(); +@@ -931,10 +930,13 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) + return false; + } + +-// Except that we do need to respond to a request by an expedited grace +-// period for a quiescent state from this CPU. Note that requests from +-// tasks are handled when removing the task from the blocked-tasks list +-// below. ++// Except that we do need to respond to a request by an expedited ++// grace period for a quiescent state from this CPU. Note that in ++// non-preemptible kernels, there can be no context switches within RCU ++// read-side critical sections, which in turn means that the leaf rcu_node ++// structure's blocked-tasks list is always empty. is therefore no need to ++// actually check it. Instead, a quiescent state from this CPU suffices, ++// and this function is only called from such a quiescent state. + notrace void rcu_preempt_deferred_qs(struct task_struct *t) + { + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); +@@ -972,7 +974,6 @@ static void rcu_flavor_sched_clock_irq(int user) + * neither access nor modify, at least not while the + * corresponding CPU is online. + */ +- + rcu_qs(); + } + } +@@ -1238,8 +1239,11 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) + cpu != outgoingcpu) + cpumask_set_cpu(cpu, cm); + cpumask_and(cm, cm, housekeeping_cpumask(HK_TYPE_RCU)); +- if (cpumask_empty(cm)) ++ if (cpumask_empty(cm)) { + cpumask_copy(cm, housekeeping_cpumask(HK_TYPE_RCU)); ++ if (outgoingcpu >= 0) ++ cpumask_clear_cpu(outgoingcpu, cm); ++ } + set_cpus_allowed_ptr(t, cm); + mutex_unlock(&rnp->boost_kthread_mutex); + free_cpumask_var(cm); +diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h +index c3fbbcc09327..5653560573e2 100644 +--- a/kernel/rcu/tree_stall.h ++++ b/kernel/rcu/tree_stall.h +@@ -368,7 +368,7 @@ static void rcu_dump_cpu_stacks(void) + if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { + if (cpu_is_offline(cpu)) + pr_err("Offline CPU %d blocking current GP.\n", cpu); +- else if (!trigger_single_cpu_backtrace(cpu)) ++ else + dump_cpu_task(cpu); + } + raw_spin_unlock_irqrestore_rcu_node(rnp, flags); +@@ -511,8 +511,7 @@ static void rcu_check_gp_kthread_starvation(void) + pr_err("RCU GP kthread last ran on offline CPU %d.\n", cpu); + } else { + pr_err("Stack dump where RCU GP kthread last ran:\n"); +- if (!trigger_single_cpu_backtrace(cpu)) +- dump_cpu_task(cpu); ++ dump_cpu_task(cpu); + } + } + wake_up_process(gpk); +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index a97ff97891b1..02ec46bf501d 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -74,6 +74,7 @@ + + #include + ++#include + #include + #include + +@@ -11204,6 +11205,19 @@ struct cgroup_subsys cpu_cgrp_subsys = { + + void dump_cpu_task(int cpu) + { ++ if (cpu == smp_processor_id() && in_hardirq()) { ++ struct pt_regs *regs; ++ ++ regs = get_irq_regs(); ++ if (regs) { ++ show_regs(regs); ++ return; ++ } ++ } ++ ++ if (trigger_single_cpu_backtrace(cpu)) ++ return; ++ + pr_info("Task dump for CPU %d:\n", cpu); + sched_show_task(cpu_curr(cpu)); + } +diff --git a/kernel/smp.c b/kernel/smp.c +index 650810a6f29b..e8cdc025a046 100644 +--- a/kernel/smp.c ++++ b/kernel/smp.c +@@ -370,8 +370,7 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 * + if (cpu >= 0) { + if (static_branch_unlikely(&csdlock_debug_extended)) + csd_lock_print_extended(csd, cpu); +- if (!trigger_single_cpu_backtrace(cpu)) +- dump_cpu_task(cpu); ++ dump_cpu_task(cpu); + if (!cpu_cur_csd) { + pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); + arch_send_call_function_single_ipi(cpu); +-- +2.38.0 + +From db7064658d7a9e2fee4d9e82b226c7ea1f0c753e Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Tue, 4 Oct 2022 14:06:15 +0200 +Subject: [PATCH 14/17] clr + +Signed-off-by: Peter Jung +--- + arch/x86/include/asm/topology.h | 1 + + arch/x86/kernel/cpu/intel_epb.c | 4 ++ + arch/x86/kernel/itmt.c | 43 +++++++++++++++++++- + arch/x86/kernel/tsc.c | 3 ++ + arch/x86/mm/fault.c | 4 +- + drivers/cpufreq/intel_pstate.c | 7 ++++ + drivers/idle/intel_idle.c | 50 ++++++++++++------------ + drivers/input/serio/i8042.c | 10 ++--- + drivers/net/dummy.c | 2 +- + drivers/nvme/host/core.c | 2 +- + drivers/pci/pci.c | 2 +- + drivers/powercap/intel_rapl_common.c | 2 +- + drivers/thermal/intel/intel_powerclamp.c | 10 +++++ + fs/xattr.c | 15 +++---- + include/linux/ipc_namespace.h | 5 ++- + include/linux/jbd2.h | 2 +- + include/linux/page_counter.h | 1 + + include/linux/percpu_counter.h | 32 +++++++++++++++ + include/linux/wait.h | 2 + + include/uapi/linux/if_bonding.h | 2 +- + init/do_mounts.c | 16 +++++++- + ipc/msg.c | 44 ++++++++++++++------- + ipc/namespace.c | 5 ++- + ipc/util.h | 4 +- + kernel/locking/rwsem.c | 4 +- + kernel/sched/fair.c | 19 +++++++++ + kernel/sched/wait.c | 24 ++++++++++++ + kernel/watchdog.c | 2 +- + lib/raid6/algos.c | 4 +- + mm/ksm.c | 11 ++++-- + mm/memcontrol.c | 2 +- + mm/page_alloc.c | 5 ++- + net/ipv4/inet_connection_sock.c | 2 +- + net/ipv4/tcp.c | 4 +- + 34 files changed, 266 insertions(+), 79 deletions(-) + +diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h +index 458c891a8273..d86eb1ebf59f 100644 +--- a/arch/x86/include/asm/topology.h ++++ b/arch/x86/include/asm/topology.h +@@ -175,6 +175,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled; + + /* Interface to set priority of a cpu */ + void sched_set_itmt_core_prio(int prio, int core_cpu); ++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu); + + /* Interface to notify scheduler that system supports ITMT */ + int sched_set_itmt_support(void); +diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c +index fbaf12e43f41..c8c2d6f1a8ac 100644 +--- a/arch/x86/kernel/cpu/intel_epb.c ++++ b/arch/x86/kernel/cpu/intel_epb.c +@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev, + if (ret < 0) + return ret; + ++ /* update the ITMT scheduler logic to use the power policy data */ ++ /* scale the val up by 2 so the range is 224 - 256 */ ++ sched_set_itmt_power_ratio(256 - val * 2, cpu); ++ + return count; + } + +diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c +index 9ff480e94511..596fd7fb7847 100644 +--- a/arch/x86/kernel/itmt.c ++++ b/arch/x86/kernel/itmt.c +@@ -25,6 +25,7 @@ + + static DEFINE_MUTEX(itmt_update_mutex); + DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); ++DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio); + + /* Boolean to track if system has ITMT capabilities */ + static bool __read_mostly sched_itmt_capable; +@@ -169,9 +170,19 @@ void sched_clear_itmt_support(void) + + int arch_asym_cpu_priority(int cpu) + { +- return per_cpu(sched_core_priority, cpu); ++ int power_ratio = per_cpu(sched_power_ratio, cpu); ++ ++ /* a power ratio of 0 (uninitialized) is assumed to be maximum */ ++ if (power_ratio == 0) ++ power_ratio = 256 - 2 * 6; ++ return per_cpu(sched_core_priority, cpu) * power_ratio / 256; + } + ++extern int best_core; ++extern int second_best_core; ++static int best_core_score; ++static int second_best_core_score; ++ + /** + * sched_set_itmt_core_prio() - Set CPU priority based on ITMT + * @prio: Priority of cpu core +@@ -201,5 +212,35 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) + smt_prio = prio * smp_num_siblings / (i * i); + per_cpu(sched_core_priority, cpu) = smt_prio; + i++; ++ ++ if (smt_prio > best_core_score) { ++ best_core = cpu; ++ best_core_score = smt_prio; ++ } else ++ if (smt_prio > second_best_core_score) { ++ second_best_core = cpu; ++ second_best_core_score = smt_prio; ++ } ++ } ++} ++ ++/** ++ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT ++ * @power_ratio: The power scaling ratio [1..256] for the core ++ * @core_cpu: The cpu number associated with the core ++ * ++ * Set a scaling to the cpu performance based on long term power ++ * settings (like EPB). ++ * ++ * Note this is for the policy not for the actual dynamic frequency; ++ * the frequency will increase itself as workloads run on a core. ++ */ ++ ++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu) ++{ ++ int cpu; ++ ++ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) { ++ per_cpu(sched_power_ratio, cpu) = power_ratio; + } + } +diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c +index cafacb2e58cc..c2f80184fd33 100644 +--- a/arch/x86/kernel/tsc.c ++++ b/arch/x86/kernel/tsc.c +@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void) + if (!constant_tsc || !mask) + return 0; + ++ if (cpu != 0) ++ return cpu_data(0).loops_per_jiffy; ++ + sibling = cpumask_any_but(mask, cpu); + if (sibling < nr_cpu_ids) + return cpu_data(sibling).loops_per_jiffy; +diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c +index fa71a5d12e87..c4bbd8c66134 100644 +--- a/arch/x86/mm/fault.c ++++ b/arch/x86/mm/fault.c +@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, + if (!printk_ratelimit()) + return; + +- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx", ++ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i", + loglvl, tsk->comm, task_pid_nr(tsk), address, +- (void *)regs->ip, (void *)regs->sp, error_code); ++ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id()); + + print_vma_addr(KERN_CONT " in ", regs->ip); + +diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c +index fc3ebeb0bbe5..40942ad7a749 100644 +--- a/drivers/cpufreq/intel_pstate.c ++++ b/drivers/cpufreq/intel_pstate.c +@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu) + * update them at any time after it has been called. + */ + sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); ++ /* ++ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff. ++ * In this case we can't use CPPC.highest_perf to enable ITMT. ++ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide. ++ */ ++ if (cppc_perf.highest_perf == 0xff) ++ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); + + if (max_highest_perf <= min_highest_perf) { + if (cppc_perf.highest_perf > max_highest_perf) +diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c +index 3e101719689a..82fc489a6fbe 100644 +--- a/drivers/idle/intel_idle.c ++++ b/drivers/idle/intel_idle.c +@@ -578,7 +578,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 120, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -586,7 +586,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 33, +- .target_residency = 100, ++ .target_residency = 900, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -594,7 +594,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 133, +- .target_residency = 400, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -602,7 +602,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x32", + .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 166, +- .target_residency = 500, ++ .target_residency = 1500, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -610,7 +610,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 300, +- .target_residency = 900, ++ .target_residency = 2000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -618,7 +618,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 600, +- .target_residency = 1800, ++ .target_residency = 5000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -626,7 +626,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 2600, +- .target_residency = 7700, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -646,7 +646,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 120, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -654,7 +654,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 40, +- .target_residency = 100, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -662,7 +662,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 133, +- .target_residency = 400, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -670,7 +670,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x32", + .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 166, +- .target_residency = 500, ++ .target_residency = 2000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -678,7 +678,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 300, +- .target_residency = 900, ++ .target_residency = 4000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -686,7 +686,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 600, +- .target_residency = 1800, ++ .target_residency = 7000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -694,7 +694,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 2600, +- .target_residency = 7700, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -715,7 +715,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 120, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -723,7 +723,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 70, +- .target_residency = 100, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -731,7 +731,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 85, +- .target_residency = 200, ++ .target_residency = 600, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -739,7 +739,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x33", + .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 124, +- .target_residency = 800, ++ .target_residency = 3000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -747,7 +747,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 200, +- .target_residency = 800, ++ .target_residency = 3200, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -755,7 +755,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 480, +- .target_residency = 5000, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -763,7 +763,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 890, +- .target_residency = 5000, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -784,7 +784,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 300, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -813,7 +813,7 @@ static struct cpuidle_state icx_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 4, +- .target_residency = 4, ++ .target_residency = 40, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -821,7 +821,7 @@ static struct cpuidle_state icx_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 170, +- .target_residency = 600, ++ .target_residency = 900, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -942,7 +942,7 @@ static struct cpuidle_state spr_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 2, +- .target_residency = 4, ++ .target_residency = 40, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c +index 3fc0a89cc785..a7c103f9dfd3 100644 +--- a/drivers/input/serio/i8042.c ++++ b/drivers/input/serio/i8042.c +@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void) + if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { + i8042_ctr &= ~I8042_CTR_KBDINT; + i8042_ctr |= I8042_CTR_KBDDIS; +- pr_err("Failed to enable KBD port\n"); ++ pr_info("Failed to enable KBD port\n"); + return -EIO; + } + +@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void) + if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { + i8042_ctr &= ~I8042_CTR_AUXINT; + i8042_ctr |= I8042_CTR_AUXDIS; +- pr_err("Failed to enable AUX port\n"); ++ pr_info("Failed to enable AUX port\n"); + return -EIO; + } + +@@ -732,7 +732,7 @@ static int i8042_check_mux(void) + i8042_ctr &= ~I8042_CTR_AUXINT; + + if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { +- pr_err("Failed to disable AUX port, can't use MUX\n"); ++ pr_info("Failed to disable AUX port, can't use MUX\n"); + return -EIO; + } + +@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void) + do { + + if (i8042_command(¶m, I8042_CMD_CTL_TEST)) { +- pr_err("i8042 controller selftest timeout\n"); ++ pr_info("i8042 controller selftest timeout\n"); + return -ENODEV; + } + +@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void) + pr_info("giving up on controller selftest, continuing anyway...\n"); + return 0; + #else +- pr_err("i8042 controller selftest failed\n"); ++ pr_info("i8042 controller selftest failed\n"); + return -EIO; + #endif + } +diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c +index f82ad7419508..5e8faa70aad6 100644 +--- a/drivers/net/dummy.c ++++ b/drivers/net/dummy.c +@@ -43,7 +43,7 @@ + + #define DRV_NAME "dummy" + +-static int numdummies = 1; ++static int numdummies = 0; + + /* fake multicast ability */ + static void set_multicast_list(struct net_device *dev) +diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c +index 8d5a7ae19844..56d1780d1337 100644 +--- a/drivers/nvme/host/core.c ++++ b/drivers/nvme/host/core.c +@@ -58,7 +58,7 @@ static u8 nvme_max_retries = 5; + module_param_named(max_retries, nvme_max_retries, byte, 0644); + MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); + +-static unsigned long default_ps_max_latency_us = 100000; ++static unsigned long default_ps_max_latency_us = 200; + module_param(default_ps_max_latency_us, ulong, 0644); + MODULE_PARM_DESC(default_ps_max_latency_us, + "max power saving latency for new devices; use PM QOS to change per device"); +diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c +index 95bc329e74c0..73114509ddff 100644 +--- a/drivers/pci/pci.c ++++ b/drivers/pci/pci.c +@@ -62,7 +62,7 @@ struct pci_pme_device { + struct pci_dev *dev; + }; + +-#define PME_TIMEOUT 1000 /* How long between PME checks */ ++#define PME_TIMEOUT 4000 /* How long between PME checks */ + + static void pci_dev_d3_sleep(struct pci_dev *dev) + { +diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c +index 21d624f9f5fb..aad4e9578d89 100644 +--- a/drivers/powercap/intel_rapl_common.c ++++ b/drivers/powercap/intel_rapl_common.c +@@ -1515,7 +1515,7 @@ static int __init rapl_init(void) + + id = x86_match_cpu(rapl_ids); + if (!id) { +- pr_err("driver does not support CPU family %d model %d\n", ++ pr_info("driver does not support CPU family %d model %d\n", + boot_cpu_data.x86, boot_cpu_data.x86_model); + + return -ENODEV; +diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c +index c841ab37e7c6..fe3d190ab12b 100644 +--- a/drivers/thermal/intel/intel_powerclamp.c ++++ b/drivers/thermal/intel/intel_powerclamp.c +@@ -644,6 +644,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = { + .set_cur_state = powerclamp_set_cur_state, + }; + ++static const struct x86_cpu_id amd_cpu[] = { ++ { X86_VENDOR_AMD }, ++ {}, ++}; ++ + static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = { + X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL), + {} +@@ -653,6 +658,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); + static int __init powerclamp_probe(void) + { + ++ if (x86_match_cpu(amd_cpu)){ ++ pr_info("Intel PowerClamp does not support AMD CPUs\n"); ++ return -ENODEV; ++ } ++ + if (!x86_match_cpu(intel_powerclamp_ids)) { + pr_err("CPU does not support MWAIT\n"); + return -ENODEV; +diff --git a/fs/xattr.c b/fs/xattr.c +index a1f4998bc6be..020e0b5ea5c6 100644 +--- a/fs/xattr.c ++++ b/fs/xattr.c +@@ -122,16 +122,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, + } + + /* +- * In the user.* namespace, only regular files and directories can have +- * extended attributes. For sticky directories, only the owner and +- * privileged users can write attributes. ++ * In the user.* namespace, only regular files, symbolic links, and ++ * directories can have extended attributes. For symbolic links and ++ * sticky directories, only the owner and privileged users can write ++ * attributes. + */ + if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { +- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) ++ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode)) + return (mask & MAY_WRITE) ? -EPERM : -ENODATA; +- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && +- (mask & MAY_WRITE) && +- !inode_owner_or_capable(mnt_userns, inode)) ++ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX)) ++ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE) ++ && !inode_owner_or_capable(mnt_userns, inode)) + return -EPERM; + } + +diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h +index e3e8c8662b49..e8240cf2611a 100644 +--- a/include/linux/ipc_namespace.h ++++ b/include/linux/ipc_namespace.h +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + + struct user_namespace; + +@@ -36,8 +37,8 @@ struct ipc_namespace { + unsigned int msg_ctlmax; + unsigned int msg_ctlmnb; + unsigned int msg_ctlmni; +- atomic_t msg_bytes; +- atomic_t msg_hdrs; ++ struct percpu_counter percpu_msg_bytes; ++ struct percpu_counter percpu_msg_hdrs; + + size_t shm_ctlmax; + size_t shm_ctlall; +diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h +index 0b7242370b56..16b8fc483b3d 100644 +--- a/include/linux/jbd2.h ++++ b/include/linux/jbd2.h +@@ -45,7 +45,7 @@ + /* + * The default maximum commit age, in seconds. + */ +-#define JBD2_DEFAULT_MAX_COMMIT_AGE 5 ++#define JBD2_DEFAULT_MAX_COMMIT_AGE 30 + + #ifdef CONFIG_JBD2_DEBUG + /* +diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h +index 679591301994..685193470d64 100644 +--- a/include/linux/page_counter.h ++++ b/include/linux/page_counter.h +@@ -8,6 +8,7 @@ + + struct page_counter { + atomic_long_t usage; ++ unsigned long padding[7]; + unsigned long min; + unsigned long low; + unsigned long high; +diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h +index 01861eebed79..8ed5fba6d156 100644 +--- a/include/linux/percpu_counter.h ++++ b/include/linux/percpu_counter.h +@@ -15,6 +15,9 @@ + #include + #include + ++/* percpu_counter batch for local add or sub */ ++#define PERCPU_COUNTER_LOCAL_BATCH INT_MAX ++ + #ifdef CONFIG_SMP + + struct percpu_counter { +@@ -56,6 +59,22 @@ static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount) + percpu_counter_add_batch(fbc, amount, percpu_counter_batch); + } + ++/* ++ * With percpu_counter_add_local() and percpu_counter_sub_local(), counts ++ * are accumulated in local per cpu counter and not in fbc->count until ++ * local count overflows PERCPU_COUNTER_LOCAL_BATCH. This makes counter ++ * write efficient. ++ * But percpu_counter_sum(), instead of percpu_counter_read(), needs to be ++ * used to add up the counts from each CPU to account for all the local ++ * counts. So percpu_counter_add_local() and percpu_counter_sub_local() ++ * should be used when a counter is updated frequently and read rarely. ++ */ ++static inline void ++percpu_counter_add_local(struct percpu_counter *fbc, s64 amount) ++{ ++ percpu_counter_add_batch(fbc, amount, PERCPU_COUNTER_LOCAL_BATCH); ++} ++ + static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) + { + s64 ret = __percpu_counter_sum(fbc); +@@ -138,6 +157,13 @@ percpu_counter_add(struct percpu_counter *fbc, s64 amount) + preempt_enable(); + } + ++/* non-SMP percpu_counter_add_local is the same with percpu_counter_add */ ++static inline void ++percpu_counter_add_local(struct percpu_counter *fbc, s64 amount) ++{ ++ percpu_counter_add(fbc, amount); ++} ++ + static inline void + percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) + { +@@ -193,4 +219,10 @@ static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount) + percpu_counter_add(fbc, -amount); + } + ++static inline void ++percpu_counter_sub_local(struct percpu_counter *fbc, s64 amount) ++{ ++ percpu_counter_add_local(fbc, -amount); ++} ++ + #endif /* _LINUX_PERCPU_COUNTER_H */ +diff --git a/include/linux/wait.h b/include/linux/wait.h +index 58cfbf81447c..d5664b5ae3e1 100644 +--- a/include/linux/wait.h ++++ b/include/linux/wait.h +@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) + + extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); ++extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + +@@ -1164,6 +1165,7 @@ do { \ + */ + void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); + bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); ++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); + long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); + void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); +diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h +index d174914a837d..bf8e2af101a3 100644 +--- a/include/uapi/linux/if_bonding.h ++++ b/include/uapi/linux/if_bonding.h +@@ -82,7 +82,7 @@ + #define BOND_STATE_ACTIVE 0 /* link is active */ + #define BOND_STATE_BACKUP 1 /* link is backup */ + +-#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */ ++#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */ + + #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */ + +diff --git a/init/do_mounts.c b/init/do_mounts.c +index 7058e14ad5f7..ce4e841ea705 100644 +--- a/init/do_mounts.c ++++ b/init/do_mounts.c +@@ -283,8 +283,18 @@ dev_t name_to_dev_t(const char *name) + if (strcmp(name, "/dev/ram") == 0) + return Root_RAM0; + #ifdef CONFIG_BLOCK +- if (strncmp(name, "PARTUUID=", 9) == 0) +- return devt_from_partuuid(name + 9); ++ if (strncmp(name, "PARTUUID=", 9) == 0) { ++ dev_t res; ++ int needtowait = 40<<1; ++ res = devt_from_partuuid(name + 9); ++ while (!res && needtowait) { ++ /* waiting 0.5 sec */ ++ msleep(500); ++ res = devt_from_partuuid(name + 9); ++ needtowait--; ++ } ++ return res; ++ } + if (strncmp(name, "PARTLABEL=", 10) == 0) + return devt_from_partlabel(name + 10); + if (strncmp(name, "/dev/", 5) == 0) +@@ -612,7 +622,9 @@ void __init prepare_namespace(void) + * For example, it is not atypical to wait 5 seconds here + * for the touchpad of a laptop to initialize. + */ ++ async_synchronize_full(); + wait_for_device_probe(); ++ async_synchronize_full(); + + md_run_setup(); + +diff --git a/ipc/msg.c b/ipc/msg.c +index a0d05775af2c..f2bb4c193ecf 100644 +--- a/ipc/msg.c ++++ b/ipc/msg.c +@@ -39,6 +39,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -285,10 +286,10 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) + rcu_read_unlock(); + + list_for_each_entry_safe(msg, t, &msq->q_messages, m_list) { +- atomic_dec(&ns->msg_hdrs); ++ percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1); + free_msg(msg); + } +- atomic_sub(msq->q_cbytes, &ns->msg_bytes); ++ percpu_counter_sub_local(&ns->percpu_msg_bytes, msq->q_cbytes); + ipc_update_pid(&msq->q_lspid, NULL); + ipc_update_pid(&msq->q_lrpid, NULL); + ipc_rcu_putref(&msq->q_perm, msg_rcu_free); +@@ -495,17 +496,18 @@ static int msgctl_info(struct ipc_namespace *ns, int msqid, + msginfo->msgssz = MSGSSZ; + msginfo->msgseg = MSGSEG; + down_read(&msg_ids(ns).rwsem); +- if (cmd == MSG_INFO) { ++ if (cmd == MSG_INFO) + msginfo->msgpool = msg_ids(ns).in_use; +- msginfo->msgmap = atomic_read(&ns->msg_hdrs); +- msginfo->msgtql = atomic_read(&ns->msg_bytes); ++ max_idx = ipc_get_maxidx(&msg_ids(ns)); ++ up_read(&msg_ids(ns).rwsem); ++ if (cmd == MSG_INFO) { ++ msginfo->msgmap = percpu_counter_sum(&ns->percpu_msg_hdrs); ++ msginfo->msgtql = percpu_counter_sum(&ns->percpu_msg_bytes); + } else { + msginfo->msgmap = MSGMAP; + msginfo->msgpool = MSGPOOL; + msginfo->msgtql = MSGTQL; + } +- max_idx = ipc_get_maxidx(&msg_ids(ns)); +- up_read(&msg_ids(ns).rwsem); + return (max_idx < 0) ? 0 : max_idx; + } + +@@ -935,8 +937,8 @@ static long do_msgsnd(int msqid, long mtype, void __user *mtext, + list_add_tail(&msg->m_list, &msq->q_messages); + msq->q_cbytes += msgsz; + msq->q_qnum++; +- atomic_add(msgsz, &ns->msg_bytes); +- atomic_inc(&ns->msg_hdrs); ++ percpu_counter_add_local(&ns->percpu_msg_bytes, msgsz); ++ percpu_counter_add_local(&ns->percpu_msg_hdrs, 1); + } + + err = 0; +@@ -1159,8 +1161,8 @@ static long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, in + msq->q_rtime = ktime_get_real_seconds(); + ipc_update_pid(&msq->q_lrpid, task_tgid(current)); + msq->q_cbytes -= msg->m_ts; +- atomic_sub(msg->m_ts, &ns->msg_bytes); +- atomic_dec(&ns->msg_hdrs); ++ percpu_counter_sub_local(&ns->percpu_msg_bytes, msg->m_ts); ++ percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1); + ss_wakeup(msq, &wake_q, false); + + goto out_unlock0; +@@ -1297,20 +1299,34 @@ COMPAT_SYSCALL_DEFINE5(msgrcv, int, msqid, compat_uptr_t, msgp, + } + #endif + +-void msg_init_ns(struct ipc_namespace *ns) ++int msg_init_ns(struct ipc_namespace *ns) + { ++ int ret; ++ + ns->msg_ctlmax = MSGMAX; + ns->msg_ctlmnb = MSGMNB; + ns->msg_ctlmni = MSGMNI; + +- atomic_set(&ns->msg_bytes, 0); +- atomic_set(&ns->msg_hdrs, 0); ++ ret = percpu_counter_init(&ns->percpu_msg_bytes, 0, GFP_KERNEL); ++ if (ret) ++ goto fail_msg_bytes; ++ ret = percpu_counter_init(&ns->percpu_msg_hdrs, 0, GFP_KERNEL); ++ if (ret) ++ goto fail_msg_hdrs; + ipc_init_ids(&ns->ids[IPC_MSG_IDS]); ++ return 0; ++ ++ fail_msg_hdrs: ++ percpu_counter_destroy(&ns->percpu_msg_bytes); ++ fail_msg_bytes: ++ return ret; + } + + #ifdef CONFIG_IPC_NS + void msg_exit_ns(struct ipc_namespace *ns) + { ++ percpu_counter_destroy(&ns->percpu_msg_bytes); ++ percpu_counter_destroy(&ns->percpu_msg_hdrs); + free_ipcs(ns, &msg_ids(ns), freeque); + idr_destroy(&ns->ids[IPC_MSG_IDS].ipcs_idr); + rhashtable_destroy(&ns->ids[IPC_MSG_IDS].key_ht); +diff --git a/ipc/namespace.c b/ipc/namespace.c +index e1fcaedba4fa..8316ea585733 100644 +--- a/ipc/namespace.c ++++ b/ipc/namespace.c +@@ -66,8 +66,11 @@ static struct ipc_namespace *create_ipc_ns(struct user_namespace *user_ns, + if (!setup_ipc_sysctls(ns)) + goto fail_mq; + ++ err = msg_init_ns(ns); ++ if (err) ++ goto fail_put; ++ + sem_init_ns(ns); +- msg_init_ns(ns); + shm_init_ns(ns); + + return ns; +diff --git a/ipc/util.h b/ipc/util.h +index 2dd7ce0416d8..1b0086c6346f 100644 +--- a/ipc/util.h ++++ b/ipc/util.h +@@ -64,7 +64,7 @@ static inline void mq_put_mnt(struct ipc_namespace *ns) { } + + #ifdef CONFIG_SYSVIPC + void sem_init_ns(struct ipc_namespace *ns); +-void msg_init_ns(struct ipc_namespace *ns); ++int msg_init_ns(struct ipc_namespace *ns); + void shm_init_ns(struct ipc_namespace *ns); + + void sem_exit_ns(struct ipc_namespace *ns); +@@ -72,7 +72,7 @@ void msg_exit_ns(struct ipc_namespace *ns); + void shm_exit_ns(struct ipc_namespace *ns); + #else + static inline void sem_init_ns(struct ipc_namespace *ns) { } +-static inline void msg_init_ns(struct ipc_namespace *ns) { } ++static inline int msg_init_ns(struct ipc_namespace *ns) { return 0;} + static inline void shm_init_ns(struct ipc_namespace *ns) { } + + static inline void sem_exit_ns(struct ipc_namespace *ns) { } +diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c +index 65f0262f635e..8a8f77930818 100644 +--- a/kernel/locking/rwsem.c ++++ b/kernel/locking/rwsem.c +@@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) + struct task_struct *new, *owner; + unsigned long flags, new_flags; + enum owner_state state; ++ int i = 0; + + lockdep_assert_preemption_disabled(); + +@@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) + break; + } + +- cpu_relax(); ++ if (i++ > 1000) ++ cpu_relax(); + } + + return state; +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index f7899278e979..e25a1e31ebb3 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -7039,6 +7039,10 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) + * + * Returns the target CPU number. + */ ++ ++int best_core = -1; ++int second_best_core = -1; ++ + static int + select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) + { +@@ -7067,6 +7071,21 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) + want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); + } + ++ if (prev_cpu != best_core && prev_cpu != second_best_core && ++ cpu_rq(prev_cpu)->nr_running != 0) { ++ if (second_best_core != -1 && cpu_rq(second_best_core)->nr_running == 0 && ++ nr_iowait_cpu(second_best_core) < 2 && cpu_to_node(prev_cpu) == cpu_to_node(second_best_core)) ++ prev_cpu = second_best_core; ++ if (best_core != -1 && cpu_rq(best_core)->nr_running == 0 && ++ nr_iowait_cpu(best_core) < 2 && cpu_to_node(prev_cpu) == cpu_to_node(best_core)) ++ prev_cpu = best_core; ++ } ++/* ++ if (prev_cpu > 0 && cpu_rq(prev_cpu)->nr_running != 0 && cpu_rq(prev_cpu - 1)->nr_running == 0) ++ prev_cpu = prev_cpu - 1; ++*/ ++ ++ + rcu_read_lock(); + for_each_domain(cpu, tmp) { + /* +diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c +index 9860bb9a847c..c7f045873a5f 100644 +--- a/kernel/sched/wait.c ++++ b/kernel/sched/wait.c +@@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ + } + EXPORT_SYMBOL_GPL(add_wait_queue_priority); + ++void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) ++{ ++ unsigned long flags; ++ ++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; ++ spin_lock_irqsave(&wq_head->lock, flags); ++ __add_wait_queue(wq_head, wq_entry); ++ spin_unlock_irqrestore(&wq_head->lock, flags); ++} ++EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); ++ + void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) + { + unsigned long flags; +@@ -289,6 +300,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent + } + EXPORT_SYMBOL(prepare_to_wait_exclusive); + ++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) ++{ ++ unsigned long flags; ++ ++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; ++ spin_lock_irqsave(&wq_head->lock, flags); ++ if (list_empty(&wq_entry->entry)) ++ __add_wait_queue(wq_head, wq_entry); ++ set_current_state(state); ++ spin_unlock_irqrestore(&wq_head->lock, flags); ++} ++EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); ++ + void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) + { + wq_entry->flags = flags; +diff --git a/kernel/watchdog.c b/kernel/watchdog.c +index 8e61f21e7e33..be1439d38f26 100644 +--- a/kernel/watchdog.c ++++ b/kernel/watchdog.c +@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled; + int __read_mostly watchdog_user_enabled = 1; + int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT; + int __read_mostly soft_watchdog_user_enabled = 1; +-int __read_mostly watchdog_thresh = 10; ++int __read_mostly watchdog_thresh = 40; + static int __read_mostly nmi_watchdog_available; + + struct cpumask watchdog_cpumask __read_mostly; +diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c +index 39b74221f4a7..ec3eab8cd6b1 100644 +--- a/lib/raid6/algos.c ++++ b/lib/raid6/algos.c +@@ -128,8 +128,10 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) + + for (best = NULL, algo = raid6_recov_algos; *algo; algo++) + if (!best || (*algo)->priority > best->priority) +- if (!(*algo)->valid || (*algo)->valid()) ++ if (!(*algo)->valid || (*algo)->valid()) { + best = *algo; ++ break; ++ } + + if (best) { + raid6_2data_recov = best->data2; +diff --git a/mm/ksm.c b/mm/ksm.c +index 63b4b9d71597..123b74d0b94f 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -2431,9 +2431,14 @@ static int ksm_scan_thread(void *nothing) + + if (ksmd_should_run()) { + sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs); +- wait_event_interruptible_timeout(ksm_iter_wait, +- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), +- msecs_to_jiffies(sleep_ms)); ++ if (sleep_ms >= 1000) ++ wait_event_interruptible_timeout(ksm_iter_wait, ++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), ++ msecs_to_jiffies(round_jiffies_relative(sleep_ms))); ++ else ++ wait_event_interruptible_timeout(ksm_iter_wait, ++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), ++ msecs_to_jiffies(sleep_ms)); + } else { + wait_event_freezable(ksm_thread_wait, + ksmd_should_run() || kthread_should_stop()); +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index fa9714d9afa9..454a283b9e85 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -625,7 +625,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) + cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); + + x = __this_cpu_add_return(stats_updates, abs(val)); +- if (x > MEMCG_CHARGE_BATCH) { ++ if (x > MEMCG_CHARGE_BATCH * 128) { + /* + * If stats_flush_threshold exceeds the threshold + * (>num_online_cpus()), cgroup stats update will be triggered +diff --git a/mm/page_alloc.c b/mm/page_alloc.c +index d04211f0ef0b..cc6179d3a7dc 100644 +--- a/mm/page_alloc.c ++++ b/mm/page_alloc.c +@@ -7027,11 +7027,11 @@ static int zone_batchsize(struct zone *zone) + + /* + * The number of pages to batch allocate is either ~0.1% +- * of the zone or 1MB, whichever is smaller. The batch ++ * of the zone or 4MB, whichever is smaller. The batch + * size is striking a balance between allocation latency + * and zone lock contention. + */ +- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); ++ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); + batch /= 4; /* We effectively *= 4 below */ + if (batch < 1) + batch = 1; +@@ -7109,6 +7109,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) + * historical relationship between high and batch. + */ + high = max(high, batch << 2); ++ high = max(high, 1024); + + return high; + #else +diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c +index eb31c7158b39..8cefa0b62fb9 100644 +--- a/net/ipv4/inet_connection_sock.c ++++ b/net/ipv4/inet_connection_sock.c +@@ -445,7 +445,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) + * having to remove and re-insert us on the wait queue. + */ + for (;;) { +- prepare_to_wait_exclusive(sk_sleep(sk), &wait, ++ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, + TASK_INTERRUPTIBLE); + release_sock(sk); + if (reqsk_queue_empty(&icsk->icsk_accept_queue)) +diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c +index b735776c0225..2fcd440df1f6 100644 +--- a/net/ipv4/tcp.c ++++ b/net/ipv4/tcp.c +@@ -4790,8 +4790,8 @@ void __init tcp_init(void) + tcp_init_mem(); + /* Set per-socket limits to no more than 1/128 the pressure threshold */ + limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); +- max_wshare = min(4UL*1024*1024, limit); +- max_rshare = min(6UL*1024*1024, limit); ++ max_wshare = min(16UL*1024*1024, limit); ++ max_rshare = min(16UL*1024*1024, limit); + + init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; + init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; +-- +2.38.0 + +From bbb7e25acdde71641e4a5e82732fbd84e5fc7ade Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 10 Oct 2022 13:17:31 +0200 +Subject: [PATCH 15/17] fs-patches + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/ext4.rst | 3 - + Documentation/filesystems/vfs.rst | 3 + + fs/btrfs/backref.c | 155 +++- + fs/btrfs/backref.h | 20 +- + fs/btrfs/block-group.c | 2 +- + fs/btrfs/ctree.c | 48 +- + fs/btrfs/ctree.h | 116 ++- + fs/btrfs/delalloc-space.c | 13 +- + fs/btrfs/delalloc-space.h | 3 +- + fs/btrfs/delayed-inode.c | 329 ++++--- + fs/btrfs/delayed-inode.h | 36 +- + fs/btrfs/disk-io.c | 13 + + fs/btrfs/extent-tree.c | 15 +- + fs/btrfs/extent_io.c | 706 ++++++++++----- + fs/btrfs/file-item.c | 4 +- + fs/btrfs/file.c | 564 ++++++++++-- + fs/btrfs/free-space-cache.c | 10 + + fs/btrfs/inode.c | 181 +--- + fs/btrfs/locking.c | 23 + + fs/btrfs/locking.h | 1 + + fs/btrfs/ordered-data.c | 49 + + fs/btrfs/ordered-data.h | 1 + + fs/btrfs/relocation.c | 2 +- + fs/btrfs/scrub.c | 4 +- + fs/btrfs/space-info.c | 3 +- + fs/btrfs/transaction.c | 105 ++- + fs/btrfs/tree-log.c | 1341 +++++++++++++++++++--------- + fs/btrfs/tree-log.h | 6 + + fs/buffer.c | 23 +- + fs/ext4/ext4.h | 10 +- + fs/ext4/extents.c | 107 +-- + fs/ext4/extents_status.c | 3 +- + fs/ext4/fast_commit.c | 210 +++-- + fs/ext4/fast_commit.h | 3 + + fs/ext4/file.c | 6 + + fs/ext4/inode.c | 17 +- + fs/ext4/ioctl.c | 7 +- + fs/ext4/mballoc.c | 429 +++++---- + fs/ext4/mballoc.h | 17 +- + fs/ext4/migrate.c | 3 +- + fs/ext4/move_extent.c | 26 +- + fs/ext4/namei.c | 17 +- + fs/ext4/resize.c | 2 +- + fs/ext4/super.c | 1257 ++++++++++++++------------ + fs/ext4/sysfs.c | 2 - + fs/ext4/verity.c | 6 +- + fs/ext4/xattr.c | 1 + + fs/fs-writeback.c | 37 +- + fs/iomap/buffered-io.c | 3 +- + fs/iomap/trace.h | 1 + + fs/jbd2/commit.c | 12 +- + fs/jbd2/journal.c | 19 +- + fs/jbd2/recovery.c | 1 + + fs/jbd2/transaction.c | 6 +- + fs/mbcache.c | 17 +- + fs/ntfs/file.c | 4 +- + fs/xfs/libxfs/xfs_bmap.c | 2 +- + fs/xfs/libxfs/xfs_da_btree.c | 2 +- + fs/xfs/libxfs/xfs_dir2.c | 50 +- + fs/xfs/libxfs/xfs_dir2.h | 4 +- + fs/xfs/libxfs/xfs_dir2_sf.c | 4 +- + fs/xfs/libxfs/xfs_inode_fork.c | 4 +- + fs/xfs/scrub/dir.c | 2 +- + fs/xfs/xfs_attr_item.c | 6 - + fs/xfs/xfs_dir2_readdir.c | 2 +- + fs/xfs/xfs_inode.c | 13 +- + fs/xfs/xfs_inode_item.c | 2 +- + fs/xfs/xfs_inode_item_recover.c | 4 +- + fs/xfs/xfs_iops.c | 6 +- + fs/xfs/xfs_iops.h | 1 - + fs/xfs/xfs_itable.c | 8 +- + fs/xfs/xfs_log.c | 10 +- + fs/xfs/xfs_mount.c | 38 +- + fs/xfs/xfs_notify_failure.c | 26 +- + fs/xfs/xfs_reflink.c | 22 +- + fs/xfs/xfs_stats.c | 4 +- + fs/xfs/xfs_super.c | 10 +- + fs/xfs/xfs_trace.h | 4 +- + include/linux/buffer_head.h | 2 +- + include/linux/fs.h | 9 +- + mm/page-writeback.c | 1 + + 81 files changed, 4131 insertions(+), 2107 deletions(-) + +diff --git a/Documentation/admin-guide/ext4.rst b/Documentation/admin-guide/ext4.rst +index 4c559e08d11e..5740d85439ff 100644 +--- a/Documentation/admin-guide/ext4.rst ++++ b/Documentation/admin-guide/ext4.rst +@@ -489,9 +489,6 @@ Files in /sys/fs/ext4/: + multiple of this tuning parameter if the stripe size is not set in the + ext4 superblock + +- mb_max_inode_prealloc +- The maximum length of per-inode ext4_prealloc_space list. +- + mb_max_to_scan + The maximum number of extents the multiblock allocator will search to + find the best extent. +diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst +index 6cd6953e175b..b2ef2449aed9 100644 +--- a/Documentation/filesystems/vfs.rst ++++ b/Documentation/filesystems/vfs.rst +@@ -274,6 +274,9 @@ or bottom half). + This is specifically for the inode itself being marked dirty, + not its data. If the update needs to be persisted by fdatasync(), + then I_DIRTY_DATASYNC will be set in the flags argument. ++ I_DIRTY_TIME will be set in the flags in case lazytime is enabled ++ and struct inode has times updated since the last ->dirty_inode ++ call. + + ``write_inode`` + this method is called when the VFS needs to write an inode to +diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c +index d385357e19b6..dce3a16996b9 100644 +--- a/fs/btrfs/backref.c ++++ b/fs/btrfs/backref.c +@@ -1511,16 +1511,118 @@ int btrfs_find_all_roots(struct btrfs_trans_handle *trans, + return ret; + } + +-/** +- * Check if an extent is shared or not ++/* ++ * The caller has joined a transaction or is holding a read lock on the ++ * fs_info->commit_root_sem semaphore, so no need to worry about the root's last ++ * snapshot field changing while updating or checking the cache. ++ */ ++static bool lookup_backref_shared_cache(struct btrfs_backref_shared_cache *cache, ++ struct btrfs_root *root, ++ u64 bytenr, int level, bool *is_shared) ++{ ++ struct btrfs_backref_shared_cache_entry *entry; ++ ++ if (WARN_ON_ONCE(level >= BTRFS_MAX_LEVEL)) ++ return false; ++ ++ /* ++ * Level -1 is used for the data extent, which is not reliable to cache ++ * because its reference count can increase or decrease without us ++ * realizing. We cache results only for extent buffers that lead from ++ * the root node down to the leaf with the file extent item. ++ */ ++ ASSERT(level >= 0); ++ ++ entry = &cache->entries[level]; ++ ++ /* Unused cache entry or being used for some other extent buffer. */ ++ if (entry->bytenr != bytenr) ++ return false; ++ ++ /* ++ * We cached a false result, but the last snapshot generation of the ++ * root changed, so we now have a snapshot. Don't trust the result. ++ */ ++ if (!entry->is_shared && ++ entry->gen != btrfs_root_last_snapshot(&root->root_item)) ++ return false; ++ ++ /* ++ * If we cached a true result and the last generation used for dropping ++ * a root changed, we can not trust the result, because the dropped root ++ * could be a snapshot sharing this extent buffer. ++ */ ++ if (entry->is_shared && ++ entry->gen != btrfs_get_last_root_drop_gen(root->fs_info)) ++ return false; ++ ++ *is_shared = entry->is_shared; ++ ++ return true; ++} ++ ++/* ++ * The caller has joined a transaction or is holding a read lock on the ++ * fs_info->commit_root_sem semaphore, so no need to worry about the root's last ++ * snapshot field changing while updating or checking the cache. ++ */ ++static void store_backref_shared_cache(struct btrfs_backref_shared_cache *cache, ++ struct btrfs_root *root, ++ u64 bytenr, int level, bool is_shared) ++{ ++ struct btrfs_backref_shared_cache_entry *entry; ++ u64 gen; ++ ++ if (WARN_ON_ONCE(level >= BTRFS_MAX_LEVEL)) ++ return; ++ ++ /* ++ * Level -1 is used for the data extent, which is not reliable to cache ++ * because its reference count can increase or decrease without us ++ * realizing. We cache results only for extent buffers that lead from ++ * the root node down to the leaf with the file extent item. ++ */ ++ ASSERT(level >= 0); ++ ++ if (is_shared) ++ gen = btrfs_get_last_root_drop_gen(root->fs_info); ++ else ++ gen = btrfs_root_last_snapshot(&root->root_item); ++ ++ entry = &cache->entries[level]; ++ entry->bytenr = bytenr; ++ entry->is_shared = is_shared; ++ entry->gen = gen; ++ ++ /* ++ * If we found an extent buffer is shared, set the cache result for all ++ * extent buffers below it to true. As nodes in the path are COWed, ++ * their sharedness is moved to their children, and if a leaf is COWed, ++ * then the sharedness of a data extent becomes direct, the refcount of ++ * data extent is increased in the extent item at the extent tree. ++ */ ++ if (is_shared) { ++ for (int i = 0; i < level; i++) { ++ entry = &cache->entries[i]; ++ entry->is_shared = is_shared; ++ entry->gen = gen; ++ } ++ } ++} ++ ++/* ++ * Check if a data extent is shared or not. + * +- * @root: root inode belongs to +- * @inum: inode number of the inode whose extent we are checking +- * @bytenr: logical bytenr of the extent we are checking +- * @roots: list of roots this extent is shared among +- * @tmp: temporary list used for iteration ++ * @root: The root the inode belongs to. ++ * @inum: Number of the inode whose extent we are checking. ++ * @bytenr: Logical bytenr of the extent we are checking. ++ * @extent_gen: Generation of the extent (file extent item) or 0 if it is ++ * not known. ++ * @roots: List of roots this extent is shared among. ++ * @tmp: Temporary list used for iteration. ++ * @cache: A backref lookup result cache. + * +- * btrfs_check_shared uses the backref walking code but will short ++ * btrfs_is_data_extent_shared uses the backref walking code but will short + * circuit as soon as it finds a root or inode that doesn't match the + * one passed in. This provides a significant performance benefit for + * callers (such as fiemap) which want to know whether the extent is +@@ -1531,8 +1633,10 @@ int btrfs_find_all_roots(struct btrfs_trans_handle *trans, + * + * Return: 0 if extent is not shared, 1 if it is shared, < 0 on error. + */ +-int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, +- struct ulist *roots, struct ulist *tmp) ++int btrfs_is_data_extent_shared(struct btrfs_root *root, u64 inum, u64 bytenr, ++ u64 extent_gen, ++ struct ulist *roots, struct ulist *tmp, ++ struct btrfs_backref_shared_cache *cache) + { + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; +@@ -1545,6 +1649,7 @@ int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, + .inum = inum, + .share_count = 0, + }; ++ int level; + + ulist_init(roots); + ulist_init(tmp); +@@ -1561,22 +1666,52 @@ int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, + btrfs_get_tree_mod_seq(fs_info, &elem); + } + ++ /* -1 means we are in the bytenr of the data extent. */ ++ level = -1; + ULIST_ITER_INIT(&uiter); + while (1) { ++ bool is_shared; ++ bool cached; ++ + ret = find_parent_nodes(trans, fs_info, bytenr, elem.seq, tmp, + roots, NULL, &shared, false); + if (ret == BACKREF_FOUND_SHARED) { + /* this is the only condition under which we return 1 */ + ret = 1; ++ if (level >= 0) ++ store_backref_shared_cache(cache, root, bytenr, ++ level, true); + break; + } + if (ret < 0 && ret != -ENOENT) + break; + ret = 0; ++ /* ++ * If our data extent is not shared through reflinks and it was ++ * created in a generation after the last one used to create a ++ * snapshot of the inode's root, then it can not be shared ++ * indirectly through subtrees, as that can only happen with ++ * snapshots. In this case bail out, no need to check for the ++ * sharedness of extent buffers. ++ */ ++ if (level == -1 && ++ extent_gen > btrfs_root_last_snapshot(&root->root_item)) ++ break; ++ ++ if (level >= 0) ++ store_backref_shared_cache(cache, root, bytenr, ++ level, false); + node = ulist_next(tmp, &uiter); + if (!node) + break; + bytenr = node->val; ++ level++; ++ cached = lookup_backref_shared_cache(cache, root, bytenr, level, ++ &is_shared); ++ if (cached) { ++ ret = (is_shared ? 1 : 0); ++ break; ++ } + shared.share_count = 0; + cond_resched(); + } +diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h +index 2759de7d324c..52ae6957b414 100644 +--- a/fs/btrfs/backref.h ++++ b/fs/btrfs/backref.h +@@ -17,6 +17,20 @@ struct inode_fs_paths { + struct btrfs_data_container *fspath; + }; + ++struct btrfs_backref_shared_cache_entry { ++ u64 bytenr; ++ u64 gen; ++ bool is_shared; ++}; ++ ++struct btrfs_backref_shared_cache { ++ /* ++ * A path from a root to a leaf that has a file extent item pointing to ++ * a given data extent should never exceed the maximum b+tree height. ++ */ ++ struct btrfs_backref_shared_cache_entry entries[BTRFS_MAX_LEVEL]; ++}; ++ + typedef int (iterate_extent_inodes_t)(u64 inum, u64 offset, u64 root, + void *ctx); + +@@ -62,8 +76,10 @@ int btrfs_find_one_extref(struct btrfs_root *root, u64 inode_objectid, + u64 start_off, struct btrfs_path *path, + struct btrfs_inode_extref **ret_extref, + u64 *found_off); +-int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, +- struct ulist *roots, struct ulist *tmp_ulist); ++int btrfs_is_data_extent_shared(struct btrfs_root *root, u64 inum, u64 bytenr, ++ u64 extent_gen, ++ struct ulist *roots, struct ulist *tmp, ++ struct btrfs_backref_shared_cache *cache); + + int __init btrfs_prelim_ref_init(void); + void __cold btrfs_prelim_ref_exit(void); +diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c +index e0375ba9d0fe..9df51245ba93 100644 +--- a/fs/btrfs/block-group.c ++++ b/fs/btrfs/block-group.c +@@ -2869,7 +2869,7 @@ static int cache_save_setup(struct btrfs_block_group *block_group, + cache_size *= fs_info->sectorsize; + + ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0, +- cache_size); ++ cache_size, false); + if (ret) + goto out_put; + +diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c +index ebfa35fe1c38..9caf0f87cbcb 100644 +--- a/fs/btrfs/ctree.c ++++ b/fs/btrfs/ctree.c +@@ -1447,6 +1447,11 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, + return 0; + } + ++ if (p->nowait) { ++ free_extent_buffer(tmp); ++ return -EAGAIN; ++ } ++ + if (unlock_up) + btrfs_unlock_up_safe(p, level + 1); + +@@ -1467,6 +1472,8 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, + ret = -EAGAIN; + + goto out; ++ } else if (p->nowait) { ++ return -EAGAIN; + } + + if (unlock_up) { +@@ -1634,7 +1641,13 @@ static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, + * We don't know the level of the root node until we actually + * have it read locked + */ +- b = btrfs_read_lock_root_node(root); ++ if (p->nowait) { ++ b = btrfs_try_read_lock_root_node(root); ++ if (IS_ERR(b)) ++ return b; ++ } else { ++ b = btrfs_read_lock_root_node(root); ++ } + level = btrfs_header_level(b); + if (level > write_lock_level) + goto out; +@@ -1910,6 +1923,13 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, + WARN_ON(p->nodes[0] != NULL); + BUG_ON(!cow && ins_len); + ++ /* ++ * For now only allow nowait for read only operations. There's no ++ * strict reason why we can't, we just only need it for reads so I'm ++ * only implementing it for reads right now. ++ */ ++ ASSERT(!p->nowait || !cow); ++ + if (ins_len < 0) { + lowest_unlock = 2; + +@@ -1936,7 +1956,12 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, + + if (p->need_commit_sem) { + ASSERT(p->search_commit_root); +- down_read(&fs_info->commit_root_sem); ++ if (p->nowait) { ++ if (!down_read_trylock(&fs_info->commit_root_sem)) ++ return -EAGAIN; ++ } else { ++ down_read(&fs_info->commit_root_sem); ++ } + } + + again: +@@ -2082,7 +2107,15 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, + btrfs_tree_lock(b); + p->locks[level] = BTRFS_WRITE_LOCK; + } else { +- btrfs_tree_read_lock(b); ++ if (p->nowait) { ++ if (!btrfs_try_tree_read_lock(b)) { ++ free_extent_buffer(b); ++ ret = -EAGAIN; ++ goto done; ++ } ++ } else { ++ btrfs_tree_read_lock(b); ++ } + p->locks[level] = BTRFS_READ_LOCK; + } + p->nodes[level] = b; +@@ -2132,6 +2165,9 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, + lowest_level = p->lowest_level; + WARN_ON(p->nodes[0] != NULL); + ++ if (WARN_ON_ONCE(p->nowait == 1)) ++ return -EINVAL; ++ + if (p->search_commit_root) { + BUG_ON(time_seq); + return btrfs_search_slot(NULL, root, key, p, 0, 0); +@@ -4432,6 +4468,9 @@ int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, + int ret = 1; + int keep_locks = path->keep_locks; + ++ if (WARN_ON_ONCE(path->nowait == 1)) ++ return -EINVAL; ++ + path->keep_locks = 1; + again: + cur = btrfs_read_lock_root_node(root); +@@ -4612,6 +4651,9 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, + int ret; + int i; + ++ if (WARN_ON_ONCE(path->nowait == 1)) ++ return -EINVAL; ++ + nritems = btrfs_header_nritems(path->nodes[0]); + if (nritems == 0) + return 1; +diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h +index df8c99c99df9..dedac9faa98a 100644 +--- a/fs/btrfs/ctree.h ++++ b/fs/btrfs/ctree.h +@@ -443,6 +443,7 @@ struct btrfs_path { + * header (ie. sizeof(struct btrfs_item) is not included). + */ + unsigned int search_for_extension:1; ++ unsigned int nowait:1; + }; + #define BTRFS_MAX_EXTENT_ITEM_SIZE(r) ((BTRFS_LEAF_DATA_SIZE(r->fs_info) >> 4) - \ + sizeof(struct btrfs_item)) +@@ -1092,6 +1093,23 @@ struct btrfs_fs_info { + /* Updates are not protected by any lock */ + struct btrfs_commit_stats commit_stats; + ++ /* ++ * Last generation where we dropped a non-relocation root. ++ * Use btrfs_set_last_root_drop_gen() and btrfs_get_last_root_drop_gen() ++ * to change it and to read it, respectively. ++ */ ++ u64 last_root_drop_gen; ++ ++ /* ++ * Annotations for transaction events (structures are empty when ++ * compiled without lockdep). ++ */ ++ struct lockdep_map btrfs_trans_num_writers_map; ++ struct lockdep_map btrfs_trans_num_extwriters_map; ++ struct lockdep_map btrfs_state_change_map[4]; ++ struct lockdep_map btrfs_trans_pending_ordered_map; ++ struct lockdep_map btrfs_ordered_extent_map; ++ + #ifdef CONFIG_BTRFS_FS_REF_VERIFY + spinlock_t ref_verify_lock; + struct rb_root block_tree; +@@ -1107,6 +1125,17 @@ struct btrfs_fs_info { + #endif + }; + ++static inline void btrfs_set_last_root_drop_gen(struct btrfs_fs_info *fs_info, ++ u64 gen) ++{ ++ WRITE_ONCE(fs_info->last_root_drop_gen, gen); ++} ++ ++static inline u64 btrfs_get_last_root_drop_gen(const struct btrfs_fs_info *fs_info) ++{ ++ return READ_ONCE(fs_info->last_root_drop_gen); ++} ++ + static inline struct btrfs_fs_info *btrfs_sb(struct super_block *sb) + { + return sb->s_fs_info; +@@ -1174,6 +1203,82 @@ enum { + BTRFS_ROOT_RESET_LOCKDEP_CLASS, + }; + ++enum btrfs_lockdep_trans_states { ++ BTRFS_LOCKDEP_TRANS_COMMIT_START, ++ BTRFS_LOCKDEP_TRANS_UNBLOCKED, ++ BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED, ++ BTRFS_LOCKDEP_TRANS_COMPLETED, ++}; ++ ++/* ++ * Lockdep annotation for wait events. ++ * ++ * @owner: The struct where the lockdep map is defined ++ * @lock: The lockdep map corresponding to a wait event ++ * ++ * This macro is used to annotate a wait event. In this case a thread acquires ++ * the lockdep map as writer (exclusive lock) because it has to block until all ++ * the threads that hold the lock as readers signal the condition for the wait ++ * event and release their locks. ++ */ ++#define btrfs_might_wait_for_event(owner, lock) \ ++ do { \ ++ rwsem_acquire(&owner->lock##_map, 0, 0, _THIS_IP_); \ ++ rwsem_release(&owner->lock##_map, _THIS_IP_); \ ++ } while (0) ++ ++/* ++ * Protection for the resource/condition of a wait event. ++ * ++ * @owner: The struct where the lockdep map is defined ++ * @lock: The lockdep map corresponding to a wait event ++ * ++ * Many threads can modify the condition for the wait event at the same time ++ * and signal the threads that block on the wait event. The threads that modify ++ * the condition and do the signaling acquire the lock as readers (shared ++ * lock). ++ */ ++#define btrfs_lockdep_acquire(owner, lock) \ ++ rwsem_acquire_read(&owner->lock##_map, 0, 0, _THIS_IP_) ++ ++/* ++ * Used after signaling the condition for a wait event to release the lockdep ++ * map held by a reader thread. ++ */ ++#define btrfs_lockdep_release(owner, lock) \ ++ rwsem_release(&owner->lock##_map, _THIS_IP_) ++ ++/* ++ * Macros for the transaction states wait events, similar to the generic wait ++ * event macros. ++ */ ++#define btrfs_might_wait_for_state(owner, i) \ ++ do { \ ++ rwsem_acquire(&owner->btrfs_state_change_map[i], 0, 0, _THIS_IP_); \ ++ rwsem_release(&owner->btrfs_state_change_map[i], _THIS_IP_); \ ++ } while (0) ++ ++#define btrfs_trans_state_lockdep_acquire(owner, i) \ ++ rwsem_acquire_read(&owner->btrfs_state_change_map[i], 0, 0, _THIS_IP_) ++ ++#define btrfs_trans_state_lockdep_release(owner, i) \ ++ rwsem_release(&owner->btrfs_state_change_map[i], _THIS_IP_) ++ ++/* Initialization of the lockdep map */ ++#define btrfs_lockdep_init_map(owner, lock) \ ++ do { \ ++ static struct lock_class_key lock##_key; \ ++ lockdep_init_map(&owner->lock##_map, #lock, &lock##_key, 0); \ ++ } while (0) ++ ++/* Initialization of the transaction states lockdep maps. */ ++#define btrfs_state_lockdep_init_map(owner, lock, state) \ ++ do { \ ++ static struct lock_class_key lock##_key; \ ++ lockdep_init_map(&owner->btrfs_state_change_map[state], #lock, \ ++ &lock##_key, 0); \ ++ } while (0) ++ + static inline void btrfs_wake_unfinished_drop(struct btrfs_fs_info *fs_info) + { + clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags); +@@ -3273,7 +3378,8 @@ int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, + blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, + u64 offset, bool one_ordered); + int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, +- struct list_head *list, int search_commit); ++ struct list_head *list, int search_commit, ++ bool nowait); + void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, + const struct btrfs_path *path, + struct btrfs_file_extent_item *fi, +@@ -3299,11 +3405,9 @@ unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, + u64 start, u64 end); + int btrfs_check_data_csum(struct inode *inode, struct btrfs_bio *bbio, + u32 bio_offset, struct page *page, u32 pgoff); +-struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, +- u64 start, u64 len); + noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, + u64 *orig_start, u64 *orig_block_len, +- u64 *ram_bytes, bool strict); ++ u64 *ram_bytes, bool nowait, bool strict); + + void __btrfs_del_delalloc_inode(struct btrfs_root *root, + struct btrfs_inode *inode); +@@ -3478,8 +3582,10 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, + struct extent_state **cached, bool noreserve); + int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end); + int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, +- size_t *write_bytes); ++ size_t *write_bytes, bool nowait); + void btrfs_check_nocow_unlock(struct btrfs_inode *inode); ++bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end, ++ u64 *delalloc_start_ret, u64 *delalloc_end_ret); + + /* tree-defrag.c */ + int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, +diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c +index 1e8f17ff829e..118b2e20b2e1 100644 +--- a/fs/btrfs/delalloc-space.c ++++ b/fs/btrfs/delalloc-space.c +@@ -127,9 +127,11 @@ int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes) + } + + int btrfs_check_data_free_space(struct btrfs_inode *inode, +- struct extent_changeset **reserved, u64 start, u64 len) ++ struct extent_changeset **reserved, u64 start, ++ u64 len, bool noflush) + { + struct btrfs_fs_info *fs_info = inode->root->fs_info; ++ enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA; + int ret; + + /* align the range */ +@@ -137,7 +139,12 @@ int btrfs_check_data_free_space(struct btrfs_inode *inode, + round_down(start, fs_info->sectorsize); + start = round_down(start, fs_info->sectorsize); + +- ret = btrfs_alloc_data_chunk_ondemand(inode, len); ++ if (noflush) ++ flush = BTRFS_RESERVE_NO_FLUSH; ++ else if (btrfs_is_free_space_inode(inode)) ++ flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; ++ ++ ret = btrfs_reserve_data_bytes(fs_info, len, flush); + if (ret < 0) + return ret; + +@@ -454,7 +461,7 @@ int btrfs_delalloc_reserve_space(struct btrfs_inode *inode, + { + int ret; + +- ret = btrfs_check_data_free_space(inode, reserved, start, len); ++ ret = btrfs_check_data_free_space(inode, reserved, start, len, false); + if (ret < 0) + return ret; + ret = btrfs_delalloc_reserve_metadata(inode, len, len, false); +diff --git a/fs/btrfs/delalloc-space.h b/fs/btrfs/delalloc-space.h +index 28bf5c3ef430..e07d46043455 100644 +--- a/fs/btrfs/delalloc-space.h ++++ b/fs/btrfs/delalloc-space.h +@@ -7,7 +7,8 @@ struct extent_changeset; + + int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes); + int btrfs_check_data_free_space(struct btrfs_inode *inode, +- struct extent_changeset **reserved, u64 start, u64 len); ++ struct extent_changeset **reserved, u64 start, u64 len, ++ bool noflush); + void btrfs_free_reserved_data_space(struct btrfs_inode *inode, + struct extent_changeset *reserved, u64 start, u64 len); + void btrfs_delalloc_release_space(struct btrfs_inode *inode, +diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c +index e7f34871a132..2837b2dbd611 100644 +--- a/fs/btrfs/delayed-inode.c ++++ b/fs/btrfs/delayed-inode.c +@@ -302,15 +302,21 @@ static inline void btrfs_release_prepared_delayed_node( + __btrfs_release_delayed_node(node, 1); + } + +-static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u32 data_len) ++static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u16 data_len, ++ struct btrfs_delayed_node *node, ++ enum btrfs_delayed_item_type type) + { + struct btrfs_delayed_item *item; ++ + item = kmalloc(sizeof(*item) + data_len, GFP_NOFS); + if (item) { + item->data_len = data_len; +- item->ins_or_del = 0; ++ item->type = type; + item->bytes_reserved = 0; +- item->delayed_node = NULL; ++ item->delayed_node = node; ++ RB_CLEAR_NODE(&item->rb_node); ++ INIT_LIST_HEAD(&item->log_list); ++ item->logged = false; + refcount_set(&item->refs, 1); + } + return item; +@@ -319,72 +325,32 @@ static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u32 data_len) + /* + * __btrfs_lookup_delayed_item - look up the delayed item by key + * @delayed_node: pointer to the delayed node +- * @key: the key to look up +- * @prev: used to store the prev item if the right item isn't found +- * @next: used to store the next item if the right item isn't found ++ * @index: the dir index value to lookup (offset of a dir index key) + * + * Note: if we don't find the right item, we will return the prev item and + * the next item. + */ + static struct btrfs_delayed_item *__btrfs_lookup_delayed_item( + struct rb_root *root, +- struct btrfs_key *key, +- struct btrfs_delayed_item **prev, +- struct btrfs_delayed_item **next) ++ u64 index) + { +- struct rb_node *node, *prev_node = NULL; ++ struct rb_node *node = root->rb_node; + struct btrfs_delayed_item *delayed_item = NULL; +- int ret = 0; +- +- node = root->rb_node; + + while (node) { + delayed_item = rb_entry(node, struct btrfs_delayed_item, + rb_node); +- prev_node = node; +- ret = btrfs_comp_cpu_keys(&delayed_item->key, key); +- if (ret < 0) ++ if (delayed_item->index < index) + node = node->rb_right; +- else if (ret > 0) ++ else if (delayed_item->index > index) + node = node->rb_left; + else + return delayed_item; + } + +- if (prev) { +- if (!prev_node) +- *prev = NULL; +- else if (ret < 0) +- *prev = delayed_item; +- else if ((node = rb_prev(prev_node)) != NULL) { +- *prev = rb_entry(node, struct btrfs_delayed_item, +- rb_node); +- } else +- *prev = NULL; +- } +- +- if (next) { +- if (!prev_node) +- *next = NULL; +- else if (ret > 0) +- *next = delayed_item; +- else if ((node = rb_next(prev_node)) != NULL) { +- *next = rb_entry(node, struct btrfs_delayed_item, +- rb_node); +- } else +- *next = NULL; +- } + return NULL; + } + +-static struct btrfs_delayed_item *__btrfs_lookup_delayed_insertion_item( +- struct btrfs_delayed_node *delayed_node, +- struct btrfs_key *key) +-{ +- return __btrfs_lookup_delayed_item(&delayed_node->ins_root.rb_root, key, +- NULL, NULL); +-} +- + static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, + struct btrfs_delayed_item *ins) + { +@@ -392,15 +358,13 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, + struct rb_node *parent_node = NULL; + struct rb_root_cached *root; + struct btrfs_delayed_item *item; +- int cmp; + bool leftmost = true; + +- if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM) ++ if (ins->type == BTRFS_DELAYED_INSERTION_ITEM) + root = &delayed_node->ins_root; +- else if (ins->ins_or_del == BTRFS_DELAYED_DELETION_ITEM) +- root = &delayed_node->del_root; + else +- BUG(); ++ root = &delayed_node->del_root; ++ + p = &root->rb_root.rb_node; + node = &ins->rb_node; + +@@ -409,11 +373,10 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, + item = rb_entry(parent_node, struct btrfs_delayed_item, + rb_node); + +- cmp = btrfs_comp_cpu_keys(&item->key, &ins->key); +- if (cmp < 0) { ++ if (item->index < ins->index) { + p = &(*p)->rb_right; + leftmost = false; +- } else if (cmp > 0) { ++ } else if (item->index > ins->index) { + p = &(*p)->rb_left; + } else { + return -EEXIST; +@@ -422,14 +385,10 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, + + rb_link_node(node, parent_node, p); + rb_insert_color_cached(node, root, leftmost); +- ins->delayed_node = delayed_node; + +- /* Delayed items are always for dir index items. */ +- ASSERT(ins->key.type == BTRFS_DIR_INDEX_KEY); +- +- if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM && +- ins->key.offset >= delayed_node->index_cnt) +- delayed_node->index_cnt = ins->key.offset + 1; ++ if (ins->type == BTRFS_DELAYED_INSERTION_ITEM && ++ ins->index >= delayed_node->index_cnt) ++ delayed_node->index_cnt = ins->index + 1; + + delayed_node->count++; + atomic_inc(&delayed_node->root->fs_info->delayed_root->items); +@@ -451,21 +410,21 @@ static void __btrfs_remove_delayed_item(struct btrfs_delayed_item *delayed_item) + struct rb_root_cached *root; + struct btrfs_delayed_root *delayed_root; + +- /* Not associated with any delayed_node */ +- if (!delayed_item->delayed_node) ++ /* Not inserted, ignore it. */ ++ if (RB_EMPTY_NODE(&delayed_item->rb_node)) + return; ++ + delayed_root = delayed_item->delayed_node->root->fs_info->delayed_root; + + BUG_ON(!delayed_root); +- BUG_ON(delayed_item->ins_or_del != BTRFS_DELAYED_DELETION_ITEM && +- delayed_item->ins_or_del != BTRFS_DELAYED_INSERTION_ITEM); + +- if (delayed_item->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM) ++ if (delayed_item->type == BTRFS_DELAYED_INSERTION_ITEM) + root = &delayed_item->delayed_node->ins_root; + else + root = &delayed_item->delayed_node->del_root; + + rb_erase_cached(&delayed_item->rb_node, root); ++ RB_CLEAR_NODE(&delayed_item->rb_node); + delayed_item->delayed_node->count--; + + finish_one_item(delayed_root); +@@ -520,12 +479,11 @@ static struct btrfs_delayed_item *__btrfs_next_delayed_item( + } + + static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, +- struct btrfs_root *root, + struct btrfs_delayed_item *item) + { + struct btrfs_block_rsv *src_rsv; + struct btrfs_block_rsv *dst_rsv; +- struct btrfs_fs_info *fs_info = root->fs_info; ++ struct btrfs_fs_info *fs_info = trans->fs_info; + u64 num_bytes; + int ret; + +@@ -545,14 +503,14 @@ static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, + ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, true); + if (!ret) { + trace_btrfs_space_reservation(fs_info, "delayed_item", +- item->key.objectid, ++ item->delayed_node->inode_id, + num_bytes, 1); + /* + * For insertions we track reserved metadata space by accounting + * for the number of leaves that will be used, based on the delayed + * node's index_items_size field. + */ +- if (item->ins_or_del == BTRFS_DELAYED_DELETION_ITEM) ++ if (item->type == BTRFS_DELAYED_DELETION_ITEM) + item->bytes_reserved = num_bytes; + } + +@@ -574,8 +532,8 @@ static void btrfs_delayed_item_release_metadata(struct btrfs_root *root, + * to release/reserve qgroup space. + */ + trace_btrfs_space_reservation(fs_info, "delayed_item", +- item->key.objectid, item->bytes_reserved, +- 0); ++ item->delayed_node->inode_id, ++ item->bytes_reserved, 0); + btrfs_block_rsv_release(fs_info, rsv, item->bytes_reserved, NULL); + } + +@@ -688,6 +646,8 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, + struct btrfs_delayed_item *next; + const int max_size = BTRFS_LEAF_DATA_SIZE(fs_info); + struct btrfs_item_batch batch; ++ struct btrfs_key first_key; ++ const u32 first_data_size = first_item->data_len; + int total_size; + char *ins_data = NULL; + int ret; +@@ -716,9 +676,9 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, + ASSERT(first_item->bytes_reserved == 0); + + list_add_tail(&first_item->tree_list, &item_list); +- batch.total_data_size = first_item->data_len; ++ batch.total_data_size = first_data_size; + batch.nr = 1; +- total_size = first_item->data_len + sizeof(struct btrfs_item); ++ total_size = first_data_size + sizeof(struct btrfs_item); + curr = first_item; + + while (true) { +@@ -732,8 +692,7 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, + * We cannot allow gaps in the key space if we're doing log + * replay. + */ +- if (continuous_keys_only && +- (next->key.offset != curr->key.offset + 1)) ++ if (continuous_keys_only && (next->index != curr->index + 1)) + break; + + ASSERT(next->bytes_reserved == 0); +@@ -750,8 +709,11 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, + } + + if (batch.nr == 1) { +- batch.keys = &first_item->key; +- batch.data_sizes = &first_item->data_len; ++ first_key.objectid = node->inode_id; ++ first_key.type = BTRFS_DIR_INDEX_KEY; ++ first_key.offset = first_item->index; ++ batch.keys = &first_key; ++ batch.data_sizes = &first_data_size; + } else { + struct btrfs_key *ins_keys; + u32 *ins_sizes; +@@ -768,7 +730,9 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, + batch.keys = ins_keys; + batch.data_sizes = ins_sizes; + list_for_each_entry(curr, &item_list, tree_list) { +- ins_keys[i] = curr->key; ++ ins_keys[i].objectid = node->inode_id; ++ ins_keys[i].type = BTRFS_DIR_INDEX_KEY; ++ ins_keys[i].offset = curr->index; + ins_sizes[i] = curr->data_len; + i++; + } +@@ -864,6 +828,7 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_delayed_item *item) + { ++ const u64 ino = item->delayed_node->inode_id; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_delayed_item *curr, *next; + struct extent_buffer *leaf = path->nodes[0]; +@@ -902,7 +867,9 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, + + slot++; + btrfs_item_key_to_cpu(leaf, &key, slot); +- if (btrfs_comp_cpu_keys(&next->key, &key) != 0) ++ if (key.objectid != ino || ++ key.type != BTRFS_DIR_INDEX_KEY || ++ key.offset != next->index) + break; + nitems++; + curr = next; +@@ -920,9 +887,8 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, + * Check btrfs_delayed_item_reserve_metadata() to see why we + * don't need to release/reserve qgroup space. + */ +- trace_btrfs_space_reservation(fs_info, "delayed_item", +- item->key.objectid, total_reserved_size, +- 0); ++ trace_btrfs_space_reservation(fs_info, "delayed_item", ino, ++ total_reserved_size, 0); + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, + total_reserved_size, NULL); + } +@@ -940,8 +906,12 @@ static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_delayed_node *node) + { ++ struct btrfs_key key; + int ret = 0; + ++ key.objectid = node->inode_id; ++ key.type = BTRFS_DIR_INDEX_KEY; ++ + while (ret == 0) { + struct btrfs_delayed_item *item; + +@@ -952,7 +922,8 @@ static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, + break; + } + +- ret = btrfs_search_slot(trans, root, &item->key, path, -1, 1); ++ key.offset = item->index; ++ ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) { + /* + * There's no matching item in the leaf. This means we +@@ -1206,43 +1177,6 @@ int btrfs_run_delayed_items_nr(struct btrfs_trans_handle *trans, int nr) + return __btrfs_run_delayed_items(trans, nr); + } + +-int btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans, +- struct btrfs_inode *inode) +-{ +- struct btrfs_delayed_node *delayed_node = btrfs_get_delayed_node(inode); +- struct btrfs_path *path; +- struct btrfs_block_rsv *block_rsv; +- int ret; +- +- if (!delayed_node) +- return 0; +- +- mutex_lock(&delayed_node->mutex); +- if (!delayed_node->count) { +- mutex_unlock(&delayed_node->mutex); +- btrfs_release_delayed_node(delayed_node); +- return 0; +- } +- mutex_unlock(&delayed_node->mutex); +- +- path = btrfs_alloc_path(); +- if (!path) { +- btrfs_release_delayed_node(delayed_node); +- return -ENOMEM; +- } +- +- block_rsv = trans->block_rsv; +- trans->block_rsv = &delayed_node->root->fs_info->delayed_block_rsv; +- +- ret = __btrfs_commit_inode_delayed_items(trans, path, delayed_node); +- +- btrfs_release_delayed_node(delayed_node); +- btrfs_free_path(path); +- trans->block_rsv = block_rsv; +- +- return ret; +-} +- + int btrfs_commit_inode_delayed_inode(struct btrfs_inode *inode) + { + struct btrfs_fs_info *fs_info = inode->root->fs_info; +@@ -1457,16 +1391,15 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, + if (IS_ERR(delayed_node)) + return PTR_ERR(delayed_node); + +- delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len); ++ delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len, ++ delayed_node, ++ BTRFS_DELAYED_INSERTION_ITEM); + if (!delayed_item) { + ret = -ENOMEM; + goto release_node; + } + +- delayed_item->key.objectid = btrfs_ino(dir); +- delayed_item->key.type = BTRFS_DIR_INDEX_KEY; +- delayed_item->key.offset = index; +- delayed_item->ins_or_del = BTRFS_DELAYED_INSERTION_ITEM; ++ delayed_item->index = index; + + dir_item = (struct btrfs_dir_item *)delayed_item->data; + dir_item->location = *disk_key; +@@ -1490,8 +1423,7 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, + } + + if (reserve_leaf_space) { +- ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, +- delayed_item); ++ ret = btrfs_delayed_item_reserve_metadata(trans, delayed_item); + /* + * Space was reserved for a dir index item insertion when we + * started the transaction, so getting a failure here should be +@@ -1538,12 +1470,12 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, + + static int btrfs_delete_delayed_insertion_item(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_node *node, +- struct btrfs_key *key) ++ u64 index) + { + struct btrfs_delayed_item *item; + + mutex_lock(&node->mutex); +- item = __btrfs_lookup_delayed_insertion_item(node, key); ++ item = __btrfs_lookup_delayed_item(&node->ins_root.rb_root, index); + if (!item) { + mutex_unlock(&node->mutex); + return 1; +@@ -1589,32 +1521,25 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, + { + struct btrfs_delayed_node *node; + struct btrfs_delayed_item *item; +- struct btrfs_key item_key; + int ret; + + node = btrfs_get_or_create_delayed_node(dir); + if (IS_ERR(node)) + return PTR_ERR(node); + +- item_key.objectid = btrfs_ino(dir); +- item_key.type = BTRFS_DIR_INDEX_KEY; +- item_key.offset = index; +- +- ret = btrfs_delete_delayed_insertion_item(trans->fs_info, node, +- &item_key); ++ ret = btrfs_delete_delayed_insertion_item(trans->fs_info, node, index); + if (!ret) + goto end; + +- item = btrfs_alloc_delayed_item(0); ++ item = btrfs_alloc_delayed_item(0, node, BTRFS_DELAYED_DELETION_ITEM); + if (!item) { + ret = -ENOMEM; + goto end; + } + +- item->key = item_key; +- item->ins_or_del = BTRFS_DELAYED_DELETION_ITEM; ++ item->index = index; + +- ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, item); ++ ret = btrfs_delayed_item_reserve_metadata(trans, item); + /* + * we have reserved enough space when we start a new transaction, + * so reserving metadata failure is impossible. +@@ -1743,9 +1668,9 @@ int btrfs_should_delete_dir_index(struct list_head *del_list, + int ret = 0; + + list_for_each_entry(curr, del_list, readdir_list) { +- if (curr->key.offset > index) ++ if (curr->index > index) + break; +- if (curr->key.offset == index) { ++ if (curr->index == index) { + ret = 1; + break; + } +@@ -1779,13 +1704,13 @@ int btrfs_readdir_delayed_dir_index(struct dir_context *ctx, + list_for_each_entry_safe(curr, next, ins_list, readdir_list) { + list_del(&curr->readdir_list); + +- if (curr->key.offset < ctx->pos) { ++ if (curr->index < ctx->pos) { + if (refcount_dec_and_test(&curr->refs)) + kfree(curr); + continue; + } + +- ctx->pos = curr->key.offset; ++ ctx->pos = curr->index; + + di = (struct btrfs_dir_item *)curr->data; + name = (char *)(di + 1); +@@ -2085,3 +2010,113 @@ void btrfs_destroy_delayed_inodes(struct btrfs_fs_info *fs_info) + } + } + ++void btrfs_log_get_delayed_items(struct btrfs_inode *inode, ++ struct list_head *ins_list, ++ struct list_head *del_list) ++{ ++ struct btrfs_delayed_node *node; ++ struct btrfs_delayed_item *item; ++ ++ node = btrfs_get_delayed_node(inode); ++ if (!node) ++ return; ++ ++ mutex_lock(&node->mutex); ++ item = __btrfs_first_delayed_insertion_item(node); ++ while (item) { ++ /* ++ * It's possible that the item is already in a log list. This ++ * can happen in case two tasks are trying to log the same ++ * directory. For example if we have tasks A and task B: ++ * ++ * Task A collected the delayed items into a log list while ++ * under the inode's log_mutex (at btrfs_log_inode()), but it ++ * only releases the items after logging the inodes they point ++ * to (if they are new inodes), which happens after unlocking ++ * the log mutex; ++ * ++ * Task B enters btrfs_log_inode() and acquires the log_mutex ++ * of the same directory inode, before task B releases the ++ * delayed items. This can happen for example when logging some ++ * inode we need to trigger logging of its parent directory, so ++ * logging two files that have the same parent directory can ++ * lead to this. ++ * ++ * If this happens, just ignore delayed items already in a log ++ * list. All the tasks logging the directory are under a log ++ * transaction and whichever finishes first can not sync the log ++ * before the other completes and leaves the log transaction. ++ */ ++ if (!item->logged && list_empty(&item->log_list)) { ++ refcount_inc(&item->refs); ++ list_add_tail(&item->log_list, ins_list); ++ } ++ item = __btrfs_next_delayed_item(item); ++ } ++ ++ item = __btrfs_first_delayed_deletion_item(node); ++ while (item) { ++ /* It may be non-empty, for the same reason mentioned above. */ ++ if (!item->logged && list_empty(&item->log_list)) { ++ refcount_inc(&item->refs); ++ list_add_tail(&item->log_list, del_list); ++ } ++ item = __btrfs_next_delayed_item(item); ++ } ++ mutex_unlock(&node->mutex); ++ ++ /* ++ * We are called during inode logging, which means the inode is in use ++ * and can not be evicted before we finish logging the inode. So we never ++ * have the last reference on the delayed inode. ++ * Also, we don't use btrfs_release_delayed_node() because that would ++ * requeue the delayed inode (change its order in the list of prepared ++ * nodes) and we don't want to do such change because we don't create or ++ * delete delayed items. ++ */ ++ ASSERT(refcount_read(&node->refs) > 1); ++ refcount_dec(&node->refs); ++} ++ ++void btrfs_log_put_delayed_items(struct btrfs_inode *inode, ++ struct list_head *ins_list, ++ struct list_head *del_list) ++{ ++ struct btrfs_delayed_node *node; ++ struct btrfs_delayed_item *item; ++ struct btrfs_delayed_item *next; ++ ++ node = btrfs_get_delayed_node(inode); ++ if (!node) ++ return; ++ ++ mutex_lock(&node->mutex); ++ ++ list_for_each_entry_safe(item, next, ins_list, log_list) { ++ item->logged = true; ++ list_del_init(&item->log_list); ++ if (refcount_dec_and_test(&item->refs)) ++ kfree(item); ++ } ++ ++ list_for_each_entry_safe(item, next, del_list, log_list) { ++ item->logged = true; ++ list_del_init(&item->log_list); ++ if (refcount_dec_and_test(&item->refs)) ++ kfree(item); ++ } ++ ++ mutex_unlock(&node->mutex); ++ ++ /* ++ * We are called during inode logging, which means the inode is in use ++ * and can not be evicted before we finish logging the inode. So we never ++ * have the last reference on the delayed inode. ++ * Also, we don't use btrfs_release_delayed_node() because that would ++ * requeue the delayed inode (change its order in the list of prepared ++ * nodes) and we don't want to do such change because we don't create or ++ * delete delayed items. ++ */ ++ ASSERT(refcount_read(&node->refs) > 1); ++ refcount_dec(&node->refs); ++} +diff --git a/fs/btrfs/delayed-inode.h b/fs/btrfs/delayed-inode.h +index 9795dc295a18..95898ddc35e1 100644 +--- a/fs/btrfs/delayed-inode.h ++++ b/fs/btrfs/delayed-inode.h +@@ -16,9 +16,10 @@ + #include + #include "ctree.h" + +-/* types of the delayed item */ +-#define BTRFS_DELAYED_INSERTION_ITEM 1 +-#define BTRFS_DELAYED_DELETION_ITEM 2 ++enum btrfs_delayed_item_type { ++ BTRFS_DELAYED_INSERTION_ITEM, ++ BTRFS_DELAYED_DELETION_ITEM ++}; + + struct btrfs_delayed_root { + spinlock_t lock; +@@ -73,14 +74,27 @@ struct btrfs_delayed_node { + + struct btrfs_delayed_item { + struct rb_node rb_node; +- struct btrfs_key key; ++ /* Offset value of the corresponding dir index key. */ ++ u64 index; + struct list_head tree_list; /* used for batch insert/delete items */ + struct list_head readdir_list; /* used for readdir items */ ++ /* ++ * Used when logging a directory. ++ * Insertions and deletions to this list are protected by the parent ++ * delayed node's mutex. ++ */ ++ struct list_head log_list; + u64 bytes_reserved; + struct btrfs_delayed_node *delayed_node; + refcount_t refs; +- int ins_or_del; +- u32 data_len; ++ enum btrfs_delayed_item_type type:1; ++ /* ++ * Track if this delayed item was already logged. ++ * Protected by the mutex of the parent delayed inode. ++ */ ++ bool logged; ++ /* The maximum leaf size is 64K, so u16 is more than enough. */ ++ u16 data_len; + char data[]; + }; + +@@ -112,8 +126,6 @@ int btrfs_run_delayed_items_nr(struct btrfs_trans_handle *trans, int nr); + + void btrfs_balance_delayed_items(struct btrfs_fs_info *fs_info); + +-int btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans, +- struct btrfs_inode *inode); + /* Used for evicting the inode. */ + void btrfs_remove_delayed_node(struct btrfs_inode *inode); + void btrfs_kill_delayed_inode_items(struct btrfs_inode *inode); +@@ -144,6 +156,14 @@ int btrfs_should_delete_dir_index(struct list_head *del_list, + int btrfs_readdir_delayed_dir_index(struct dir_context *ctx, + struct list_head *ins_list); + ++/* Used during directory logging. */ ++void btrfs_log_get_delayed_items(struct btrfs_inode *inode, ++ struct list_head *ins_list, ++ struct list_head *del_list); ++void btrfs_log_put_delayed_items(struct btrfs_inode *inode, ++ struct list_head *ins_list, ++ struct list_head *del_list); ++ + /* for init */ + int __init btrfs_delayed_inode_init(void); + void __cold btrfs_delayed_inode_exit(void); +diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c +index 2633137c3e9f..e0e1730e67d7 100644 +--- a/fs/btrfs/disk-io.c ++++ b/fs/btrfs/disk-io.c +@@ -2990,6 +2990,19 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) + mutex_init(&fs_info->zoned_data_reloc_io_lock); + seqlock_init(&fs_info->profiles_lock); + ++ btrfs_lockdep_init_map(fs_info, btrfs_trans_num_writers); ++ btrfs_lockdep_init_map(fs_info, btrfs_trans_num_extwriters); ++ btrfs_lockdep_init_map(fs_info, btrfs_trans_pending_ordered); ++ btrfs_lockdep_init_map(fs_info, btrfs_ordered_extent); ++ btrfs_state_lockdep_init_map(fs_info, btrfs_trans_commit_start, ++ BTRFS_LOCKDEP_TRANS_COMMIT_START); ++ btrfs_state_lockdep_init_map(fs_info, btrfs_trans_unblocked, ++ BTRFS_LOCKDEP_TRANS_UNBLOCKED); ++ btrfs_state_lockdep_init_map(fs_info, btrfs_trans_super_committed, ++ BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); ++ btrfs_state_lockdep_init_map(fs_info, btrfs_trans_completed, ++ BTRFS_LOCKDEP_TRANS_COMPLETED); ++ + INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots); + INIT_LIST_HEAD(&fs_info->space_info); + INIT_LIST_HEAD(&fs_info->tree_mod_seq_list); +diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c +index 6914cd8024ba..56bb12ece500 100644 +--- a/fs/btrfs/extent-tree.c ++++ b/fs/btrfs/extent-tree.c +@@ -2220,6 +2220,11 @@ static noinline int check_delayed_ref(struct btrfs_root *root, + } + + if (!mutex_trylock(&head->mutex)) { ++ if (path->nowait) { ++ spin_unlock(&delayed_refs->lock); ++ return -EAGAIN; ++ } ++ + refcount_inc(&head->refs); + spin_unlock(&delayed_refs->lock); + +@@ -5639,6 +5644,8 @@ static noinline int walk_up_tree(struct btrfs_trans_handle *trans, + */ + int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) + { ++ const bool is_reloc_root = (root->root_key.objectid == ++ BTRFS_TREE_RELOC_OBJECTID); + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_path *path; + struct btrfs_trans_handle *trans; +@@ -5798,6 +5805,9 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) + goto out_end_trans; + } + ++ if (!is_reloc_root) ++ btrfs_set_last_root_drop_gen(fs_info, trans->transid); ++ + btrfs_end_transaction_throttle(trans); + if (!for_reloc && btrfs_need_cleaner_sleep(fs_info)) { + btrfs_debug(fs_info, +@@ -5832,7 +5842,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) + goto out_end_trans; + } + +- if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) { ++ if (!is_reloc_root) { + ret = btrfs_find_root(tree_root, &root->root_key, path, + NULL, NULL); + if (ret < 0) { +@@ -5864,6 +5874,9 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) + btrfs_put_root(root); + root_dropped = true; + out_end_trans: ++ if (!is_reloc_root) ++ btrfs_set_last_root_drop_gen(fs_info, trans->transid); ++ + btrfs_end_transaction_throttle(trans); + out_free: + kfree(wc); +diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c +index 80fe313f8461..6bb43c7d5fb4 100644 +--- a/fs/btrfs/extent_io.c ++++ b/fs/btrfs/extent_io.c +@@ -5370,42 +5370,6 @@ int try_release_extent_mapping(struct page *page, gfp_t mask) + return try_release_extent_state(tree, page, mask); + } + +-/* +- * helper function for fiemap, which doesn't want to see any holes. +- * This maps until we find something past 'last' +- */ +-static struct extent_map *get_extent_skip_holes(struct btrfs_inode *inode, +- u64 offset, u64 last) +-{ +- u64 sectorsize = btrfs_inode_sectorsize(inode); +- struct extent_map *em; +- u64 len; +- +- if (offset >= last) +- return NULL; +- +- while (1) { +- len = last - offset; +- if (len == 0) +- break; +- len = ALIGN(len, sectorsize); +- em = btrfs_get_extent_fiemap(inode, offset, len); +- if (IS_ERR(em)) +- return em; +- +- /* if this isn't a hole return it */ +- if (em->block_start != EXTENT_MAP_HOLE) +- return em; +- +- /* this is a hole, advance to the next extent */ +- offset = extent_map_end(em); +- free_extent_map(em); +- if (offset >= last) +- break; +- } +- return NULL; +-} +- + /* + * To cache previous fiemap extent + * +@@ -5435,6 +5399,9 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, + { + int ret = 0; + ++ /* Set at the end of extent_fiemap(). */ ++ ASSERT((flags & FIEMAP_EXTENT_LAST) == 0); ++ + if (!cache->cached) + goto assign; + +@@ -5458,16 +5425,13 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, + * So truly compressed (physical size smaller than logical size) + * extents won't get merged with each other + * +- * 3) Share same flags except FIEMAP_EXTENT_LAST +- * So regular extent won't get merged with prealloc extent ++ * 3) Share same flags + */ + if (cache->offset + cache->len == offset && + cache->phys + cache->len == phys && +- (cache->flags & ~FIEMAP_EXTENT_LAST) == +- (flags & ~FIEMAP_EXTENT_LAST)) { ++ cache->flags == flags) { + cache->len += len; +- cache->flags |= flags; +- goto try_submit_last; ++ return 0; + } + + /* Not mergeable, need to submit cached one */ +@@ -5482,13 +5446,8 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, + cache->phys = phys; + cache->len = len; + cache->flags = flags; +-try_submit_last: +- if (cache->flags & FIEMAP_EXTENT_LAST) { +- ret = fiemap_fill_next_extent(fieinfo, cache->offset, +- cache->phys, cache->len, cache->flags); +- cache->cached = false; +- } +- return ret; ++ ++ return 0; + } + + /* +@@ -5518,215 +5477,534 @@ static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo, + return ret; + } + +-int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo, +- u64 start, u64 len) ++static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path) + { +- int ret = 0; +- u64 off; +- u64 max = start + len; +- u32 flags = 0; +- u32 found_type; +- u64 last; +- u64 last_for_get_extent = 0; +- u64 disko = 0; +- u64 isize = i_size_read(&inode->vfs_inode); +- struct btrfs_key found_key; +- struct extent_map *em = NULL; +- struct extent_state *cached_state = NULL; +- struct btrfs_path *path; +- struct btrfs_root *root = inode->root; +- struct fiemap_cache cache = { 0 }; +- struct ulist *roots; +- struct ulist *tmp_ulist; +- int end = 0; +- u64 em_start = 0; +- u64 em_len = 0; +- u64 em_end = 0; ++ struct extent_buffer *clone; ++ struct btrfs_key key; ++ int slot; ++ int ret; + +- if (len == 0) +- return -EINVAL; ++ path->slots[0]++; ++ if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) ++ return 0; + +- path = btrfs_alloc_path(); +- if (!path) ++ ret = btrfs_next_leaf(inode->root, path); ++ if (ret != 0) ++ return ret; ++ ++ /* ++ * Don't bother with cloning if there are no more file extent items for ++ * our inode. ++ */ ++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); ++ if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY) ++ return 1; ++ ++ /* See the comment at fiemap_search_slot() about why we clone. */ ++ clone = btrfs_clone_extent_buffer(path->nodes[0]); ++ if (!clone) + return -ENOMEM; + +- roots = ulist_alloc(GFP_KERNEL); +- tmp_ulist = ulist_alloc(GFP_KERNEL); +- if (!roots || !tmp_ulist) { +- ret = -ENOMEM; +- goto out_free_ulist; ++ slot = path->slots[0]; ++ btrfs_release_path(path); ++ path->nodes[0] = clone; ++ path->slots[0] = slot; ++ ++ return 0; ++} ++ ++/* ++ * Search for the first file extent item that starts at a given file offset or ++ * the one that starts immediately before that offset. ++ * Returns: 0 on success, < 0 on error, 1 if not found. ++ */ ++static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path, ++ u64 file_offset) ++{ ++ const u64 ino = btrfs_ino(inode); ++ struct btrfs_root *root = inode->root; ++ struct extent_buffer *clone; ++ struct btrfs_key key; ++ int slot; ++ int ret; ++ ++ key.objectid = ino; ++ key.type = BTRFS_EXTENT_DATA_KEY; ++ key.offset = file_offset; ++ ++ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); ++ if (ret < 0) ++ return ret; ++ ++ if (ret > 0 && path->slots[0] > 0) { ++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); ++ if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) ++ path->slots[0]--; ++ } ++ ++ if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { ++ ret = btrfs_next_leaf(root, path); ++ if (ret != 0) ++ return ret; ++ ++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); ++ if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) ++ return 1; + } + + /* +- * We can't initialize that to 'start' as this could miss extents due +- * to extent item merging ++ * We clone the leaf and use it during fiemap. This is because while ++ * using the leaf we do expensive things like checking if an extent is ++ * shared, which can take a long time. In order to prevent blocking ++ * other tasks for too long, we use a clone of the leaf. We have locked ++ * the file range in the inode's io tree, so we know none of our file ++ * extent items can change. This way we avoid blocking other tasks that ++ * want to insert items for other inodes in the same leaf or b+tree ++ * rebalance operations (triggered for example when someone is trying ++ * to push items into this leaf when trying to insert an item in a ++ * neighbour leaf). ++ * We also need the private clone because holding a read lock on an ++ * extent buffer of the subvolume's b+tree will make lockdep unhappy ++ * when we call fiemap_fill_next_extent(), because that may cause a page ++ * fault when filling the user space buffer with fiemap data. + */ +- off = 0; +- start = round_down(start, btrfs_inode_sectorsize(inode)); +- len = round_up(max, btrfs_inode_sectorsize(inode)) - start; ++ clone = btrfs_clone_extent_buffer(path->nodes[0]); ++ if (!clone) ++ return -ENOMEM; ++ ++ slot = path->slots[0]; ++ btrfs_release_path(path); ++ path->nodes[0] = clone; ++ path->slots[0] = slot; ++ ++ return 0; ++} ++ ++/* ++ * Process a range which is a hole or a prealloc extent in the inode's subvolume ++ * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc ++ * extent. The end offset (@end) is inclusive. ++ */ ++static int fiemap_process_hole(struct btrfs_inode *inode, ++ struct fiemap_extent_info *fieinfo, ++ struct fiemap_cache *cache, ++ struct btrfs_backref_shared_cache *backref_cache, ++ u64 disk_bytenr, u64 extent_offset, ++ u64 extent_gen, ++ struct ulist *roots, struct ulist *tmp_ulist, ++ u64 start, u64 end) ++{ ++ const u64 i_size = i_size_read(&inode->vfs_inode); ++ const u64 ino = btrfs_ino(inode); ++ u64 cur_offset = start; ++ u64 last_delalloc_end = 0; ++ u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN; ++ bool checked_extent_shared = false; ++ int ret; + + /* +- * lookup the last file extent. We're not using i_size here +- * because there might be preallocation past i_size ++ * There can be no delalloc past i_size, so don't waste time looking for ++ * it beyond i_size. + */ +- ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1, +- 0); +- if (ret < 0) { +- goto out_free_ulist; +- } else { +- WARN_ON(!ret); +- if (ret == 1) +- ret = 0; +- } ++ while (cur_offset < end && cur_offset < i_size) { ++ u64 delalloc_start; ++ u64 delalloc_end; ++ u64 prealloc_start; ++ u64 prealloc_len = 0; ++ bool delalloc; ++ ++ delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end, ++ &delalloc_start, ++ &delalloc_end); ++ if (!delalloc) ++ break; + +- path->slots[0]--; +- btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); +- found_type = found_key.type; +- +- /* No extents, but there might be delalloc bits */ +- if (found_key.objectid != btrfs_ino(inode) || +- found_type != BTRFS_EXTENT_DATA_KEY) { +- /* have to trust i_size as the end */ +- last = (u64)-1; +- last_for_get_extent = isize; +- } else { + /* +- * remember the start of the last extent. There are a +- * bunch of different factors that go into the length of the +- * extent, so its much less complex to remember where it started ++ * If this is a prealloc extent we have to report every section ++ * of it that has no delalloc. + */ +- last = found_key.offset; +- last_for_get_extent = last + 1; ++ if (disk_bytenr != 0) { ++ if (last_delalloc_end == 0) { ++ prealloc_start = start; ++ prealloc_len = delalloc_start - start; ++ } else { ++ prealloc_start = last_delalloc_end + 1; ++ prealloc_len = delalloc_start - prealloc_start; ++ } ++ } ++ ++ if (prealloc_len > 0) { ++ if (!checked_extent_shared && fieinfo->fi_extents_max) { ++ ret = btrfs_is_data_extent_shared(inode->root, ++ ino, disk_bytenr, ++ extent_gen, roots, ++ tmp_ulist, ++ backref_cache); ++ if (ret < 0) ++ return ret; ++ else if (ret > 0) ++ prealloc_flags |= FIEMAP_EXTENT_SHARED; ++ ++ checked_extent_shared = true; ++ } ++ ret = emit_fiemap_extent(fieinfo, cache, prealloc_start, ++ disk_bytenr + extent_offset, ++ prealloc_len, prealloc_flags); ++ if (ret) ++ return ret; ++ extent_offset += prealloc_len; ++ } ++ ++ ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0, ++ delalloc_end + 1 - delalloc_start, ++ FIEMAP_EXTENT_DELALLOC | ++ FIEMAP_EXTENT_UNKNOWN); ++ if (ret) ++ return ret; ++ ++ last_delalloc_end = delalloc_end; ++ cur_offset = delalloc_end + 1; ++ extent_offset += cur_offset - delalloc_start; ++ cond_resched(); + } +- btrfs_release_path(path); + + /* +- * we might have some extents allocated but more delalloc past those +- * extents. so, we trust isize unless the start of the last extent is +- * beyond isize ++ * Either we found no delalloc for the whole prealloc extent or we have ++ * a prealloc extent that spans i_size or starts at or after i_size. ++ */ ++ if (disk_bytenr != 0 && last_delalloc_end < end) { ++ u64 prealloc_start; ++ u64 prealloc_len; ++ ++ if (last_delalloc_end == 0) { ++ prealloc_start = start; ++ prealloc_len = end + 1 - start; ++ } else { ++ prealloc_start = last_delalloc_end + 1; ++ prealloc_len = end + 1 - prealloc_start; ++ } ++ ++ if (!checked_extent_shared && fieinfo->fi_extents_max) { ++ ret = btrfs_is_data_extent_shared(inode->root, ++ ino, disk_bytenr, ++ extent_gen, roots, ++ tmp_ulist, ++ backref_cache); ++ if (ret < 0) ++ return ret; ++ else if (ret > 0) ++ prealloc_flags |= FIEMAP_EXTENT_SHARED; ++ } ++ ret = emit_fiemap_extent(fieinfo, cache, prealloc_start, ++ disk_bytenr + extent_offset, ++ prealloc_len, prealloc_flags); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int fiemap_find_last_extent_offset(struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ u64 *last_extent_end_ret) ++{ ++ const u64 ino = btrfs_ino(inode); ++ struct btrfs_root *root = inode->root; ++ struct extent_buffer *leaf; ++ struct btrfs_file_extent_item *ei; ++ struct btrfs_key key; ++ u64 disk_bytenr; ++ int ret; ++ ++ /* ++ * Lookup the last file extent. We're not using i_size here because ++ * there might be preallocation past i_size. ++ */ ++ ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0); ++ /* There can't be a file extent item at offset (u64)-1 */ ++ ASSERT(ret != 0); ++ if (ret < 0) ++ return ret; ++ ++ /* ++ * For a non-existing key, btrfs_search_slot() always leaves us at a ++ * slot > 0, except if the btree is empty, which is impossible because ++ * at least it has the inode item for this inode and all the items for ++ * the root inode 256. + */ +- if (last < isize) { +- last = (u64)-1; +- last_for_get_extent = isize; ++ ASSERT(path->slots[0] > 0); ++ path->slots[0]--; ++ leaf = path->nodes[0]; ++ btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); ++ if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) { ++ /* No file extent items in the subvolume tree. */ ++ *last_extent_end_ret = 0; ++ return 0; + } + +- lock_extent_bits(&inode->io_tree, start, start + len - 1, +- &cached_state); ++ /* ++ * For an inline extent, the disk_bytenr is where inline data starts at, ++ * so first check if we have an inline extent item before checking if we ++ * have an implicit hole (disk_bytenr == 0). ++ */ ++ ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); ++ if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) { ++ *last_extent_end_ret = btrfs_file_extent_end(path); ++ return 0; ++ } + +- em = get_extent_skip_holes(inode, start, last_for_get_extent); +- if (!em) +- goto out; +- if (IS_ERR(em)) { +- ret = PTR_ERR(em); ++ /* ++ * Find the last file extent item that is not a hole (when NO_HOLES is ++ * not enabled). This should take at most 2 iterations in the worst ++ * case: we have one hole file extent item at slot 0 of a leaf and ++ * another hole file extent item as the last item in the previous leaf. ++ * This is because we merge file extent items that represent holes. ++ */ ++ disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); ++ while (disk_bytenr == 0) { ++ ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY); ++ if (ret < 0) { ++ return ret; ++ } else if (ret > 0) { ++ /* No file extent items that are not holes. */ ++ *last_extent_end_ret = 0; ++ return 0; ++ } ++ leaf = path->nodes[0]; ++ ei = btrfs_item_ptr(leaf, path->slots[0], ++ struct btrfs_file_extent_item); ++ disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); ++ } ++ ++ *last_extent_end_ret = btrfs_file_extent_end(path); ++ return 0; ++} ++ ++int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo, ++ u64 start, u64 len) ++{ ++ const u64 ino = btrfs_ino(inode); ++ struct extent_state *cached_state = NULL; ++ struct btrfs_path *path; ++ struct btrfs_root *root = inode->root; ++ struct fiemap_cache cache = { 0 }; ++ struct btrfs_backref_shared_cache *backref_cache; ++ struct ulist *roots; ++ struct ulist *tmp_ulist; ++ u64 last_extent_end; ++ u64 prev_extent_end; ++ u64 lockstart; ++ u64 lockend; ++ bool stopped = false; ++ int ret; ++ ++ backref_cache = kzalloc(sizeof(*backref_cache), GFP_KERNEL); ++ path = btrfs_alloc_path(); ++ roots = ulist_alloc(GFP_KERNEL); ++ tmp_ulist = ulist_alloc(GFP_KERNEL); ++ if (!backref_cache || !path || !roots || !tmp_ulist) { ++ ret = -ENOMEM; + goto out; + } + +- while (!end) { +- u64 offset_in_extent = 0; ++ lockstart = round_down(start, btrfs_inode_sectorsize(inode)); ++ lockend = round_up(start + len, btrfs_inode_sectorsize(inode)); ++ prev_extent_end = lockstart; + +- /* break if the extent we found is outside the range */ +- if (em->start >= max || extent_map_end(em) < off) +- break; ++ lock_extent_bits(&inode->io_tree, lockstart, lockend, &cached_state); + +- /* +- * get_extent may return an extent that starts before our +- * requested range. We have to make sure the ranges +- * we return to fiemap always move forward and don't +- * overlap, so adjust the offsets here +- */ +- em_start = max(em->start, off); ++ ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); ++ if (ret < 0) ++ goto out_unlock; ++ btrfs_release_path(path); + ++ path->reada = READA_FORWARD; ++ ret = fiemap_search_slot(inode, path, lockstart); ++ if (ret < 0) { ++ goto out_unlock; ++ } else if (ret > 0) { + /* +- * record the offset from the start of the extent +- * for adjusting the disk offset below. Only do this if the +- * extent isn't compressed since our in ram offset may be past +- * what we have actually allocated on disk. ++ * No file extent item found, but we may have delalloc between ++ * the current offset and i_size. So check for that. + */ +- if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) +- offset_in_extent = em_start - em->start; +- em_end = extent_map_end(em); +- em_len = em_end - em_start; +- flags = 0; +- if (em->block_start < EXTENT_MAP_LAST_BYTE) +- disko = em->block_start + offset_in_extent; +- else +- disko = 0; ++ ret = 0; ++ goto check_eof_delalloc; ++ } ++ ++ while (prev_extent_end < lockend) { ++ struct extent_buffer *leaf = path->nodes[0]; ++ struct btrfs_file_extent_item *ei; ++ struct btrfs_key key; ++ u64 extent_end; ++ u64 extent_len; ++ u64 extent_offset = 0; ++ u64 extent_gen; ++ u64 disk_bytenr = 0; ++ u64 flags = 0; ++ int extent_type; ++ u8 compression; ++ ++ btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); ++ if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) ++ break; ++ ++ extent_end = btrfs_file_extent_end(path); + + /* +- * bump off for our next call to get_extent ++ * The first iteration can leave us at an extent item that ends ++ * before our range's start. Move to the next item. + */ +- off = extent_map_end(em); +- if (off >= max) +- end = 1; +- +- if (em->block_start == EXTENT_MAP_LAST_BYTE) { +- end = 1; +- flags |= FIEMAP_EXTENT_LAST; +- } else if (em->block_start == EXTENT_MAP_INLINE) { +- flags |= (FIEMAP_EXTENT_DATA_INLINE | +- FIEMAP_EXTENT_NOT_ALIGNED); +- } else if (em->block_start == EXTENT_MAP_DELALLOC) { +- flags |= (FIEMAP_EXTENT_DELALLOC | +- FIEMAP_EXTENT_UNKNOWN); +- } else if (fieinfo->fi_extents_max) { +- u64 bytenr = em->block_start - +- (em->start - em->orig_start); ++ if (extent_end <= lockstart) ++ goto next_item; + +- /* +- * As btrfs supports shared space, this information +- * can be exported to userspace tools via +- * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0 +- * then we're just getting a count and we can skip the +- * lookup stuff. +- */ +- ret = btrfs_check_shared(root, btrfs_ino(inode), +- bytenr, roots, tmp_ulist); +- if (ret < 0) +- goto out_free; +- if (ret) +- flags |= FIEMAP_EXTENT_SHARED; +- ret = 0; ++ /* We have in implicit hole (NO_HOLES feature enabled). */ ++ if (prev_extent_end < key.offset) { ++ const u64 range_end = min(key.offset, lockend) - 1; ++ ++ ret = fiemap_process_hole(inode, fieinfo, &cache, ++ backref_cache, 0, 0, 0, ++ roots, tmp_ulist, ++ prev_extent_end, range_end); ++ if (ret < 0) { ++ goto out_unlock; ++ } else if (ret > 0) { ++ /* fiemap_fill_next_extent() told us to stop. */ ++ stopped = true; ++ break; ++ } ++ ++ /* We've reached the end of the fiemap range, stop. */ ++ if (key.offset >= lockend) { ++ stopped = true; ++ break; ++ } + } +- if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) ++ ++ extent_len = extent_end - key.offset; ++ ei = btrfs_item_ptr(leaf, path->slots[0], ++ struct btrfs_file_extent_item); ++ compression = btrfs_file_extent_compression(leaf, ei); ++ extent_type = btrfs_file_extent_type(leaf, ei); ++ extent_gen = btrfs_file_extent_generation(leaf, ei); ++ ++ if (extent_type != BTRFS_FILE_EXTENT_INLINE) { ++ disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); ++ if (compression == BTRFS_COMPRESS_NONE) ++ extent_offset = btrfs_file_extent_offset(leaf, ei); ++ } ++ ++ if (compression != BTRFS_COMPRESS_NONE) + flags |= FIEMAP_EXTENT_ENCODED; +- if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) +- flags |= FIEMAP_EXTENT_UNWRITTEN; + +- free_extent_map(em); +- em = NULL; +- if ((em_start >= last) || em_len == (u64)-1 || +- (last == (u64)-1 && isize <= em_end)) { +- flags |= FIEMAP_EXTENT_LAST; +- end = 1; ++ if (extent_type == BTRFS_FILE_EXTENT_INLINE) { ++ flags |= FIEMAP_EXTENT_DATA_INLINE; ++ flags |= FIEMAP_EXTENT_NOT_ALIGNED; ++ ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0, ++ extent_len, flags); ++ } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { ++ ret = fiemap_process_hole(inode, fieinfo, &cache, ++ backref_cache, ++ disk_bytenr, extent_offset, ++ extent_gen, roots, tmp_ulist, ++ key.offset, extent_end - 1); ++ } else if (disk_bytenr == 0) { ++ /* We have an explicit hole. */ ++ ret = fiemap_process_hole(inode, fieinfo, &cache, ++ backref_cache, 0, 0, 0, ++ roots, tmp_ulist, ++ key.offset, extent_end - 1); ++ } else { ++ /* We have a regular extent. */ ++ if (fieinfo->fi_extents_max) { ++ ret = btrfs_is_data_extent_shared(root, ino, ++ disk_bytenr, ++ extent_gen, ++ roots, ++ tmp_ulist, ++ backref_cache); ++ if (ret < 0) ++ goto out_unlock; ++ else if (ret > 0) ++ flags |= FIEMAP_EXTENT_SHARED; ++ } ++ ++ ret = emit_fiemap_extent(fieinfo, &cache, key.offset, ++ disk_bytenr + extent_offset, ++ extent_len, flags); + } + +- /* now scan forward to see if this is really the last extent. */ +- em = get_extent_skip_holes(inode, off, last_for_get_extent); +- if (IS_ERR(em)) { +- ret = PTR_ERR(em); +- goto out; ++ if (ret < 0) { ++ goto out_unlock; ++ } else if (ret > 0) { ++ /* fiemap_fill_next_extent() told us to stop. */ ++ stopped = true; ++ break; + } +- if (!em) { +- flags |= FIEMAP_EXTENT_LAST; +- end = 1; ++ ++ prev_extent_end = extent_end; ++next_item: ++ if (fatal_signal_pending(current)) { ++ ret = -EINTR; ++ goto out_unlock; + } +- ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko, +- em_len, flags); +- if (ret) { +- if (ret == 1) +- ret = 0; +- goto out_free; ++ ++ ret = fiemap_next_leaf_item(inode, path); ++ if (ret < 0) { ++ goto out_unlock; ++ } else if (ret > 0) { ++ /* No more file extent items for this inode. */ ++ break; + } ++ cond_resched(); + } +-out_free: +- if (!ret) +- ret = emit_last_fiemap_cache(fieinfo, &cache); +- free_extent_map(em); +-out: +- unlock_extent_cached(&inode->io_tree, start, start + len - 1, +- &cached_state); + +-out_free_ulist: ++check_eof_delalloc: ++ /* ++ * Release (and free) the path before emitting any final entries to ++ * fiemap_fill_next_extent() to keep lockdep happy. This is because ++ * once we find no more file extent items exist, we may have a ++ * non-cloned leaf, and fiemap_fill_next_extent() can trigger page ++ * faults when copying data to the user space buffer. ++ */ ++ btrfs_free_path(path); ++ path = NULL; ++ ++ if (!stopped && prev_extent_end < lockend) { ++ ret = fiemap_process_hole(inode, fieinfo, &cache, backref_cache, ++ 0, 0, 0, roots, tmp_ulist, ++ prev_extent_end, lockend - 1); ++ if (ret < 0) ++ goto out_unlock; ++ prev_extent_end = lockend; ++ } ++ ++ if (cache.cached && cache.offset + cache.len >= last_extent_end) { ++ const u64 i_size = i_size_read(&inode->vfs_inode); ++ ++ if (prev_extent_end < i_size) { ++ u64 delalloc_start; ++ u64 delalloc_end; ++ bool delalloc; ++ ++ delalloc = btrfs_find_delalloc_in_range(inode, ++ prev_extent_end, ++ i_size - 1, ++ &delalloc_start, ++ &delalloc_end); ++ if (!delalloc) ++ cache.flags |= FIEMAP_EXTENT_LAST; ++ } else { ++ cache.flags |= FIEMAP_EXTENT_LAST; ++ } ++ } ++ ++ ret = emit_last_fiemap_cache(fieinfo, &cache); ++ ++out_unlock: ++ unlock_extent_cached(&inode->io_tree, lockstart, lockend, &cached_state); ++out: ++ kfree(backref_cache); + btrfs_free_path(path); + ulist_free(roots); + ulist_free(tmp_ulist); +diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c +index c828f971a346..fcc6ce861409 100644 +--- a/fs/btrfs/file-item.c ++++ b/fs/btrfs/file-item.c +@@ -503,7 +503,8 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst + } + + int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, +- struct list_head *list, int search_commit) ++ struct list_head *list, int search_commit, ++ bool nowait) + { + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; +@@ -525,6 +526,7 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, + if (!path) + return -ENOMEM; + ++ path->nowait = nowait; + if (search_commit) { + path->skip_locking = 1; + path->reada = READA_FORWARD; +diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c +index 5a3f6e0d9688..0984aa60c1b5 100644 +--- a/fs/btrfs/file.c ++++ b/fs/btrfs/file.c +@@ -1339,26 +1339,55 @@ static int prepare_uptodate_page(struct inode *inode, + return 0; + } + ++static int get_prepare_fgp_flags(bool nowait) ++{ ++ int fgp_flags; ++ ++ fgp_flags = FGP_LOCK|FGP_ACCESSED|FGP_CREAT; ++ if (nowait) ++ fgp_flags |= FGP_NOWAIT; ++ ++ return fgp_flags; ++} ++ ++static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait) ++{ ++ gfp_t gfp; ++ ++ gfp = btrfs_alloc_write_mask(inode->i_mapping); ++ if (nowait) { ++ gfp &= ~__GFP_DIRECT_RECLAIM; ++ gfp |= GFP_NOWAIT; ++ } ++ ++ return gfp; ++} ++ + /* + * this just gets pages into the page cache and locks them down. + */ + static noinline int prepare_pages(struct inode *inode, struct page **pages, + size_t num_pages, loff_t pos, +- size_t write_bytes, bool force_uptodate) ++ size_t write_bytes, bool force_uptodate, ++ bool nowait) + { + int i; + unsigned long index = pos >> PAGE_SHIFT; +- gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping); ++ gfp_t mask = get_prepare_gfp_flags(inode, nowait); ++ int fgp_flags = get_prepare_fgp_flags(nowait); + int err = 0; + int faili; + + for (i = 0; i < num_pages; i++) { + again: +- pages[i] = find_or_create_page(inode->i_mapping, index + i, +- mask | __GFP_WRITE); ++ pages[i] = pagecache_get_page(inode->i_mapping, index + i, ++ fgp_flags, mask | __GFP_WRITE); + if (!pages[i]) { + faili = i - 1; +- err = -ENOMEM; ++ if (nowait) ++ err = -EAGAIN; ++ else ++ err = -ENOMEM; + goto fail; + } + +@@ -1376,7 +1405,7 @@ static noinline int prepare_pages(struct inode *inode, struct page **pages, + pos + write_bytes, false); + if (err) { + put_page(pages[i]); +- if (err == -EAGAIN) { ++ if (!nowait && err == -EAGAIN) { + err = 0; + goto again; + } +@@ -1411,7 +1440,7 @@ static noinline int + lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, + size_t num_pages, loff_t pos, + size_t write_bytes, +- u64 *lockstart, u64 *lockend, ++ u64 *lockstart, u64 *lockend, bool nowait, + struct extent_state **cached_state) + { + struct btrfs_fs_info *fs_info = inode->root->fs_info; +@@ -1426,8 +1455,20 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, + if (start_pos < inode->vfs_inode.i_size) { + struct btrfs_ordered_extent *ordered; + +- lock_extent_bits(&inode->io_tree, start_pos, last_pos, ++ if (nowait) { ++ if (!try_lock_extent(&inode->io_tree, start_pos, last_pos)) { ++ for (i = 0; i < num_pages; i++) { ++ unlock_page(pages[i]); ++ put_page(pages[i]); ++ } ++ ++ return -EAGAIN; ++ } ++ } else { ++ lock_extent_bits(&inode->io_tree, start_pos, last_pos, + cached_state); ++ } ++ + ordered = btrfs_lookup_ordered_range(inode, start_pos, + last_pos - start_pos + 1); + if (ordered && +@@ -1481,7 +1522,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, + * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0. + */ + int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, +- size_t *write_bytes) ++ size_t *write_bytes, bool nowait) + { + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; +@@ -1500,16 +1541,21 @@ int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, + fs_info->sectorsize) - 1; + num_bytes = lockend - lockstart + 1; + +- btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend, NULL); ++ if (nowait) { ++ if (!btrfs_try_lock_ordered_range(inode, lockstart, lockend)) { ++ btrfs_drew_write_unlock(&root->snapshot_lock); ++ return -EAGAIN; ++ } ++ } else { ++ btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend, NULL); ++ } + ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes, +- NULL, NULL, NULL, false); +- if (ret <= 0) { +- ret = 0; ++ NULL, NULL, NULL, nowait, false); ++ if (ret <= 0) + btrfs_drew_write_unlock(&root->snapshot_lock); +- } else { ++ else + *write_bytes = min_t(size_t, *write_bytes , + num_bytes - pos + lockstart); +- } + unlock_extent(&inode->io_tree, lockstart, lockend); + + return ret; +@@ -1607,8 +1653,10 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + bool force_page_uptodate = false; + loff_t old_isize = i_size_read(inode); + unsigned int ilock_flags = 0; ++ bool nowait = iocb->ki_flags & IOCB_NOWAIT; ++ unsigned int bdp_flags = nowait ? BDP_ASYNC : 0; + +- if (iocb->ki_flags & IOCB_NOWAIT) ++ if (nowait) + ilock_flags |= BTRFS_ILOCK_TRY; + + ret = btrfs_inode_lock(inode, ilock_flags); +@@ -1664,18 +1712,29 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + extent_changeset_release(data_reserved); + ret = btrfs_check_data_free_space(BTRFS_I(inode), + &data_reserved, pos, +- write_bytes); ++ write_bytes, nowait); + if (ret < 0) { ++ int tmp; ++ ++ if (nowait && (ret == -ENOSPC || ret == -EAGAIN)) { ++ ret = -EAGAIN; ++ break; ++ } ++ + /* + * If we don't have to COW at the offset, reserve + * metadata only. write_bytes may get smaller than + * requested here. + */ +- if (btrfs_check_nocow_lock(BTRFS_I(inode), pos, +- &write_bytes) > 0) +- only_release_metadata = true; +- else ++ tmp = btrfs_check_nocow_lock(BTRFS_I(inode), pos, ++ &write_bytes, nowait); ++ if (tmp < 0) ++ ret = tmp; ++ if (tmp > 0) ++ ret = 0; ++ if (ret) + break; ++ only_release_metadata = true; + } + + num_pages = DIV_ROUND_UP(write_bytes + offset, PAGE_SIZE); +@@ -1685,7 +1744,7 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + WARN_ON(reserve_bytes == 0); + ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), + reserve_bytes, +- reserve_bytes, false); ++ reserve_bytes, nowait); + if (ret) { + if (!only_release_metadata) + btrfs_free_reserved_data_space(BTRFS_I(inode), +@@ -1698,6 +1757,10 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + + release_bytes = reserve_bytes; + again: ++ ret = balance_dirty_pages_ratelimited_flags(inode->i_mapping, bdp_flags); ++ if (unlikely(ret)) ++ break; ++ + /* + * This is going to setup the pages array with the number of + * pages we want, so we don't really need to worry about the +@@ -1705,7 +1768,7 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + */ + ret = prepare_pages(inode, pages, num_pages, + pos, write_bytes, +- force_page_uptodate); ++ force_page_uptodate, false); + if (ret) { + btrfs_delalloc_release_extents(BTRFS_I(inode), + reserve_bytes); +@@ -1715,10 +1778,11 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + extents_locked = lock_and_cleanup_extent_if_need( + BTRFS_I(inode), pages, + num_pages, pos, write_bytes, &lockstart, +- &lockend, &cached_state); ++ &lockend, nowait, &cached_state); + if (extents_locked < 0) { +- if (extents_locked == -EAGAIN) ++ if (!nowait && extents_locked == -EAGAIN) + goto again; ++ + btrfs_delalloc_release_extents(BTRFS_I(inode), + reserve_bytes); + ret = extents_locked; +@@ -1801,8 +1865,6 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, + + cond_resched(); + +- balance_dirty_pages_ratelimited(inode->i_mapping); +- + pos += copied; + num_written += copied; + } +@@ -2045,13 +2107,13 @@ ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from, + if (BTRFS_FS_ERROR(inode->root->fs_info)) + return -EROFS; + +- if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) +- return -EOPNOTSUPP; +- + if (sync) + atomic_inc(&inode->sync_writers); + + if (encoded) { ++ if (iocb->ki_flags & IOCB_NOWAIT) ++ return -EOPNOTSUPP; ++ + num_written = btrfs_encoded_write(iocb, from, encoded); + num_sync = encoded->len; + } else if (iocb->ki_flags & IOCB_DIRECT) { +@@ -2380,6 +2442,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) + ret = btrfs_commit_transaction(trans); + out: + ASSERT(list_empty(&ctx.list)); ++ ASSERT(list_empty(&ctx.conflict_inodes)); + err = file_check_and_advance_wb_err(file); + if (!ret) + ret = err; +@@ -3600,22 +3663,281 @@ static long btrfs_fallocate(struct file *file, int mode, + return ret; + } + ++/* ++ * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range ++ * that has unflushed and/or flushing delalloc. There might be other adjacent ++ * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps ++ * looping while it gets adjacent subranges, and merging them together. ++ */ ++static bool find_delalloc_subrange(struct btrfs_inode *inode, u64 start, u64 end, ++ u64 *delalloc_start_ret, u64 *delalloc_end_ret) ++{ ++ const u64 len = end + 1 - start; ++ struct extent_map_tree *em_tree = &inode->extent_tree; ++ struct extent_map *em; ++ u64 em_end; ++ u64 delalloc_len; ++ ++ /* ++ * Search the io tree first for EXTENT_DELALLOC. If we find any, it ++ * means we have delalloc (dirty pages) for which writeback has not ++ * started yet. ++ */ ++ *delalloc_start_ret = start; ++ delalloc_len = count_range_bits(&inode->io_tree, delalloc_start_ret, end, ++ len, EXTENT_DELALLOC, 1); ++ /* ++ * If delalloc was found then *delalloc_start_ret has a sector size ++ * aligned value (rounded down). ++ */ ++ if (delalloc_len > 0) ++ *delalloc_end_ret = *delalloc_start_ret + delalloc_len - 1; ++ ++ /* ++ * Now also check if there's any extent map in the range that does not ++ * map to a hole or prealloc extent. We do this because: ++ * ++ * 1) When delalloc is flushed, the file range is locked, we clear the ++ * EXTENT_DELALLOC bit from the io tree and create an extent map for ++ * an allocated extent. So we might just have been called after ++ * delalloc is flushed and before the ordered extent completes and ++ * inserts the new file extent item in the subvolume's btree; ++ * ++ * 2) We may have an extent map created by flushing delalloc for a ++ * subrange that starts before the subrange we found marked with ++ * EXTENT_DELALLOC in the io tree. ++ */ ++ read_lock(&em_tree->lock); ++ em = lookup_extent_mapping(em_tree, start, len); ++ read_unlock(&em_tree->lock); ++ ++ /* extent_map_end() returns a non-inclusive end offset. */ ++ em_end = em ? extent_map_end(em) : 0; ++ ++ /* ++ * If we have a hole/prealloc extent map, check the next one if this one ++ * ends before our range's end. ++ */ ++ if (em && (em->block_start == EXTENT_MAP_HOLE || ++ test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) && em_end < end) { ++ struct extent_map *next_em; ++ ++ read_lock(&em_tree->lock); ++ next_em = lookup_extent_mapping(em_tree, em_end, len - em_end); ++ read_unlock(&em_tree->lock); ++ ++ free_extent_map(em); ++ em_end = next_em ? extent_map_end(next_em) : 0; ++ em = next_em; ++ } ++ ++ if (em && (em->block_start == EXTENT_MAP_HOLE || ++ test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { ++ free_extent_map(em); ++ em = NULL; ++ } ++ ++ /* ++ * No extent map or one for a hole or prealloc extent. Use the delalloc ++ * range we found in the io tree if we have one. ++ */ ++ if (!em) ++ return (delalloc_len > 0); ++ ++ /* ++ * We don't have any range as EXTENT_DELALLOC in the io tree, so the ++ * extent map is the only subrange representing delalloc. ++ */ ++ if (delalloc_len == 0) { ++ *delalloc_start_ret = em->start; ++ *delalloc_end_ret = min(end, em_end - 1); ++ free_extent_map(em); ++ return true; ++ } ++ ++ /* ++ * The extent map represents a delalloc range that starts before the ++ * delalloc range we found in the io tree. ++ */ ++ if (em->start < *delalloc_start_ret) { ++ *delalloc_start_ret = em->start; ++ /* ++ * If the ranges are adjacent, return a combined range. ++ * Otherwise return the extent map's range. ++ */ ++ if (em_end < *delalloc_start_ret) ++ *delalloc_end_ret = min(end, em_end - 1); ++ ++ free_extent_map(em); ++ return true; ++ } ++ ++ /* ++ * The extent map starts after the delalloc range we found in the io ++ * tree. If it's adjacent, return a combined range, otherwise return ++ * the range found in the io tree. ++ */ ++ if (*delalloc_end_ret + 1 == em->start) ++ *delalloc_end_ret = min(end, em_end - 1); ++ ++ free_extent_map(em); ++ return true; ++} ++ ++/* ++ * Check if there's delalloc in a given range. ++ * ++ * @inode: The inode. ++ * @start: The start offset of the range. It does not need to be ++ * sector size aligned. ++ * @end: The end offset (inclusive value) of the search range. ++ * It does not need to be sector size aligned. ++ * @delalloc_start_ret: Output argument, set to the start offset of the ++ * subrange found with delalloc (may not be sector size ++ * aligned). ++ * @delalloc_end_ret: Output argument, set to he end offset (inclusive value) ++ * of the subrange found with delalloc. ++ * ++ * Returns true if a subrange with delalloc is found within the given range, and ++ * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and ++ * end offsets of the subrange. ++ */ ++bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end, ++ u64 *delalloc_start_ret, u64 *delalloc_end_ret) ++{ ++ u64 cur_offset = round_down(start, inode->root->fs_info->sectorsize); ++ u64 prev_delalloc_end = 0; ++ bool ret = false; ++ ++ while (cur_offset < end) { ++ u64 delalloc_start; ++ u64 delalloc_end; ++ bool delalloc; ++ ++ delalloc = find_delalloc_subrange(inode, cur_offset, end, ++ &delalloc_start, ++ &delalloc_end); ++ if (!delalloc) ++ break; ++ ++ if (prev_delalloc_end == 0) { ++ /* First subrange found. */ ++ *delalloc_start_ret = max(delalloc_start, start); ++ *delalloc_end_ret = delalloc_end; ++ ret = true; ++ } else if (delalloc_start == prev_delalloc_end + 1) { ++ /* Subrange adjacent to the previous one, merge them. */ ++ *delalloc_end_ret = delalloc_end; ++ } else { ++ /* Subrange not adjacent to the previous one, exit. */ ++ break; ++ } ++ ++ prev_delalloc_end = delalloc_end; ++ cur_offset = delalloc_end + 1; ++ cond_resched(); ++ } ++ ++ return ret; ++} ++ ++/* ++ * Check if there's a hole or delalloc range in a range representing a hole (or ++ * prealloc extent) found in the inode's subvolume btree. ++ * ++ * @inode: The inode. ++ * @whence: Seek mode (SEEK_DATA or SEEK_HOLE). ++ * @start: Start offset of the hole region. It does not need to be sector ++ * size aligned. ++ * @end: End offset (inclusive value) of the hole region. It does not ++ * need to be sector size aligned. ++ * @start_ret: Return parameter, used to set the start of the subrange in the ++ * hole that matches the search criteria (seek mode), if such ++ * subrange is found (return value of the function is true). ++ * The value returned here may not be sector size aligned. ++ * ++ * Returns true if a subrange matching the given seek mode is found, and if one ++ * is found, it updates @start_ret with the start of the subrange. ++ */ ++static bool find_desired_extent_in_hole(struct btrfs_inode *inode, int whence, ++ u64 start, u64 end, u64 *start_ret) ++{ ++ u64 delalloc_start; ++ u64 delalloc_end; ++ bool delalloc; ++ ++ delalloc = btrfs_find_delalloc_in_range(inode, start, end, ++ &delalloc_start, &delalloc_end); ++ if (delalloc && whence == SEEK_DATA) { ++ *start_ret = delalloc_start; ++ return true; ++ } ++ ++ if (delalloc && whence == SEEK_HOLE) { ++ /* ++ * We found delalloc but it starts after out start offset. So we ++ * have a hole between our start offset and the delalloc start. ++ */ ++ if (start < delalloc_start) { ++ *start_ret = start; ++ return true; ++ } ++ /* ++ * Delalloc range starts at our start offset. ++ * If the delalloc range's length is smaller than our range, ++ * then it means we have a hole that starts where the delalloc ++ * subrange ends. ++ */ ++ if (delalloc_end < end) { ++ *start_ret = delalloc_end + 1; ++ return true; ++ } ++ ++ /* There's delalloc for the whole range. */ ++ return false; ++ } ++ ++ if (!delalloc && whence == SEEK_HOLE) { ++ *start_ret = start; ++ return true; ++ } ++ ++ /* ++ * No delalloc in the range and we are seeking for data. The caller has ++ * to iterate to the next extent item in the subvolume btree. ++ */ ++ return false; ++} ++ + static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset, + int whence) + { + struct btrfs_fs_info *fs_info = inode->root->fs_info; +- struct extent_map *em = NULL; + struct extent_state *cached_state = NULL; +- loff_t i_size = inode->vfs_inode.i_size; ++ const loff_t i_size = i_size_read(&inode->vfs_inode); ++ const u64 ino = btrfs_ino(inode); ++ struct btrfs_root *root = inode->root; ++ struct btrfs_path *path; ++ struct btrfs_key key; ++ u64 last_extent_end; + u64 lockstart; + u64 lockend; + u64 start; +- u64 len; +- int ret = 0; ++ int ret; ++ bool found = false; + + if (i_size == 0 || offset >= i_size) + return -ENXIO; + ++ /* ++ * Quick path. If the inode has no prealloc extents and its number of ++ * bytes used matches its i_size, then it can not have holes. ++ */ ++ if (whence == SEEK_HOLE && ++ !(inode->flags & BTRFS_INODE_PREALLOC) && ++ inode_get_bytes(&inode->vfs_inode) == i_size) ++ return i_size; ++ + /* + * offset can be negative, in this case we start finding DATA/HOLE from + * the very start of the file. +@@ -3627,45 +3949,165 @@ static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset, + if (lockend <= lockstart) + lockend = lockstart + fs_info->sectorsize; + lockend--; +- len = lockend - lockstart + 1; ++ ++ path = btrfs_alloc_path(); ++ if (!path) ++ return -ENOMEM; ++ path->reada = READA_FORWARD; ++ ++ key.objectid = ino; ++ key.type = BTRFS_EXTENT_DATA_KEY; ++ key.offset = start; ++ ++ last_extent_end = lockstart; + + lock_extent_bits(&inode->io_tree, lockstart, lockend, &cached_state); + ++ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); ++ if (ret < 0) { ++ goto out; ++ } else if (ret > 0 && path->slots[0] > 0) { ++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); ++ if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) ++ path->slots[0]--; ++ } ++ + while (start < i_size) { +- em = btrfs_get_extent_fiemap(inode, start, len); +- if (IS_ERR(em)) { +- ret = PTR_ERR(em); +- em = NULL; +- break; ++ struct extent_buffer *leaf = path->nodes[0]; ++ struct btrfs_file_extent_item *extent; ++ u64 extent_end; ++ ++ if (path->slots[0] >= btrfs_header_nritems(leaf)) { ++ ret = btrfs_next_leaf(root, path); ++ if (ret < 0) ++ goto out; ++ else if (ret > 0) ++ break; ++ ++ leaf = path->nodes[0]; + } + +- if (whence == SEEK_HOLE && +- (em->block_start == EXTENT_MAP_HOLE || +- test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) +- break; +- else if (whence == SEEK_DATA && +- (em->block_start != EXTENT_MAP_HOLE && +- !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) ++ btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); ++ if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + break; + +- start = em->start + em->len; +- free_extent_map(em); +- em = NULL; ++ extent_end = btrfs_file_extent_end(path); ++ ++ /* ++ * In the first iteration we may have a slot that points to an ++ * extent that ends before our start offset, so skip it. ++ */ ++ if (extent_end <= start) { ++ path->slots[0]++; ++ continue; ++ } ++ ++ /* We have an implicit hole, NO_HOLES feature is likely set. */ ++ if (last_extent_end < key.offset) { ++ u64 search_start = last_extent_end; ++ u64 found_start; ++ ++ /* ++ * First iteration, @start matches @offset and it's ++ * within the hole. ++ */ ++ if (start == offset) ++ search_start = offset; ++ ++ found = find_desired_extent_in_hole(inode, whence, ++ search_start, ++ key.offset - 1, ++ &found_start); ++ if (found) { ++ start = found_start; ++ break; ++ } ++ /* ++ * Didn't find data or a hole (due to delalloc) in the ++ * implicit hole range, so need to analyze the extent. ++ */ ++ } ++ ++ extent = btrfs_item_ptr(leaf, path->slots[0], ++ struct btrfs_file_extent_item); ++ ++ if (btrfs_file_extent_disk_bytenr(leaf, extent) == 0 || ++ btrfs_file_extent_type(leaf, extent) == ++ BTRFS_FILE_EXTENT_PREALLOC) { ++ /* ++ * Explicit hole or prealloc extent, search for delalloc. ++ * A prealloc extent is treated like a hole. ++ */ ++ u64 search_start = key.offset; ++ u64 found_start; ++ ++ /* ++ * First iteration, @start matches @offset and it's ++ * within the hole. ++ */ ++ if (start == offset) ++ search_start = offset; ++ ++ found = find_desired_extent_in_hole(inode, whence, ++ search_start, ++ extent_end - 1, ++ &found_start); ++ if (found) { ++ start = found_start; ++ break; ++ } ++ /* ++ * Didn't find data or a hole (due to delalloc) in the ++ * implicit hole range, so need to analyze the next ++ * extent item. ++ */ ++ } else { ++ /* ++ * Found a regular or inline extent. ++ * If we are seeking for data, adjust the start offset ++ * and stop, we're done. ++ */ ++ if (whence == SEEK_DATA) { ++ start = max_t(u64, key.offset, offset); ++ found = true; ++ break; ++ } ++ /* ++ * Else, we are seeking for a hole, check the next file ++ * extent item. ++ */ ++ } ++ ++ start = extent_end; ++ last_extent_end = extent_end; ++ path->slots[0]++; ++ if (fatal_signal_pending(current)) { ++ ret = -EINTR; ++ goto out; ++ } + cond_resched(); + } +- free_extent_map(em); ++ ++ /* We have an implicit hole from the last extent found up to i_size. */ ++ if (!found && start < i_size) { ++ found = find_desired_extent_in_hole(inode, whence, start, ++ i_size - 1, &start); ++ if (!found) ++ start = i_size; ++ } ++ ++out: + unlock_extent_cached(&inode->io_tree, lockstart, lockend, + &cached_state); +- if (ret) { +- offset = ret; +- } else { +- if (whence == SEEK_DATA && start >= i_size) +- offset = -ENXIO; +- else +- offset = min_t(loff_t, start, i_size); +- } ++ btrfs_free_path(path); ++ ++ if (ret < 0) ++ return ret; ++ ++ if (whence == SEEK_DATA && start >= i_size) ++ return -ENXIO; + +- return offset; ++ return min_t(loff_t, start, i_size); + } + + static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence) +@@ -3693,7 +4135,7 @@ static int btrfs_file_open(struct inode *inode, struct file *filp) + { + int ret; + +- filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC; ++ filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC | FMODE_BUF_WASYNC; + + ret = fsverity_file_open(inode, filp); + if (ret) +diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c +index 996da650ecdc..e6ffe519e5c4 100644 +--- a/fs/btrfs/free-space-cache.c ++++ b/fs/btrfs/free-space-cache.c +@@ -914,6 +914,8 @@ static int copy_free_space_cache(struct btrfs_block_group *block_group, + return ret; + } + ++static struct lock_class_key btrfs_free_space_inode_key; ++ + int load_free_space_cache(struct btrfs_block_group *block_group) + { + struct btrfs_fs_info *fs_info = block_group->fs_info; +@@ -983,6 +985,14 @@ int load_free_space_cache(struct btrfs_block_group *block_group) + } + spin_unlock(&block_group->lock); + ++ /* ++ * Reinitialize the class of struct inode's mapping->invalidate_lock for ++ * free space inodes to prevent false positives related to locks for normal ++ * inodes. ++ */ ++ lockdep_set_class(&(&inode->i_data)->invalidate_lock, ++ &btrfs_free_space_inode_key); ++ + ret = __load_free_space_cache(fs_info->tree_root, inode, &tmp_ctl, + path, block_group->start); + btrfs_free_path(path); +diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c +index 1372210869b1..24c52c459748 100644 +--- a/fs/btrfs/inode.c ++++ b/fs/btrfs/inode.c +@@ -1666,7 +1666,7 @@ static noinline int run_delalloc_zoned(struct btrfs_inode *inode, + } + + static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, +- u64 bytenr, u64 num_bytes) ++ u64 bytenr, u64 num_bytes, bool nowait) + { + struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bytenr); + struct btrfs_ordered_sum *sums; +@@ -1674,7 +1674,8 @@ static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, + LIST_HEAD(list); + + ret = btrfs_lookup_csums_range(csum_root, bytenr, +- bytenr + num_bytes - 1, &list, 0); ++ bytenr + num_bytes - 1, &list, 0, ++ nowait); + if (ret == 0 && list_empty(&list)) + return 0; + +@@ -1800,6 +1801,7 @@ static int can_nocow_file_extent(struct btrfs_path *path, + u8 extent_type; + int can_nocow = 0; + int ret = 0; ++ bool nowait = path->nowait; + + fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, fi); +@@ -1876,7 +1878,8 @@ static int can_nocow_file_extent(struct btrfs_path *path, + * Force COW if csums exist in the range. This ensures that csums for a + * given extent are either valid or do not exist. + */ +- ret = csum_exist_in_range(root->fs_info, args->disk_bytenr, args->num_bytes); ++ ret = csum_exist_in_range(root->fs_info, args->disk_bytenr, args->num_bytes, ++ nowait); + WARN_ON_ONCE(ret > 0 && is_freespace_inode); + if (ret != 0) + goto out; +@@ -3225,6 +3228,8 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) + clear_bits |= EXTENT_DELALLOC_NEW; + + freespace_inode = btrfs_is_free_space_inode(inode); ++ if (!freespace_inode) ++ btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent); + + if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { + ret = -EIO; +@@ -4878,9 +4883,9 @@ int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len, + block_end = block_start + blocksize - 1; + + ret = btrfs_check_data_free_space(inode, &data_reserved, block_start, +- blocksize); ++ blocksize, false); + if (ret < 0) { +- if (btrfs_check_nocow_lock(inode, block_start, &write_bytes) > 0) { ++ if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) { + /* For nocow case, no need to reserve data space */ + only_release_metadata = true; + } else { +@@ -7065,133 +7070,6 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, + return em; + } + +-struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, +- u64 start, u64 len) +-{ +- struct extent_map *em; +- struct extent_map *hole_em = NULL; +- u64 delalloc_start = start; +- u64 end; +- u64 delalloc_len; +- u64 delalloc_end; +- int err = 0; +- +- em = btrfs_get_extent(inode, NULL, 0, start, len); +- if (IS_ERR(em)) +- return em; +- /* +- * If our em maps to: +- * - a hole or +- * - a pre-alloc extent, +- * there might actually be delalloc bytes behind it. +- */ +- if (em->block_start != EXTENT_MAP_HOLE && +- !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) +- return em; +- else +- hole_em = em; +- +- /* check to see if we've wrapped (len == -1 or similar) */ +- end = start + len; +- if (end < start) +- end = (u64)-1; +- else +- end -= 1; +- +- em = NULL; +- +- /* ok, we didn't find anything, lets look for delalloc */ +- delalloc_len = count_range_bits(&inode->io_tree, &delalloc_start, +- end, len, EXTENT_DELALLOC, 1); +- delalloc_end = delalloc_start + delalloc_len; +- if (delalloc_end < delalloc_start) +- delalloc_end = (u64)-1; +- +- /* +- * We didn't find anything useful, return the original results from +- * get_extent() +- */ +- if (delalloc_start > end || delalloc_end <= start) { +- em = hole_em; +- hole_em = NULL; +- goto out; +- } +- +- /* +- * Adjust the delalloc_start to make sure it doesn't go backwards from +- * the start they passed in +- */ +- delalloc_start = max(start, delalloc_start); +- delalloc_len = delalloc_end - delalloc_start; +- +- if (delalloc_len > 0) { +- u64 hole_start; +- u64 hole_len; +- const u64 hole_end = extent_map_end(hole_em); +- +- em = alloc_extent_map(); +- if (!em) { +- err = -ENOMEM; +- goto out; +- } +- +- ASSERT(hole_em); +- /* +- * When btrfs_get_extent can't find anything it returns one +- * huge hole +- * +- * Make sure what it found really fits our range, and adjust to +- * make sure it is based on the start from the caller +- */ +- if (hole_end <= start || hole_em->start > end) { +- free_extent_map(hole_em); +- hole_em = NULL; +- } else { +- hole_start = max(hole_em->start, start); +- hole_len = hole_end - hole_start; +- } +- +- if (hole_em && delalloc_start > hole_start) { +- /* +- * Our hole starts before our delalloc, so we have to +- * return just the parts of the hole that go until the +- * delalloc starts +- */ +- em->len = min(hole_len, delalloc_start - hole_start); +- em->start = hole_start; +- em->orig_start = hole_start; +- /* +- * Don't adjust block start at all, it is fixed at +- * EXTENT_MAP_HOLE +- */ +- em->block_start = hole_em->block_start; +- em->block_len = hole_len; +- if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) +- set_bit(EXTENT_FLAG_PREALLOC, &em->flags); +- } else { +- /* +- * Hole is out of passed range or it starts after +- * delalloc range +- */ +- em->start = delalloc_start; +- em->len = delalloc_len; +- em->orig_start = delalloc_start; +- em->block_start = EXTENT_MAP_DELALLOC; +- em->block_len = delalloc_len; +- } +- } else { +- return hole_em; +- } +-out: +- +- free_extent_map(hole_em); +- if (err) { +- free_extent_map(em); +- return ERR_PTR(err); +- } +- return em; +-} +- + static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, + const u64 start, + const u64 len, +@@ -7292,7 +7170,7 @@ static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr) + */ + noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, + u64 *orig_start, u64 *orig_block_len, +- u64 *ram_bytes, bool strict) ++ u64 *ram_bytes, bool nowait, bool strict) + { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + struct can_nocow_file_extent_args nocow_args = { 0 }; +@@ -7308,6 +7186,7 @@ noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; ++ path->nowait = nowait; + + ret = btrfs_lookup_file_extent(NULL, root, path, + btrfs_ino(BTRFS_I(inode)), offset, 0); +@@ -7577,7 +7456,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, + block_start = em->block_start + (start - em->start); + + if (can_nocow_extent(inode, start, &len, &orig_start, +- &orig_block_len, &ram_bytes, false) == 1) { ++ &orig_block_len, &ram_bytes, false, false) == 1) { + bg = btrfs_inc_nocow_writers(fs_info, block_start); + if (bg) + can_nocow = true; +@@ -7762,7 +7641,7 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, + if (write && !(flags & IOMAP_NOWAIT)) { + ret = btrfs_check_data_free_space(BTRFS_I(inode), + &dio_data->data_reserved, +- start, data_alloc_len); ++ start, data_alloc_len, false); + if (!ret) + dio_data->data_space_reserved = true; + else if (ret && !(BTRFS_I(inode)->flags & +@@ -8259,6 +8138,25 @@ static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + if (ret) + return ret; + ++ /* ++ * fiemap_prep() called filemap_write_and_wait() for the whole possible ++ * file range (0 to LLONG_MAX), but that is not enough if we have ++ * compression enabled. The first filemap_fdatawrite_range() only kicks ++ * in the compression of data (in an async thread) and will return ++ * before the compression is done and writeback is started. A second ++ * filemap_fdatawrite_range() is needed to wait for the compression to ++ * complete and writeback to start. We also need to wait for ordered ++ * extents to complete, because our fiemap implementation uses mainly ++ * file extent items to list the extents, searching for extent maps ++ * only for file ranges with holes or prealloc extents to figure out ++ * if we have delalloc in those ranges. ++ */ ++ if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { ++ ret = btrfs_wait_ordered_range(inode, 0, LLONG_MAX); ++ if (ret) ++ return ret; ++ } ++ + return extent_fiemap(BTRFS_I(inode), fieinfo, start, len); + } + +@@ -8959,6 +8857,7 @@ void btrfs_destroy_inode(struct inode *vfs_inode) + struct btrfs_ordered_extent *ordered; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + struct btrfs_root *root = inode->root; ++ bool freespace_inode; + + WARN_ON(!hlist_empty(&vfs_inode->i_dentry)); + WARN_ON(vfs_inode->i_data.nrpages); +@@ -8980,6 +8879,12 @@ void btrfs_destroy_inode(struct inode *vfs_inode) + if (!root) + return; + ++ /* ++ * If this is a free space inode do not take the ordered extents lockdep ++ * map. ++ */ ++ freespace_inode = btrfs_is_free_space_inode(inode); ++ + while (1) { + ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); + if (!ordered) +@@ -8988,6 +8893,10 @@ void btrfs_destroy_inode(struct inode *vfs_inode) + btrfs_err(root->fs_info, + "found ordered extent %llu %llu on inode cleanup", + ordered->file_offset, ordered->num_bytes); ++ ++ if (!freespace_inode) ++ btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent); ++ + btrfs_remove_ordered_extent(inode, ordered); + btrfs_put_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); +@@ -11242,7 +11151,7 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, + free_extent_map(em); + em = NULL; + +- ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, true); ++ ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, false, true); + if (ret < 0) { + goto out; + } else if (ret) { +diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c +index 9063072b399b..d6c88922d3e2 100644 +--- a/fs/btrfs/locking.c ++++ b/fs/btrfs/locking.c +@@ -285,6 +285,29 @@ struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) + return eb; + } + ++/* ++ * Loop around taking references on and locking the root node of the tree in ++ * nowait mode until we end up with a lock on the root node or returning to ++ * avoid blocking. ++ * ++ * Return: root extent buffer with read lock held or -EWOULDBLOCK. ++ */ ++struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root) ++{ ++ struct extent_buffer *eb; ++ ++ while (1) { ++ eb = btrfs_root_node(root); ++ if (!btrfs_try_tree_read_lock(eb)) ++ return ERR_PTR(-EAGAIN); ++ if (eb == root->node) ++ break; ++ btrfs_tree_read_unlock(eb); ++ free_extent_buffer(eb); ++ } ++ return eb; ++} ++ + /* + * DREW locks + * ========== +diff --git a/fs/btrfs/locking.h b/fs/btrfs/locking.h +index ab268be09bb5..490c7a79e995 100644 +--- a/fs/btrfs/locking.h ++++ b/fs/btrfs/locking.h +@@ -94,6 +94,7 @@ int btrfs_try_tree_read_lock(struct extent_buffer *eb); + int btrfs_try_tree_write_lock(struct extent_buffer *eb); + struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root); + struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root); ++struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root); + + #ifdef CONFIG_BTRFS_DEBUG + static inline void btrfs_assert_tree_write_locked(struct extent_buffer *eb) +diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c +index 1952ac85222c..b1676cb818bb 100644 +--- a/fs/btrfs/ordered-data.c ++++ b/fs/btrfs/ordered-data.c +@@ -524,7 +524,15 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *node; + bool pending; ++ bool freespace_inode; + ++ /* ++ * If this is a free space inode the thread has not acquired the ordered ++ * extents lockdep map. ++ */ ++ freespace_inode = btrfs_is_free_space_inode(btrfs_inode); ++ ++ btrfs_lockdep_acquire(fs_info, btrfs_trans_pending_ordered); + /* This is paired with btrfs_add_ordered_extent. */ + spin_lock(&btrfs_inode->lock); + btrfs_mod_outstanding_extents(btrfs_inode, -1); +@@ -580,6 +588,8 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, + } + } + ++ btrfs_lockdep_release(fs_info, btrfs_trans_pending_ordered); ++ + spin_lock(&root->ordered_extent_lock); + list_del_init(&entry->root_extent_list); + root->nr_ordered_extents--; +@@ -594,6 +604,8 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, + } + spin_unlock(&root->ordered_extent_lock); + wake_up(&entry->wait); ++ if (!freespace_inode) ++ btrfs_lockdep_release(fs_info, btrfs_ordered_extent); + } + + static void btrfs_run_ordered_extent_work(struct btrfs_work *work) +@@ -712,9 +724,16 @@ void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait) + u64 start = entry->file_offset; + u64 end = start + entry->num_bytes - 1; + struct btrfs_inode *inode = BTRFS_I(entry->inode); ++ bool freespace_inode; + + trace_btrfs_ordered_extent_start(inode, entry); + ++ /* ++ * If this is a free space inode do not take the ordered extents lockdep ++ * map. ++ */ ++ freespace_inode = btrfs_is_free_space_inode(inode); ++ + /* + * pages in the range can be dirty, clean or writeback. We + * start IO on any dirty ones so the wait doesn't stall waiting +@@ -723,6 +742,8 @@ void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait) + if (!test_bit(BTRFS_ORDERED_DIRECT, &entry->flags)) + filemap_fdatawrite_range(inode->vfs_inode.i_mapping, start, end); + if (wait) { ++ if (!freespace_inode) ++ btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent); + wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, + &entry->flags)); + } +@@ -1041,6 +1062,34 @@ void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, + } + } + ++/* ++ * btrfs_try_lock_ordered_range - lock the passed range and ensure all pending ++ * ordered extents in it are run to completion in nowait mode. ++ * ++ * @inode: Inode whose ordered tree is to be searched ++ * @start: Beginning of range to flush ++ * @end: Last byte of range to lock ++ * ++ * This function returns 1 if btrfs_lock_ordered_range does not return any ++ * extents, otherwise 0. ++ */ ++int btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end) ++{ ++ struct btrfs_ordered_extent *ordered; ++ ++ if (!try_lock_extent(&inode->io_tree, start, end)) ++ return 0; ++ ++ ordered = btrfs_lookup_ordered_range(inode, start, end - start + 1); ++ if (!ordered) ++ return 1; ++ ++ btrfs_put_ordered_extent(ordered); ++ unlock_extent(&inode->io_tree, start, end); ++ return 0; ++} ++ ++ + static int clone_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pos, + u64 len) + { +diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h +index 87792f85e2c4..ec27ebf0af4b 100644 +--- a/fs/btrfs/ordered-data.h ++++ b/fs/btrfs/ordered-data.h +@@ -218,6 +218,7 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, + void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, + u64 end, + struct extent_state **cached_state); ++int btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end); + int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre, + u64 post); + int __init ordered_data_init(void); +diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c +index 45c02aba2492..dfc3f6c04b13 100644 +--- a/fs/btrfs/relocation.c ++++ b/fs/btrfs/relocation.c +@@ -4339,7 +4339,7 @@ int btrfs_reloc_clone_csums(struct btrfs_inode *inode, u64 file_pos, u64 len) + disk_bytenr = file_pos + inode->index_cnt; + csum_root = btrfs_csum_root(fs_info, disk_bytenr); + ret = btrfs_lookup_csums_range(csum_root, disk_bytenr, +- disk_bytenr + len - 1, &list, 0); ++ disk_bytenr + len - 1, &list, 0, false); + if (ret) + goto out; + +diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c +index 3afe5fa50a63..1cb3eed8b917 100644 +--- a/fs/btrfs/scrub.c ++++ b/fs/btrfs/scrub.c +@@ -3077,7 +3077,7 @@ static int scrub_raid56_data_stripe_for_parity(struct scrub_ctx *sctx, + + ret = btrfs_lookup_csums_range(csum_root, extent_start, + extent_start + extent_size - 1, +- &sctx->csum_list, 1); ++ &sctx->csum_list, 1, false); + if (ret) { + scrub_parity_mark_sectors_error(sparity, extent_start, + extent_size); +@@ -3303,7 +3303,7 @@ static int scrub_simple_mirror(struct scrub_ctx *sctx, + if (extent_flags & BTRFS_EXTENT_FLAG_DATA) { + ret = btrfs_lookup_csums_range(csum_root, cur_logical, + cur_logical + scrub_len - 1, +- &sctx->csum_list, 1); ++ &sctx->csum_list, 1, false); + if (ret) + break; + } +diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c +index 435559ba94fa..a9d4bd374462 100644 +--- a/fs/btrfs/space-info.c ++++ b/fs/btrfs/space-info.c +@@ -1737,7 +1737,8 @@ int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes, + int ret; + + ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA || +- flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE); ++ flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE || ++ flush == BTRFS_RESERVE_NO_FLUSH); + ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA); + + ret = __reserve_bytes(fs_info, data_sinfo, bytes, flush); +diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c +index 0bec10740ad3..6e3b2cb6a04a 100644 +--- a/fs/btrfs/transaction.c ++++ b/fs/btrfs/transaction.c +@@ -313,6 +313,8 @@ static noinline int join_transaction(struct btrfs_fs_info *fs_info, + atomic_inc(&cur_trans->num_writers); + extwriter_counter_inc(cur_trans, type); + spin_unlock(&fs_info->trans_lock); ++ btrfs_lockdep_acquire(fs_info, btrfs_trans_num_writers); ++ btrfs_lockdep_acquire(fs_info, btrfs_trans_num_extwriters); + return 0; + } + spin_unlock(&fs_info->trans_lock); +@@ -334,16 +336,23 @@ static noinline int join_transaction(struct btrfs_fs_info *fs_info, + if (!cur_trans) + return -ENOMEM; + ++ btrfs_lockdep_acquire(fs_info, btrfs_trans_num_writers); ++ btrfs_lockdep_acquire(fs_info, btrfs_trans_num_extwriters); ++ + spin_lock(&fs_info->trans_lock); + if (fs_info->running_transaction) { + /* + * someone started a transaction after we unlocked. Make sure + * to redo the checks above + */ ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + kfree(cur_trans); + goto loop; + } else if (BTRFS_FS_ERROR(fs_info)) { + spin_unlock(&fs_info->trans_lock); ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + kfree(cur_trans); + return -EROFS; + } +@@ -541,6 +550,7 @@ static void wait_current_trans(struct btrfs_fs_info *fs_info) + refcount_inc(&cur_trans->use_count); + spin_unlock(&fs_info->trans_lock); + ++ btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + wait_event(fs_info->transaction_wait, + cur_trans->state >= TRANS_STATE_UNBLOCKED || + TRANS_ABORTED(cur_trans)); +@@ -859,6 +869,15 @@ static noinline void wait_for_commit(struct btrfs_transaction *commit, + u64 transid = commit->transid; + bool put = false; + ++ /* ++ * At the moment this function is called with min_state either being ++ * TRANS_STATE_COMPLETED or TRANS_STATE_SUPER_COMMITTED. ++ */ ++ if (min_state == TRANS_STATE_COMPLETED) ++ btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); ++ else ++ btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); ++ + while (1) { + wait_event(commit->commit_wait, commit->state >= min_state); + if (put) +@@ -1022,6 +1041,10 @@ static int __btrfs_end_transaction(struct btrfs_trans_handle *trans, + extwriter_counter_dec(cur_trans, trans->type); + + cond_wake_up(&cur_trans->writer_wait); ++ ++ btrfs_lockdep_release(info, btrfs_trans_num_extwriters); ++ btrfs_lockdep_release(info, btrfs_trans_num_writers); ++ + btrfs_put_transaction(cur_trans); + + if (current->journal_info == trans) +@@ -1967,6 +1990,7 @@ void btrfs_commit_transaction_async(struct btrfs_trans_handle *trans) + * Wait for the current transaction commit to start and block + * subsequent transaction joins + */ ++ btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); + wait_event(fs_info->transaction_blocked_wait, + cur_trans->state >= TRANS_STATE_COMMIT_START || + TRANS_ABORTED(cur_trans)); +@@ -1994,6 +2018,12 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) + if (cur_trans == fs_info->running_transaction) { + cur_trans->state = TRANS_STATE_COMMIT_DOING; + spin_unlock(&fs_info->trans_lock); ++ ++ /* ++ * The thread has already released the lockdep map as reader ++ * already in btrfs_commit_transaction(). ++ */ ++ btrfs_might_wait_for_event(fs_info, btrfs_trans_num_writers); + wait_event(cur_trans->writer_wait, + atomic_read(&cur_trans->num_writers) == 1); + +@@ -2118,12 +2148,12 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + ktime_t interval; + + ASSERT(refcount_read(&trans->use_count) == 1); ++ btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); + + /* Stop the commit early if ->aborted is set */ + if (TRANS_ABORTED(cur_trans)) { + ret = cur_trans->aborted; +- btrfs_end_transaction(trans); +- return ret; ++ goto lockdep_trans_commit_start_release; + } + + btrfs_trans_release_metadata(trans); +@@ -2140,10 +2170,8 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + * Any running threads may add more while we are here. + */ + ret = btrfs_run_delayed_refs(trans, 0); +- if (ret) { +- btrfs_end_transaction(trans); +- return ret; +- } ++ if (ret) ++ goto lockdep_trans_commit_start_release; + } + + btrfs_create_pending_block_groups(trans); +@@ -2172,10 +2200,8 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + if (run_it) { + ret = btrfs_start_dirty_block_groups(trans); +- if (ret) { +- btrfs_end_transaction(trans); +- return ret; +- } ++ if (ret) ++ goto lockdep_trans_commit_start_release; + } + } + +@@ -2190,6 +2216,9 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + if (trans->in_fsync) + want_state = TRANS_STATE_SUPER_COMMITTED; ++ ++ btrfs_trans_state_lockdep_release(fs_info, ++ BTRFS_LOCKDEP_TRANS_COMMIT_START); + ret = btrfs_end_transaction(trans); + wait_for_commit(cur_trans, want_state); + +@@ -2203,6 +2232,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + cur_trans->state = TRANS_STATE_COMMIT_START; + wake_up(&fs_info->transaction_blocked_wait); ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); + + if (cur_trans->list.prev != &fs_info->trans_list) { + enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED; +@@ -2222,7 +2252,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + btrfs_put_transaction(prev_trans); + if (ret) +- goto cleanup_transaction; ++ goto lockdep_release; + } else { + spin_unlock(&fs_info->trans_lock); + } +@@ -2236,7 +2266,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + */ + if (BTRFS_FS_ERROR(fs_info)) { + ret = -EROFS; +- goto cleanup_transaction; ++ goto lockdep_release; + } + } + +@@ -2250,19 +2280,28 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + ret = btrfs_start_delalloc_flush(fs_info); + if (ret) +- goto cleanup_transaction; ++ goto lockdep_release; + + ret = btrfs_run_delayed_items(trans); + if (ret) +- goto cleanup_transaction; ++ goto lockdep_release; + ++ /* ++ * The thread has started/joined the transaction thus it holds the ++ * lockdep map as a reader. It has to release it before acquiring the ++ * lockdep map as a writer. ++ */ ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); ++ btrfs_might_wait_for_event(fs_info, btrfs_trans_num_extwriters); + wait_event(cur_trans->writer_wait, + extwriter_counter_read(cur_trans) == 0); + + /* some pending stuffs might be added after the previous flush. */ + ret = btrfs_run_delayed_items(trans); +- if (ret) ++ if (ret) { ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + goto cleanup_transaction; ++ } + + btrfs_wait_delalloc_flush(fs_info); + +@@ -2271,6 +2310,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + * transaction. Otherwise if this transaction commits before the ordered + * extents complete we lose logged data after a power failure. + */ ++ btrfs_might_wait_for_event(fs_info, btrfs_trans_pending_ordered); + wait_event(cur_trans->pending_wait, + atomic_read(&cur_trans->pending_ordered) == 0); + +@@ -2284,9 +2324,27 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + add_pending_snapshot(trans); + cur_trans->state = TRANS_STATE_COMMIT_DOING; + spin_unlock(&fs_info->trans_lock); ++ ++ /* ++ * The thread has started/joined the transaction thus it holds the ++ * lockdep map as a reader. It has to release it before acquiring the ++ * lockdep map as a writer. ++ */ ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); ++ btrfs_might_wait_for_event(fs_info, btrfs_trans_num_writers); + wait_event(cur_trans->writer_wait, + atomic_read(&cur_trans->num_writers) == 1); + ++ /* ++ * Make lockdep happy by acquiring the state locks after ++ * btrfs_trans_num_writers is released. If we acquired the state locks ++ * before releasing the btrfs_trans_num_writers lock then lockdep would ++ * complain because we did not follow the reverse order unlocking rule. ++ */ ++ btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); ++ btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); ++ btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); ++ + /* + * We've started the commit, clear the flag in case we were triggered to + * do an async commit but somebody else started before the transaction +@@ -2296,6 +2354,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + if (TRANS_ABORTED(cur_trans)) { + ret = cur_trans->aborted; ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + goto scrub_continue; + } + /* +@@ -2430,6 +2489,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + mutex_unlock(&fs_info->reloc_mutex); + + wake_up(&fs_info->transaction_wait); ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + + ret = btrfs_write_and_wait_transaction(trans); + if (ret) { +@@ -2461,6 +2521,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + */ + cur_trans->state = TRANS_STATE_SUPER_COMMITTED; + wake_up(&cur_trans->commit_wait); ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + + btrfs_finish_extent_commit(trans); + +@@ -2474,6 +2535,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + */ + cur_trans->state = TRANS_STATE_COMPLETED; + wake_up(&cur_trans->commit_wait); ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + + spin_lock(&fs_info->trans_lock); + list_del_init(&cur_trans->list); +@@ -2502,7 +2564,10 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + + unlock_reloc: + mutex_unlock(&fs_info->reloc_mutex); ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + scrub_continue: ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + btrfs_scrub_continue(fs_info); + cleanup_transaction: + btrfs_trans_release_metadata(trans); +@@ -2515,6 +2580,16 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) + cleanup_transaction(trans, ret); + + return ret; ++ ++lockdep_release: ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); ++ btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); ++ goto cleanup_transaction; ++ ++lockdep_trans_commit_start_release: ++ btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); ++ btrfs_end_transaction(trans); ++ return ret; + } + + /* +diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c +index 9205c4a5ca81..96ac72de3ee1 100644 +--- a/fs/btrfs/tree-log.c ++++ b/fs/btrfs/tree-log.c +@@ -22,6 +22,8 @@ + #include "zoned.h" + #include "inode-item.h" + ++#define MAX_CONFLICT_INODES 10 ++ + /* magic values for the inode_only field in btrfs_log_inode: + * + * LOG_INODE_ALL means to log everything +@@ -31,8 +33,6 @@ + enum { + LOG_INODE_ALL, + LOG_INODE_EXISTS, +- LOG_OTHER_INODE, +- LOG_OTHER_INODE_ALL, + }; + + /* +@@ -801,7 +801,7 @@ static noinline int replay_one_extent(struct btrfs_trans_handle *trans, + + ret = btrfs_lookup_csums_range(root->log_root, + csum_start, csum_end - 1, +- &ordered_sums, 0); ++ &ordered_sums, 0, false); + if (ret) + goto out; + /* +@@ -4135,6 +4135,56 @@ static noinline int log_dir_items(struct btrfs_trans_handle *trans, + return err; + } + ++/* ++ * If the inode was logged before and it was evicted, then its ++ * last_dir_index_offset is (u64)-1, so we don't the value of the last index ++ * key offset. If that's the case, search for it and update the inode. This ++ * is to avoid lookups in the log tree every time we try to insert a dir index ++ * key from a leaf changed in the current transaction, and to allow us to always ++ * do batch insertions of dir index keys. ++ */ ++static int update_last_dir_index_offset(struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ const struct btrfs_log_ctx *ctx) ++{ ++ const u64 ino = btrfs_ino(inode); ++ struct btrfs_key key; ++ int ret; ++ ++ lockdep_assert_held(&inode->log_mutex); ++ ++ if (inode->last_dir_index_offset != (u64)-1) ++ return 0; ++ ++ if (!ctx->logged_before) { ++ inode->last_dir_index_offset = BTRFS_DIR_START_INDEX - 1; ++ return 0; ++ } ++ ++ key.objectid = ino; ++ key.type = BTRFS_DIR_INDEX_KEY; ++ key.offset = (u64)-1; ++ ++ ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); ++ if (ret <= 0) ++ goto out; ++ ++ ret = 0; ++ inode->last_dir_index_offset = BTRFS_DIR_START_INDEX - 1; ++ ++ if (path->slots[0] == 0) ++ goto out; ++ ++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); ++ if (key.objectid == ino && key.type == BTRFS_DIR_INDEX_KEY) ++ inode->last_dir_index_offset = key.offset; ++ ++out: ++ btrfs_release_path(path); ++ ++ return ret; ++} ++ + /* + * logging directories is very similar to logging inodes, We find all the items + * from the current transaction and write them to the log. +@@ -4157,6 +4207,10 @@ static noinline int log_directory_changes(struct btrfs_trans_handle *trans, + u64 max_key; + int ret; + ++ ret = update_last_dir_index_offset(inode, path, ctx); ++ if (ret) ++ return ret; ++ + min_key = BTRFS_DIR_START_INDEX; + max_key = 0; + ctx->last_dir_item_offset = inode->last_dir_index_offset; +@@ -4513,7 +4567,7 @@ static noinline int copy_items(struct btrfs_trans_handle *trans, + disk_bytenr += extent_offset; + ret = btrfs_lookup_csums_range(csum_root, disk_bytenr, + disk_bytenr + extent_num_bytes - 1, +- &ordered_sums, 0); ++ &ordered_sums, 0, false); + if (ret) + goto out; + +@@ -4709,7 +4763,7 @@ static int log_extent_csums(struct btrfs_trans_handle *trans, + ret = btrfs_lookup_csums_range(csum_root, + em->block_start + csum_offset, + em->block_start + csum_offset + +- csum_len - 1, &ordered_sums, 0); ++ csum_len - 1, &ordered_sums, 0, false); + if (ret) + return ret; + +@@ -5399,190 +5453,488 @@ static int btrfs_check_ref_name_override(struct extent_buffer *eb, + return ret; + } + +-struct btrfs_ino_list { ++/* ++ * Check if we need to log an inode. This is used in contexts where while ++ * logging an inode we need to log another inode (either that it exists or in ++ * full mode). This is used instead of btrfs_inode_in_log() because the later ++ * requires the inode to be in the log and have the log transaction committed, ++ * while here we do not care if the log transaction was already committed - our ++ * caller will commit the log later - and we want to avoid logging an inode ++ * multiple times when multiple tasks have joined the same log transaction. ++ */ ++static bool need_log_inode(const struct btrfs_trans_handle *trans, ++ const struct btrfs_inode *inode) ++{ ++ /* ++ * If a directory was not modified, no dentries added or removed, we can ++ * and should avoid logging it. ++ */ ++ if (S_ISDIR(inode->vfs_inode.i_mode) && inode->last_trans < trans->transid) ++ return false; ++ ++ /* ++ * If this inode does not have new/updated/deleted xattrs since the last ++ * time it was logged and is flagged as logged in the current transaction, ++ * we can skip logging it. As for new/deleted names, those are updated in ++ * the log by link/unlink/rename operations. ++ * In case the inode was logged and then evicted and reloaded, its ++ * logged_trans will be 0, in which case we have to fully log it since ++ * logged_trans is a transient field, not persisted. ++ */ ++ if (inode->logged_trans == trans->transid && ++ !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) ++ return false; ++ ++ return true; ++} ++ ++struct btrfs_dir_list { + u64 ino; +- u64 parent; + struct list_head list; + }; + +-static int log_conflicting_inodes(struct btrfs_trans_handle *trans, +- struct btrfs_root *root, +- struct btrfs_path *path, +- struct btrfs_log_ctx *ctx, +- u64 ino, u64 parent) ++/* ++ * Log the inodes of the new dentries of a directory. ++ * See process_dir_items_leaf() for details about why it is needed. ++ * This is a recursive operation - if an existing dentry corresponds to a ++ * directory, that directory's new entries are logged too (same behaviour as ++ * ext3/4, xfs, f2fs, reiserfs, nilfs2). Note that when logging the inodes ++ * the dentries point to we do not acquire their VFS lock, otherwise lockdep ++ * complains about the following circular lock dependency / possible deadlock: ++ * ++ * CPU0 CPU1 ++ * ---- ---- ++ * lock(&type->i_mutex_dir_key#3/2); ++ * lock(sb_internal#2); ++ * lock(&type->i_mutex_dir_key#3/2); ++ * lock(&sb->s_type->i_mutex_key#14); ++ * ++ * Where sb_internal is the lock (a counter that works as a lock) acquired by ++ * sb_start_intwrite() in btrfs_start_transaction(). ++ * Not acquiring the VFS lock of the inodes is still safe because: ++ * ++ * 1) For regular files we log with a mode of LOG_INODE_EXISTS. It's possible ++ * that while logging the inode new references (names) are added or removed ++ * from the inode, leaving the logged inode item with a link count that does ++ * not match the number of logged inode reference items. This is fine because ++ * at log replay time we compute the real number of links and correct the ++ * link count in the inode item (see replay_one_buffer() and ++ * link_to_fixup_dir()); ++ * ++ * 2) For directories we log with a mode of LOG_INODE_ALL. It's possible that ++ * while logging the inode's items new index items (key type ++ * BTRFS_DIR_INDEX_KEY) are added to fs/subvol tree and the logged inode item ++ * has a size that doesn't match the sum of the lengths of all the logged ++ * names - this is ok, not a problem, because at log replay time we set the ++ * directory's i_size to the correct value (see replay_one_name() and ++ * do_overwrite_item()). ++ */ ++static int log_new_dir_dentries(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *start_inode, ++ struct btrfs_log_ctx *ctx) + { +- struct btrfs_ino_list *ino_elem; +- LIST_HEAD(inode_list); ++ struct btrfs_root *root = start_inode->root; ++ struct btrfs_fs_info *fs_info = root->fs_info; ++ struct btrfs_path *path; ++ LIST_HEAD(dir_list); ++ struct btrfs_dir_list *dir_elem; ++ u64 ino = btrfs_ino(start_inode); + int ret = 0; + +- ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); +- if (!ino_elem) +- return -ENOMEM; +- ino_elem->ino = ino; +- ino_elem->parent = parent; +- list_add_tail(&ino_elem->list, &inode_list); ++ /* ++ * If we are logging a new name, as part of a link or rename operation, ++ * don't bother logging new dentries, as we just want to log the names ++ * of an inode and that any new parents exist. ++ */ ++ if (ctx->logging_new_name) ++ return 0; + +- while (!list_empty(&inode_list)) { +- struct btrfs_fs_info *fs_info = root->fs_info; +- struct btrfs_key key; +- struct inode *inode; ++ path = btrfs_alloc_path(); ++ if (!path) ++ return -ENOMEM; + +- ino_elem = list_first_entry(&inode_list, struct btrfs_ino_list, +- list); +- ino = ino_elem->ino; +- parent = ino_elem->parent; +- list_del(&ino_elem->list); +- kfree(ino_elem); +- if (ret) +- continue; ++ while (true) { ++ struct extent_buffer *leaf; ++ struct btrfs_key min_key; ++ bool continue_curr_inode = true; ++ int nritems; ++ int i; + ++ min_key.objectid = ino; ++ min_key.type = BTRFS_DIR_INDEX_KEY; ++ min_key.offset = 0; ++again: + btrfs_release_path(path); +- +- inode = btrfs_iget(fs_info->sb, ino, root); +- /* +- * If the other inode that had a conflicting dir entry was +- * deleted in the current transaction, we need to log its parent +- * directory. +- */ +- if (IS_ERR(inode)) { +- ret = PTR_ERR(inode); +- if (ret == -ENOENT) { +- inode = btrfs_iget(fs_info->sb, parent, root); +- if (IS_ERR(inode)) { +- ret = PTR_ERR(inode); +- } else { +- ret = btrfs_log_inode(trans, +- BTRFS_I(inode), +- LOG_OTHER_INODE_ALL, +- ctx); +- btrfs_add_delayed_iput(inode); +- } +- } +- continue; +- } +- /* +- * If the inode was already logged skip it - otherwise we can +- * hit an infinite loop. Example: +- * +- * From the commit root (previous transaction) we have the +- * following inodes: +- * +- * inode 257 a directory +- * inode 258 with references "zz" and "zz_link" on inode 257 +- * inode 259 with reference "a" on inode 257 +- * +- * And in the current (uncommitted) transaction we have: +- * +- * inode 257 a directory, unchanged +- * inode 258 with references "a" and "a2" on inode 257 +- * inode 259 with reference "zz_link" on inode 257 +- * inode 261 with reference "zz" on inode 257 +- * +- * When logging inode 261 the following infinite loop could +- * happen if we don't skip already logged inodes: +- * +- * - we detect inode 258 as a conflicting inode, with inode 261 +- * on reference "zz", and log it; +- * +- * - we detect inode 259 as a conflicting inode, with inode 258 +- * on reference "a", and log it; +- * +- * - we detect inode 258 as a conflicting inode, with inode 259 +- * on reference "zz_link", and log it - again! After this we +- * repeat the above steps forever. +- */ +- spin_lock(&BTRFS_I(inode)->lock); +- /* +- * Check the inode's logged_trans only instead of +- * btrfs_inode_in_log(). This is because the last_log_commit of +- * the inode is not updated when we only log that it exists (see +- * btrfs_log_inode()). +- */ +- if (BTRFS_I(inode)->logged_trans == trans->transid) { +- spin_unlock(&BTRFS_I(inode)->lock); +- btrfs_add_delayed_iput(inode); +- continue; +- } +- spin_unlock(&BTRFS_I(inode)->lock); +- /* +- * We are safe logging the other inode without acquiring its +- * lock as long as we log with the LOG_INODE_EXISTS mode. We +- * are safe against concurrent renames of the other inode as +- * well because during a rename we pin the log and update the +- * log with the new name before we unpin it. +- */ +- ret = btrfs_log_inode(trans, BTRFS_I(inode), LOG_OTHER_INODE, ctx); +- if (ret) { +- btrfs_add_delayed_iput(inode); +- continue; +- } +- +- key.objectid = ino; +- key.type = BTRFS_INODE_REF_KEY; +- key.offset = 0; +- ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); ++ ret = btrfs_search_forward(root, &min_key, path, trans->transid); + if (ret < 0) { +- btrfs_add_delayed_iput(inode); +- continue; ++ break; ++ } else if (ret > 0) { ++ ret = 0; ++ goto next; + } + +- while (true) { +- struct extent_buffer *leaf = path->nodes[0]; +- int slot = path->slots[0]; +- u64 other_ino = 0; +- u64 other_parent = 0; ++ leaf = path->nodes[0]; ++ nritems = btrfs_header_nritems(leaf); ++ for (i = path->slots[0]; i < nritems; i++) { ++ struct btrfs_dir_item *di; ++ struct btrfs_key di_key; ++ struct inode *di_inode; ++ int log_mode = LOG_INODE_EXISTS; ++ int type; + +- if (slot >= btrfs_header_nritems(leaf)) { +- ret = btrfs_next_leaf(root, path); +- if (ret < 0) { +- break; +- } else if (ret > 0) { +- ret = 0; +- break; +- } ++ btrfs_item_key_to_cpu(leaf, &min_key, i); ++ if (min_key.objectid != ino || ++ min_key.type != BTRFS_DIR_INDEX_KEY) { ++ continue_curr_inode = false; ++ break; ++ } ++ ++ di = btrfs_item_ptr(leaf, i, struct btrfs_dir_item); ++ type = btrfs_dir_type(leaf, di); ++ if (btrfs_dir_transid(leaf, di) < trans->transid) ++ continue; ++ btrfs_dir_item_key_to_cpu(leaf, di, &di_key); ++ if (di_key.type == BTRFS_ROOT_ITEM_KEY) + continue; ++ ++ btrfs_release_path(path); ++ di_inode = btrfs_iget(fs_info->sb, di_key.objectid, root); ++ if (IS_ERR(di_inode)) { ++ ret = PTR_ERR(di_inode); ++ goto out; + } + +- btrfs_item_key_to_cpu(leaf, &key, slot); +- if (key.objectid != ino || +- (key.type != BTRFS_INODE_REF_KEY && +- key.type != BTRFS_INODE_EXTREF_KEY)) { +- ret = 0; ++ if (!need_log_inode(trans, BTRFS_I(di_inode))) { ++ btrfs_add_delayed_iput(di_inode); + break; + } + +- ret = btrfs_check_ref_name_override(leaf, slot, &key, +- BTRFS_I(inode), &other_ino, +- &other_parent); +- if (ret < 0) +- break; +- if (ret > 0) { +- ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); +- if (!ino_elem) { ++ ctx->log_new_dentries = false; ++ if (type == BTRFS_FT_DIR) ++ log_mode = LOG_INODE_ALL; ++ ret = btrfs_log_inode(trans, BTRFS_I(di_inode), ++ log_mode, ctx); ++ btrfs_add_delayed_iput(di_inode); ++ if (ret) ++ goto out; ++ if (ctx->log_new_dentries) { ++ dir_elem = kmalloc(sizeof(*dir_elem), GFP_NOFS); ++ if (!dir_elem) { + ret = -ENOMEM; +- break; ++ goto out; + } +- ino_elem->ino = other_ino; +- ino_elem->parent = other_parent; +- list_add_tail(&ino_elem->list, &inode_list); +- ret = 0; ++ dir_elem->ino = di_key.objectid; ++ list_add_tail(&dir_elem->list, &dir_list); + } +- path->slots[0]++; ++ break; + } +- btrfs_add_delayed_iput(inode); +- } + +- return ret; +-} ++ if (continue_curr_inode && min_key.offset < (u64)-1) { ++ min_key.offset++; ++ goto again; ++ } + +-static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, +- struct btrfs_inode *inode, +- struct btrfs_key *min_key, +- const struct btrfs_key *max_key, +- struct btrfs_path *path, ++next: ++ if (list_empty(&dir_list)) ++ break; ++ ++ dir_elem = list_first_entry(&dir_list, struct btrfs_dir_list, list); ++ ino = dir_elem->ino; ++ list_del(&dir_elem->list); ++ kfree(dir_elem); ++ } ++out: ++ btrfs_free_path(path); ++ if (ret) { ++ struct btrfs_dir_list *next; ++ ++ list_for_each_entry_safe(dir_elem, next, &dir_list, list) ++ kfree(dir_elem); ++ } ++ ++ return ret; ++} ++ ++struct btrfs_ino_list { ++ u64 ino; ++ u64 parent; ++ struct list_head list; ++}; ++ ++static void free_conflicting_inodes(struct btrfs_log_ctx *ctx) ++{ ++ struct btrfs_ino_list *curr; ++ struct btrfs_ino_list *next; ++ ++ list_for_each_entry_safe(curr, next, &ctx->conflict_inodes, list) { ++ list_del(&curr->list); ++ kfree(curr); ++ } ++} ++ ++static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino, ++ struct btrfs_path *path) ++{ ++ struct btrfs_key key; ++ int ret; ++ ++ key.objectid = ino; ++ key.type = BTRFS_INODE_ITEM_KEY; ++ key.offset = 0; ++ ++ path->search_commit_root = 1; ++ path->skip_locking = 1; ++ ++ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); ++ if (WARN_ON_ONCE(ret > 0)) { ++ /* ++ * We have previously found the inode through the commit root ++ * so this should not happen. If it does, just error out and ++ * fallback to a transaction commit. ++ */ ++ ret = -ENOENT; ++ } else if (ret == 0) { ++ struct btrfs_inode_item *item; ++ ++ item = btrfs_item_ptr(path->nodes[0], path->slots[0], ++ struct btrfs_inode_item); ++ if (S_ISDIR(btrfs_inode_mode(path->nodes[0], item))) ++ ret = 1; ++ } ++ ++ btrfs_release_path(path); ++ path->search_commit_root = 0; ++ path->skip_locking = 0; ++ ++ return ret; ++} ++ ++static int add_conflicting_inode(struct btrfs_trans_handle *trans, ++ struct btrfs_root *root, ++ struct btrfs_path *path, ++ u64 ino, u64 parent, ++ struct btrfs_log_ctx *ctx) ++{ ++ struct btrfs_ino_list *ino_elem; ++ struct inode *inode; ++ ++ /* ++ * It's rare to have a lot of conflicting inodes, in practice it is not ++ * common to have more than 1 or 2. We don't want to collect too many, ++ * as we could end up logging too many inodes (even if only in ++ * LOG_INODE_EXISTS mode) and slow down other fsyncs or transaction ++ * commits. ++ */ ++ if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) ++ return BTRFS_LOG_FORCE_COMMIT; ++ ++ inode = btrfs_iget(root->fs_info->sb, ino, root); ++ /* ++ * If the other inode that had a conflicting dir entry was deleted in ++ * the current transaction then we either: ++ * ++ * 1) Log the parent directory (later after adding it to the list) if ++ * the inode is a directory. This is because it may be a deleted ++ * subvolume/snapshot or it may be a regular directory that had ++ * deleted subvolumes/snapshots (or subdirectories that had them), ++ * and at the moment we can't deal with dropping subvolumes/snapshots ++ * during log replay. So we just log the parent, which will result in ++ * a fallback to a transaction commit if we are dealing with those ++ * cases (last_unlink_trans will match the current transaction); ++ * ++ * 2) Do nothing if it's not a directory. During log replay we simply ++ * unlink the conflicting dentry from the parent directory and then ++ * add the dentry for our inode. Like this we can avoid logging the ++ * parent directory (and maybe fallback to a transaction commit in ++ * case it has a last_unlink_trans == trans->transid, due to moving ++ * some inode from it to some other directory). ++ */ ++ if (IS_ERR(inode)) { ++ int ret = PTR_ERR(inode); ++ ++ if (ret != -ENOENT) ++ return ret; ++ ++ ret = conflicting_inode_is_dir(root, ino, path); ++ /* Not a directory or we got an error. */ ++ if (ret <= 0) ++ return ret; ++ ++ /* Conflicting inode is a directory, so we'll log its parent. */ ++ ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); ++ if (!ino_elem) ++ return -ENOMEM; ++ ino_elem->ino = ino; ++ ino_elem->parent = parent; ++ list_add_tail(&ino_elem->list, &ctx->conflict_inodes); ++ ctx->num_conflict_inodes++; ++ ++ return 0; ++ } ++ ++ /* ++ * If the inode was already logged skip it - otherwise we can hit an ++ * infinite loop. Example: ++ * ++ * From the commit root (previous transaction) we have the following ++ * inodes: ++ * ++ * inode 257 a directory ++ * inode 258 with references "zz" and "zz_link" on inode 257 ++ * inode 259 with reference "a" on inode 257 ++ * ++ * And in the current (uncommitted) transaction we have: ++ * ++ * inode 257 a directory, unchanged ++ * inode 258 with references "a" and "a2" on inode 257 ++ * inode 259 with reference "zz_link" on inode 257 ++ * inode 261 with reference "zz" on inode 257 ++ * ++ * When logging inode 261 the following infinite loop could ++ * happen if we don't skip already logged inodes: ++ * ++ * - we detect inode 258 as a conflicting inode, with inode 261 ++ * on reference "zz", and log it; ++ * ++ * - we detect inode 259 as a conflicting inode, with inode 258 ++ * on reference "a", and log it; ++ * ++ * - we detect inode 258 as a conflicting inode, with inode 259 ++ * on reference "zz_link", and log it - again! After this we ++ * repeat the above steps forever. ++ * ++ * Here we can use need_log_inode() because we only need to log the ++ * inode in LOG_INODE_EXISTS mode and rename operations update the log, ++ * so that the log ends up with the new name and without the old name. ++ */ ++ if (!need_log_inode(trans, BTRFS_I(inode))) { ++ btrfs_add_delayed_iput(inode); ++ return 0; ++ } ++ ++ btrfs_add_delayed_iput(inode); ++ ++ ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); ++ if (!ino_elem) ++ return -ENOMEM; ++ ino_elem->ino = ino; ++ ino_elem->parent = parent; ++ list_add_tail(&ino_elem->list, &ctx->conflict_inodes); ++ ctx->num_conflict_inodes++; ++ ++ return 0; ++} ++ ++static int log_conflicting_inodes(struct btrfs_trans_handle *trans, ++ struct btrfs_root *root, ++ struct btrfs_log_ctx *ctx) ++{ ++ struct btrfs_fs_info *fs_info = root->fs_info; ++ int ret = 0; ++ ++ /* ++ * Conflicting inodes are logged by the first call to btrfs_log_inode(), ++ * otherwise we could have unbounded recursion of btrfs_log_inode() ++ * calls. This check guarantees we can have only 1 level of recursion. ++ */ ++ if (ctx->logging_conflict_inodes) ++ return 0; ++ ++ ctx->logging_conflict_inodes = true; ++ ++ /* ++ * New conflicting inodes may be found and added to the list while we ++ * are logging a conflicting inode, so keep iterating while the list is ++ * not empty. ++ */ ++ while (!list_empty(&ctx->conflict_inodes)) { ++ struct btrfs_ino_list *curr; ++ struct inode *inode; ++ u64 ino; ++ u64 parent; ++ ++ curr = list_first_entry(&ctx->conflict_inodes, ++ struct btrfs_ino_list, list); ++ ino = curr->ino; ++ parent = curr->parent; ++ list_del(&curr->list); ++ kfree(curr); ++ ++ inode = btrfs_iget(fs_info->sb, ino, root); ++ /* ++ * If the other inode that had a conflicting dir entry was ++ * deleted in the current transaction, we need to log its parent ++ * directory. See the comment at add_conflicting_inode(). ++ */ ++ if (IS_ERR(inode)) { ++ ret = PTR_ERR(inode); ++ if (ret != -ENOENT) ++ break; ++ ++ inode = btrfs_iget(fs_info->sb, parent, root); ++ if (IS_ERR(inode)) { ++ ret = PTR_ERR(inode); ++ break; ++ } ++ ++ /* ++ * Always log the directory, we cannot make this ++ * conditional on need_log_inode() because the directory ++ * might have been logged in LOG_INODE_EXISTS mode or ++ * the dir index of the conflicting inode is not in a ++ * dir index key range logged for the directory. So we ++ * must make sure the deletion is recorded. ++ */ ++ ret = btrfs_log_inode(trans, BTRFS_I(inode), ++ LOG_INODE_ALL, ctx); ++ btrfs_add_delayed_iput(inode); ++ if (ret) ++ break; ++ continue; ++ } ++ ++ /* ++ * Here we can use need_log_inode() because we only need to log ++ * the inode in LOG_INODE_EXISTS mode and rename operations ++ * update the log, so that the log ends up with the new name and ++ * without the old name. ++ * ++ * We did this check at add_conflicting_inode(), but here we do ++ * it again because if some other task logged the inode after ++ * that, we can avoid doing it again. ++ */ ++ if (!need_log_inode(trans, BTRFS_I(inode))) { ++ btrfs_add_delayed_iput(inode); ++ continue; ++ } ++ ++ /* ++ * We are safe logging the other inode without acquiring its ++ * lock as long as we log with the LOG_INODE_EXISTS mode. We ++ * are safe against concurrent renames of the other inode as ++ * well because during a rename we pin the log and update the ++ * log with the new name before we unpin it. ++ */ ++ ret = btrfs_log_inode(trans, BTRFS_I(inode), LOG_INODE_EXISTS, ctx); ++ btrfs_add_delayed_iput(inode); ++ if (ret) ++ break; ++ } ++ ++ ctx->logging_conflict_inodes = false; ++ if (ret) ++ free_conflicting_inodes(ctx); ++ ++ return ret; ++} ++ ++static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ struct btrfs_key *min_key, ++ const struct btrfs_key *max_key, ++ struct btrfs_path *path, + struct btrfs_path *dst_path, + const u64 logged_isize, +- const bool recursive_logging, + const int inode_only, + struct btrfs_log_ctx *ctx, + bool *need_log_inode_item) +@@ -5621,8 +5973,8 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, + break; + } else if ((min_key->type == BTRFS_INODE_REF_KEY || + min_key->type == BTRFS_INODE_EXTREF_KEY) && +- inode->generation == trans->transid && +- !recursive_logging) { ++ (inode->generation == trans->transid || ++ ctx->logging_conflict_inodes)) { + u64 other_ino = 0; + u64 other_parent = 0; + +@@ -5646,11 +5998,12 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, + return ret; + ins_nr = 0; + +- ret = log_conflicting_inodes(trans, root, path, +- ctx, other_ino, other_parent); ++ btrfs_release_path(path); ++ ret = add_conflicting_inode(trans, root, path, ++ other_ino, ++ other_parent, ctx); + if (ret) + return ret; +- btrfs_release_path(path); + goto next_key; + } + } else if (min_key->type == BTRFS_XATTR_ITEM_KEY) { +@@ -5708,28 +6061,389 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, + } + + /* +- * We may process many leaves full of items for our inode, so +- * avoid monopolizing a cpu for too long by rescheduling while +- * not holding locks on any tree. ++ * We may process many leaves full of items for our inode, so ++ * avoid monopolizing a cpu for too long by rescheduling while ++ * not holding locks on any tree. ++ */ ++ cond_resched(); ++ } ++ if (ins_nr) { ++ ret = copy_items(trans, inode, dst_path, path, ins_start_slot, ++ ins_nr, inode_only, logged_isize); ++ if (ret) ++ return ret; ++ } ++ ++ if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { ++ /* ++ * Release the path because otherwise we might attempt to double ++ * lock the same leaf with btrfs_log_prealloc_extents() below. ++ */ ++ btrfs_release_path(path); ++ ret = btrfs_log_prealloc_extents(trans, inode, dst_path); ++ } ++ ++ return ret; ++} ++ ++static int insert_delayed_items_batch(struct btrfs_trans_handle *trans, ++ struct btrfs_root *log, ++ struct btrfs_path *path, ++ const struct btrfs_item_batch *batch, ++ const struct btrfs_delayed_item *first_item) ++{ ++ const struct btrfs_delayed_item *curr = first_item; ++ int ret; ++ ++ ret = btrfs_insert_empty_items(trans, log, path, batch); ++ if (ret) ++ return ret; ++ ++ for (int i = 0; i < batch->nr; i++) { ++ char *data_ptr; ++ ++ data_ptr = btrfs_item_ptr(path->nodes[0], path->slots[0], char); ++ write_extent_buffer(path->nodes[0], &curr->data, ++ (unsigned long)data_ptr, curr->data_len); ++ curr = list_next_entry(curr, log_list); ++ path->slots[0]++; ++ } ++ ++ btrfs_release_path(path); ++ ++ return 0; ++} ++ ++static int log_delayed_insertion_items(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ const struct list_head *delayed_ins_list, ++ struct btrfs_log_ctx *ctx) ++{ ++ /* 195 (4095 bytes of keys and sizes) fits in a single 4K page. */ ++ const int max_batch_size = 195; ++ const int leaf_data_size = BTRFS_LEAF_DATA_SIZE(trans->fs_info); ++ const u64 ino = btrfs_ino(inode); ++ struct btrfs_root *log = inode->root->log_root; ++ struct btrfs_item_batch batch = { ++ .nr = 0, ++ .total_data_size = 0, ++ }; ++ const struct btrfs_delayed_item *first = NULL; ++ const struct btrfs_delayed_item *curr; ++ char *ins_data; ++ struct btrfs_key *ins_keys; ++ u32 *ins_sizes; ++ u64 curr_batch_size = 0; ++ int batch_idx = 0; ++ int ret; ++ ++ /* We are adding dir index items to the log tree. */ ++ lockdep_assert_held(&inode->log_mutex); ++ ++ /* ++ * We collect delayed items before copying index keys from the subvolume ++ * to the log tree. However just after we collected them, they may have ++ * been flushed (all of them or just some of them), and therefore we ++ * could have copied them from the subvolume tree to the log tree. ++ * So find the first delayed item that was not yet logged (they are ++ * sorted by index number). ++ */ ++ list_for_each_entry(curr, delayed_ins_list, log_list) { ++ if (curr->index > inode->last_dir_index_offset) { ++ first = curr; ++ break; ++ } ++ } ++ ++ /* Empty list or all delayed items were already logged. */ ++ if (!first) ++ return 0; ++ ++ ins_data = kmalloc(max_batch_size * sizeof(u32) + ++ max_batch_size * sizeof(struct btrfs_key), GFP_NOFS); ++ if (!ins_data) ++ return -ENOMEM; ++ ins_sizes = (u32 *)ins_data; ++ batch.data_sizes = ins_sizes; ++ ins_keys = (struct btrfs_key *)(ins_data + max_batch_size * sizeof(u32)); ++ batch.keys = ins_keys; ++ ++ curr = first; ++ while (!list_entry_is_head(curr, delayed_ins_list, log_list)) { ++ const u32 curr_size = curr->data_len + sizeof(struct btrfs_item); ++ ++ if (curr_batch_size + curr_size > leaf_data_size || ++ batch.nr == max_batch_size) { ++ ret = insert_delayed_items_batch(trans, log, path, ++ &batch, first); ++ if (ret) ++ goto out; ++ batch_idx = 0; ++ batch.nr = 0; ++ batch.total_data_size = 0; ++ curr_batch_size = 0; ++ first = curr; ++ } ++ ++ ins_sizes[batch_idx] = curr->data_len; ++ ins_keys[batch_idx].objectid = ino; ++ ins_keys[batch_idx].type = BTRFS_DIR_INDEX_KEY; ++ ins_keys[batch_idx].offset = curr->index; ++ curr_batch_size += curr_size; ++ batch.total_data_size += curr->data_len; ++ batch.nr++; ++ batch_idx++; ++ curr = list_next_entry(curr, log_list); ++ } ++ ++ ASSERT(batch.nr >= 1); ++ ret = insert_delayed_items_batch(trans, log, path, &batch, first); ++ ++ curr = list_last_entry(delayed_ins_list, struct btrfs_delayed_item, ++ log_list); ++ inode->last_dir_index_offset = curr->index; ++out: ++ kfree(ins_data); ++ ++ return ret; ++} ++ ++static int log_delayed_deletions_full(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ const struct list_head *delayed_del_list, ++ struct btrfs_log_ctx *ctx) ++{ ++ const u64 ino = btrfs_ino(inode); ++ const struct btrfs_delayed_item *curr; ++ ++ curr = list_first_entry(delayed_del_list, struct btrfs_delayed_item, ++ log_list); ++ ++ while (!list_entry_is_head(curr, delayed_del_list, log_list)) { ++ u64 first_dir_index = curr->index; ++ u64 last_dir_index; ++ const struct btrfs_delayed_item *next; ++ int ret; ++ ++ /* ++ * Find a range of consecutive dir index items to delete. Like ++ * this we log a single dir range item spanning several contiguous ++ * dir items instead of logging one range item per dir index item. ++ */ ++ next = list_next_entry(curr, log_list); ++ while (!list_entry_is_head(next, delayed_del_list, log_list)) { ++ if (next->index != curr->index + 1) ++ break; ++ curr = next; ++ next = list_next_entry(next, log_list); ++ } ++ ++ last_dir_index = curr->index; ++ ASSERT(last_dir_index >= first_dir_index); ++ ++ ret = insert_dir_log_key(trans, inode->root->log_root, path, ++ ino, first_dir_index, last_dir_index); ++ if (ret) ++ return ret; ++ curr = list_next_entry(curr, log_list); ++ } ++ ++ return 0; ++} ++ ++static int batch_delete_dir_index_items(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ struct btrfs_log_ctx *ctx, ++ const struct list_head *delayed_del_list, ++ const struct btrfs_delayed_item *first, ++ const struct btrfs_delayed_item **last_ret) ++{ ++ const struct btrfs_delayed_item *next; ++ struct extent_buffer *leaf = path->nodes[0]; ++ const int last_slot = btrfs_header_nritems(leaf) - 1; ++ int slot = path->slots[0] + 1; ++ const u64 ino = btrfs_ino(inode); ++ ++ next = list_next_entry(first, log_list); ++ ++ while (slot < last_slot && ++ !list_entry_is_head(next, delayed_del_list, log_list)) { ++ struct btrfs_key key; ++ ++ btrfs_item_key_to_cpu(leaf, &key, slot); ++ if (key.objectid != ino || ++ key.type != BTRFS_DIR_INDEX_KEY || ++ key.offset != next->index) ++ break; ++ ++ slot++; ++ *last_ret = next; ++ next = list_next_entry(next, log_list); ++ } ++ ++ return btrfs_del_items(trans, inode->root->log_root, path, ++ path->slots[0], slot - path->slots[0]); ++} ++ ++static int log_delayed_deletions_incremental(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ const struct list_head *delayed_del_list, ++ struct btrfs_log_ctx *ctx) ++{ ++ struct btrfs_root *log = inode->root->log_root; ++ const struct btrfs_delayed_item *curr; ++ u64 last_range_start; ++ u64 last_range_end = 0; ++ struct btrfs_key key; ++ ++ key.objectid = btrfs_ino(inode); ++ key.type = BTRFS_DIR_INDEX_KEY; ++ curr = list_first_entry(delayed_del_list, struct btrfs_delayed_item, ++ log_list); ++ ++ while (!list_entry_is_head(curr, delayed_del_list, log_list)) { ++ const struct btrfs_delayed_item *last = curr; ++ u64 first_dir_index = curr->index; ++ u64 last_dir_index; ++ bool deleted_items = false; ++ int ret; ++ ++ key.offset = curr->index; ++ ret = btrfs_search_slot(trans, log, &key, path, -1, 1); ++ if (ret < 0) { ++ return ret; ++ } else if (ret == 0) { ++ ret = batch_delete_dir_index_items(trans, inode, path, ctx, ++ delayed_del_list, curr, ++ &last); ++ if (ret) ++ return ret; ++ deleted_items = true; ++ } ++ ++ btrfs_release_path(path); ++ ++ /* ++ * If we deleted items from the leaf, it means we have a range ++ * item logging their range, so no need to add one or update an ++ * existing one. Otherwise we have to log a dir range item. + */ +- cond_resched(); +- } +- if (ins_nr) { +- ret = copy_items(trans, inode, dst_path, path, ins_start_slot, +- ins_nr, inode_only, logged_isize); ++ if (deleted_items) ++ goto next_batch; ++ ++ last_dir_index = last->index; ++ ASSERT(last_dir_index >= first_dir_index); ++ /* ++ * If this range starts right after where the previous one ends, ++ * then we want to reuse the previous range item and change its ++ * end offset to the end of this range. This is just to minimize ++ * leaf space usage, by avoiding adding a new range item. ++ */ ++ if (last_range_end != 0 && first_dir_index == last_range_end + 1) ++ first_dir_index = last_range_start; ++ ++ ret = insert_dir_log_key(trans, log, path, key.objectid, ++ first_dir_index, last_dir_index); + if (ret) + return ret; ++ ++ last_range_start = first_dir_index; ++ last_range_end = last_dir_index; ++next_batch: ++ curr = list_next_entry(last, log_list); + } + +- if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { +- /* +- * Release the path because otherwise we might attempt to double +- * lock the same leaf with btrfs_log_prealloc_extents() below. +- */ +- btrfs_release_path(path); +- ret = btrfs_log_prealloc_extents(trans, inode, dst_path); ++ return 0; ++} ++ ++static int log_delayed_deletion_items(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ struct btrfs_path *path, ++ const struct list_head *delayed_del_list, ++ struct btrfs_log_ctx *ctx) ++{ ++ /* ++ * We are deleting dir index items from the log tree or adding range ++ * items to it. ++ */ ++ lockdep_assert_held(&inode->log_mutex); ++ ++ if (list_empty(delayed_del_list)) ++ return 0; ++ ++ if (ctx->logged_before) ++ return log_delayed_deletions_incremental(trans, inode, path, ++ delayed_del_list, ctx); ++ ++ return log_delayed_deletions_full(trans, inode, path, delayed_del_list, ++ ctx); ++} ++ ++/* ++ * Similar logic as for log_new_dir_dentries(), but it iterates over the delayed ++ * items instead of the subvolume tree. ++ */ ++static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, ++ struct btrfs_inode *inode, ++ const struct list_head *delayed_ins_list, ++ struct btrfs_log_ctx *ctx) ++{ ++ const bool orig_log_new_dentries = ctx->log_new_dentries; ++ struct btrfs_fs_info *fs_info = trans->fs_info; ++ struct btrfs_delayed_item *item; ++ int ret = 0; ++ ++ /* ++ * No need for the log mutex, plus to avoid potential deadlocks or ++ * lockdep annotations due to nesting of delayed inode mutexes and log ++ * mutexes. ++ */ ++ lockdep_assert_not_held(&inode->log_mutex); ++ ++ list_for_each_entry(item, delayed_ins_list, log_list) { ++ struct btrfs_dir_item *dir_item; ++ struct inode *di_inode; ++ struct btrfs_key key; ++ int log_mode = LOG_INODE_EXISTS; ++ ++ dir_item = (struct btrfs_dir_item *)item->data; ++ btrfs_disk_key_to_cpu(&key, &dir_item->location); ++ ++ if (key.type == BTRFS_ROOT_ITEM_KEY) ++ continue; ++ ++ di_inode = btrfs_iget(fs_info->sb, key.objectid, inode->root); ++ if (IS_ERR(di_inode)) { ++ ret = PTR_ERR(di_inode); ++ break; ++ } ++ ++ if (!need_log_inode(trans, BTRFS_I(di_inode))) { ++ btrfs_add_delayed_iput(di_inode); ++ continue; ++ } ++ ++ if (btrfs_stack_dir_type(dir_item) == BTRFS_FT_DIR) ++ log_mode = LOG_INODE_ALL; ++ ++ ctx->log_new_dentries = false; ++ ret = btrfs_log_inode(trans, BTRFS_I(di_inode), log_mode, ctx); ++ ++ if (!ret && ctx->log_new_dentries) ++ ret = log_new_dir_dentries(trans, BTRFS_I(di_inode), ctx); ++ ++ btrfs_add_delayed_iput(di_inode); ++ ++ if (ret) ++ break; + } + ++ ctx->log_new_dentries = orig_log_new_dentries; ++ + return ret; + } + +@@ -5764,9 +6478,9 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, + u64 logged_isize = 0; + bool need_log_inode_item = true; + bool xattrs_logged = false; +- bool recursive_logging = false; + bool inode_item_dropped = true; +- const bool orig_logged_before = ctx->logged_before; ++ LIST_HEAD(delayed_ins_list); ++ LIST_HEAD(delayed_del_list); + + path = btrfs_alloc_path(); + if (!path) +@@ -5794,27 +6508,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, + max_key.type = (u8)-1; + max_key.offset = (u64)-1; + +- /* +- * Only run delayed items if we are a directory. We want to make sure +- * all directory indexes hit the fs/subvolume tree so we can find them +- * and figure out which index ranges have to be logged. +- */ +- if (S_ISDIR(inode->vfs_inode.i_mode)) { +- ret = btrfs_commit_inode_delayed_items(trans, inode); +- if (ret) +- goto out; +- } +- +- if (inode_only == LOG_OTHER_INODE || inode_only == LOG_OTHER_INODE_ALL) { +- recursive_logging = true; +- if (inode_only == LOG_OTHER_INODE) +- inode_only = LOG_INODE_EXISTS; +- else +- inode_only = LOG_INODE_ALL; +- mutex_lock_nested(&inode->log_mutex, SINGLE_DEPTH_NESTING); +- } else { +- mutex_lock(&inode->log_mutex); +- } ++ mutex_lock(&inode->log_mutex); + + /* + * For symlinks, we must always log their content, which is stored in an +@@ -5859,14 +6553,10 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, + * copies of everything. + */ + if (S_ISDIR(inode->vfs_inode.i_mode)) { +- int max_key_type = BTRFS_DIR_LOG_INDEX_KEY; +- + clear_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); +- if (inode_only == LOG_INODE_EXISTS) +- max_key_type = BTRFS_XATTR_ITEM_KEY; + if (ctx->logged_before) + ret = drop_inode_items(trans, log, path, inode, +- max_key_type); ++ BTRFS_XATTR_ITEM_KEY); + } else { + if (inode_only == LOG_INODE_EXISTS && ctx->logged_before) { + /* +@@ -5922,9 +6612,19 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, + if (ret) + goto out_unlock; + ++ /* ++ * If we are logging a directory in full mode, collect the delayed items ++ * before iterating the subvolume tree, so that we don't miss any new ++ * dir index items in case they get flushed while or right after we are ++ * iterating the subvolume tree. ++ */ ++ if (S_ISDIR(inode->vfs_inode.i_mode) && inode_only == LOG_INODE_ALL) ++ btrfs_log_get_delayed_items(inode, &delayed_ins_list, ++ &delayed_del_list); ++ + ret = copy_inode_items_to_log(trans, inode, &min_key, &max_key, + path, dst_path, logged_isize, +- recursive_logging, inode_only, ctx, ++ inode_only, ctx, + &need_log_inode_item); + if (ret) + goto out_unlock; +@@ -5981,6 +6681,14 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, + ret = log_directory_changes(trans, inode, path, dst_path, ctx); + if (ret) + goto out_unlock; ++ ret = log_delayed_insertion_items(trans, inode, path, ++ &delayed_ins_list, ctx); ++ if (ret) ++ goto out_unlock; ++ ret = log_delayed_deletion_items(trans, inode, path, ++ &delayed_del_list, ctx); ++ if (ret) ++ goto out_unlock; + } + + spin_lock(&inode->lock); +@@ -6029,212 +6737,24 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, + + out_unlock: + mutex_unlock(&inode->log_mutex); +-out: ++ + btrfs_free_path(path); + btrfs_free_path(dst_path); + +- if (recursive_logging) +- ctx->logged_before = orig_logged_before; +- +- return ret; +-} +- +-/* +- * Check if we need to log an inode. This is used in contexts where while +- * logging an inode we need to log another inode (either that it exists or in +- * full mode). This is used instead of btrfs_inode_in_log() because the later +- * requires the inode to be in the log and have the log transaction committed, +- * while here we do not care if the log transaction was already committed - our +- * caller will commit the log later - and we want to avoid logging an inode +- * multiple times when multiple tasks have joined the same log transaction. +- */ +-static bool need_log_inode(struct btrfs_trans_handle *trans, +- struct btrfs_inode *inode) +-{ +- /* +- * If a directory was not modified, no dentries added or removed, we can +- * and should avoid logging it. +- */ +- if (S_ISDIR(inode->vfs_inode.i_mode) && inode->last_trans < trans->transid) +- return false; +- +- /* +- * If this inode does not have new/updated/deleted xattrs since the last +- * time it was logged and is flagged as logged in the current transaction, +- * we can skip logging it. As for new/deleted names, those are updated in +- * the log by link/unlink/rename operations. +- * In case the inode was logged and then evicted and reloaded, its +- * logged_trans will be 0, in which case we have to fully log it since +- * logged_trans is a transient field, not persisted. +- */ +- if (inode->logged_trans == trans->transid && +- !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) +- return false; +- +- return true; +-} +- +-struct btrfs_dir_list { +- u64 ino; +- struct list_head list; +-}; +- +-/* +- * Log the inodes of the new dentries of a directory. See log_dir_items() for +- * details about the why it is needed. +- * This is a recursive operation - if an existing dentry corresponds to a +- * directory, that directory's new entries are logged too (same behaviour as +- * ext3/4, xfs, f2fs, reiserfs, nilfs2). Note that when logging the inodes +- * the dentries point to we do not lock their i_mutex, otherwise lockdep +- * complains about the following circular lock dependency / possible deadlock: +- * +- * CPU0 CPU1 +- * ---- ---- +- * lock(&type->i_mutex_dir_key#3/2); +- * lock(sb_internal#2); +- * lock(&type->i_mutex_dir_key#3/2); +- * lock(&sb->s_type->i_mutex_key#14); +- * +- * Where sb_internal is the lock (a counter that works as a lock) acquired by +- * sb_start_intwrite() in btrfs_start_transaction(). +- * Not locking i_mutex of the inodes is still safe because: +- * +- * 1) For regular files we log with a mode of LOG_INODE_EXISTS. It's possible +- * that while logging the inode new references (names) are added or removed +- * from the inode, leaving the logged inode item with a link count that does +- * not match the number of logged inode reference items. This is fine because +- * at log replay time we compute the real number of links and correct the +- * link count in the inode item (see replay_one_buffer() and +- * link_to_fixup_dir()); +- * +- * 2) For directories we log with a mode of LOG_INODE_ALL. It's possible that +- * while logging the inode's items new index items (key type +- * BTRFS_DIR_INDEX_KEY) are added to fs/subvol tree and the logged inode item +- * has a size that doesn't match the sum of the lengths of all the logged +- * names - this is ok, not a problem, because at log replay time we set the +- * directory's i_size to the correct value (see replay_one_name() and +- * do_overwrite_item()). +- */ +-static int log_new_dir_dentries(struct btrfs_trans_handle *trans, +- struct btrfs_root *root, +- struct btrfs_inode *start_inode, +- struct btrfs_log_ctx *ctx) +-{ +- struct btrfs_fs_info *fs_info = root->fs_info; +- struct btrfs_path *path; +- LIST_HEAD(dir_list); +- struct btrfs_dir_list *dir_elem; +- int ret = 0; +- +- /* +- * If we are logging a new name, as part of a link or rename operation, +- * don't bother logging new dentries, as we just want to log the names +- * of an inode and that any new parents exist. +- */ +- if (ctx->logging_new_name) +- return 0; +- +- path = btrfs_alloc_path(); +- if (!path) +- return -ENOMEM; +- +- dir_elem = kmalloc(sizeof(*dir_elem), GFP_NOFS); +- if (!dir_elem) { +- btrfs_free_path(path); +- return -ENOMEM; +- } +- dir_elem->ino = btrfs_ino(start_inode); +- list_add_tail(&dir_elem->list, &dir_list); +- +- while (!list_empty(&dir_list)) { +- struct extent_buffer *leaf; +- struct btrfs_key min_key; +- int nritems; +- int i; +- +- dir_elem = list_first_entry(&dir_list, struct btrfs_dir_list, +- list); +- if (ret) +- goto next_dir_inode; +- +- min_key.objectid = dir_elem->ino; +- min_key.type = BTRFS_DIR_INDEX_KEY; +- min_key.offset = 0; +-again: +- btrfs_release_path(path); +- ret = btrfs_search_forward(root, &min_key, path, trans->transid); +- if (ret < 0) { +- goto next_dir_inode; +- } else if (ret > 0) { +- ret = 0; +- goto next_dir_inode; +- } +- +- leaf = path->nodes[0]; +- nritems = btrfs_header_nritems(leaf); +- for (i = path->slots[0]; i < nritems; i++) { +- struct btrfs_dir_item *di; +- struct btrfs_key di_key; +- struct inode *di_inode; +- struct btrfs_dir_list *new_dir_elem; +- int log_mode = LOG_INODE_EXISTS; +- int type; +- +- btrfs_item_key_to_cpu(leaf, &min_key, i); +- if (min_key.objectid != dir_elem->ino || +- min_key.type != BTRFS_DIR_INDEX_KEY) +- goto next_dir_inode; +- +- di = btrfs_item_ptr(leaf, i, struct btrfs_dir_item); +- type = btrfs_dir_type(leaf, di); +- if (btrfs_dir_transid(leaf, di) < trans->transid) +- continue; +- btrfs_dir_item_key_to_cpu(leaf, di, &di_key); +- if (di_key.type == BTRFS_ROOT_ITEM_KEY) +- continue; +- +- btrfs_release_path(path); +- di_inode = btrfs_iget(fs_info->sb, di_key.objectid, root); +- if (IS_ERR(di_inode)) { +- ret = PTR_ERR(di_inode); +- goto next_dir_inode; +- } ++ if (ret) ++ free_conflicting_inodes(ctx); ++ else ++ ret = log_conflicting_inodes(trans, inode->root, ctx); + +- if (!need_log_inode(trans, BTRFS_I(di_inode))) { +- btrfs_add_delayed_iput(di_inode); +- break; +- } ++ if (S_ISDIR(inode->vfs_inode.i_mode) && inode_only == LOG_INODE_ALL) { ++ if (!ret) ++ ret = log_new_delayed_dentries(trans, inode, ++ &delayed_ins_list, ctx); + +- ctx->log_new_dentries = false; +- if (type == BTRFS_FT_DIR) +- log_mode = LOG_INODE_ALL; +- ret = btrfs_log_inode(trans, BTRFS_I(di_inode), +- log_mode, ctx); +- btrfs_add_delayed_iput(di_inode); +- if (ret) +- goto next_dir_inode; +- if (ctx->log_new_dentries) { +- new_dir_elem = kmalloc(sizeof(*new_dir_elem), +- GFP_NOFS); +- if (!new_dir_elem) { +- ret = -ENOMEM; +- goto next_dir_inode; +- } +- new_dir_elem->ino = di_key.objectid; +- list_add_tail(&new_dir_elem->list, &dir_list); +- } +- break; +- } +- if (min_key.offset < (u64)-1) { +- min_key.offset++; +- goto again; +- } +-next_dir_inode: +- list_del(&dir_elem->list); +- kfree(dir_elem); ++ btrfs_log_put_delayed_items(inode, &delayed_ins_list, ++ &delayed_del_list); + } + +- btrfs_free_path(path); + return ret; + } + +@@ -6346,7 +6866,7 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, + ret = btrfs_log_inode(trans, BTRFS_I(dir_inode), + LOG_INODE_ALL, ctx); + if (!ret && ctx->log_new_dentries) +- ret = log_new_dir_dentries(trans, root, ++ ret = log_new_dir_dentries(trans, + BTRFS_I(dir_inode), ctx); + btrfs_add_delayed_iput(dir_inode); + if (ret) +@@ -6661,7 +7181,7 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, + goto end_trans; + + if (log_dentries) +- ret = log_new_dir_dentries(trans, root, inode, ctx); ++ ret = log_new_dir_dentries(trans, inode, ctx); + else + ret = 0; + end_trans: +@@ -7088,6 +7608,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, + * inconsistent state after a rename operation. + */ + btrfs_log_inode_parent(trans, inode, parent, LOG_INODE_EXISTS, &ctx); ++ ASSERT(list_empty(&ctx.conflict_inodes)); + out: + /* + * If an error happened mark the log for a full commit because it's not +diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h +index 57ab5f3b8dc7..4e34fe4b7762 100644 +--- a/fs/btrfs/tree-log.h ++++ b/fs/btrfs/tree-log.h +@@ -28,6 +28,9 @@ struct btrfs_log_ctx { + struct list_head list; + /* Only used for fast fsyncs. */ + struct list_head ordered_extents; ++ struct list_head conflict_inodes; ++ int num_conflict_inodes; ++ bool logging_conflict_inodes; + }; + + static inline void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx, +@@ -41,6 +44,9 @@ static inline void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx, + ctx->inode = inode; + INIT_LIST_HEAD(&ctx->list); + INIT_LIST_HEAD(&ctx->ordered_extents); ++ INIT_LIST_HEAD(&ctx->conflict_inodes); ++ ctx->num_conflict_inodes = 0; ++ ctx->logging_conflict_inodes = false; + } + + static inline void btrfs_release_log_ctx_extents(struct btrfs_log_ctx *ctx) +diff --git a/fs/buffer.c b/fs/buffer.c +index 55e762a58eb6..0a7ba84c1905 100644 +--- a/fs/buffer.c ++++ b/fs/buffer.c +@@ -52,8 +52,8 @@ + #include "internal.h" + + static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); +-static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, +- struct writeback_control *wbc); ++static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, ++ struct writeback_control *wbc); + + #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) + +@@ -2673,8 +2673,8 @@ static void end_bio_bh_io_sync(struct bio *bio) + bio_put(bio); + } + +-static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, +- struct writeback_control *wbc) ++static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, ++ struct writeback_control *wbc) + { + const enum req_op op = opf & REQ_OP_MASK; + struct bio *bio; +@@ -2717,12 +2717,11 @@ static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + } + + submit_bio(bio); +- return 0; + } + +-int submit_bh(blk_opf_t opf, struct buffer_head *bh) ++void submit_bh(blk_opf_t opf, struct buffer_head *bh) + { +- return submit_bh_wbc(opf, bh, NULL); ++ submit_bh_wbc(opf, bh, NULL); + } + EXPORT_SYMBOL(submit_bh); + +@@ -2801,8 +2800,6 @@ EXPORT_SYMBOL(write_dirty_buffer); + */ + int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) + { +- int ret = 0; +- + WARN_ON(atomic_read(&bh->b_count) < 1); + lock_buffer(bh); + if (test_clear_buffer_dirty(bh)) { +@@ -2817,14 +2814,14 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) + + get_bh(bh); + bh->b_end_io = end_buffer_write_sync; +- ret = submit_bh(REQ_OP_WRITE | op_flags, bh); ++ submit_bh(REQ_OP_WRITE | op_flags, bh); + wait_on_buffer(bh); +- if (!ret && !buffer_uptodate(bh)) +- ret = -EIO; ++ if (!buffer_uptodate(bh)) ++ return -EIO; + } else { + unlock_buffer(bh); + } +- return ret; ++ return 0; + } + EXPORT_SYMBOL(__sync_dirty_buffer); + +diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h +index 3bf9a6926798..1d2ed29278e2 100644 +--- a/fs/ext4/ext4.h ++++ b/fs/ext4/ext4.h +@@ -1120,8 +1120,8 @@ struct ext4_inode_info { + + /* mballoc */ + atomic_t i_prealloc_active; +- struct list_head i_prealloc_list; +- spinlock_t i_prealloc_lock; ++ struct rb_root i_prealloc_node; ++ rwlock_t i_prealloc_lock; + + /* extents status tree */ + struct ext4_es_tree i_es_tree; +@@ -1612,7 +1612,6 @@ struct ext4_sb_info { + unsigned int s_mb_stats; + unsigned int s_mb_order2_reqs; + unsigned int s_mb_group_prealloc; +- unsigned int s_mb_max_inode_prealloc; + unsigned int s_max_dir_size_kb; + /* where last allocation was done - for stream allocation */ + unsigned long s_mb_last_group; +@@ -3591,9 +3590,6 @@ extern bool empty_inline_dir(struct inode *dir, int *has_inline_data); + extern struct buffer_head *ext4_get_first_inline_block(struct inode *inode, + struct ext4_dir_entry_2 **parent_de, + int *retval); +-extern int ext4_inline_data_fiemap(struct inode *inode, +- struct fiemap_extent_info *fieinfo, +- int *has_inline, __u64 start, __u64 len); + extern void *ext4_read_inline_link(struct inode *inode); + + struct iomap; +@@ -3712,7 +3708,7 @@ extern int ext4_ext_insert_extent(handle_t *, struct inode *, + extern struct ext4_ext_path *ext4_find_extent(struct inode *, ext4_lblk_t, + struct ext4_ext_path **, + int flags); +-extern void ext4_ext_drop_refs(struct ext4_ext_path *); ++extern void ext4_free_ext_path(struct ext4_ext_path *); + extern int ext4_ext_check_inode(struct inode *inode); + extern ext4_lblk_t ext4_ext_next_allocated_block(struct ext4_ext_path *path); + extern int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, +diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c +index 5235974126bd..f1956288307f 100644 +--- a/fs/ext4/extents.c ++++ b/fs/ext4/extents.c +@@ -106,6 +106,25 @@ static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped) + return 0; + } + ++static void ext4_ext_drop_refs(struct ext4_ext_path *path) ++{ ++ int depth, i; ++ ++ if (!path) ++ return; ++ depth = path->p_depth; ++ for (i = 0; i <= depth; i++, path++) { ++ brelse(path->p_bh); ++ path->p_bh = NULL; ++ } ++} ++ ++void ext4_free_ext_path(struct ext4_ext_path *path) ++{ ++ ext4_ext_drop_refs(path); ++ kfree(path); ++} ++ + /* + * Make sure 'handle' has at least 'check_cred' credits. If not, restart + * transaction with 'restart_cred' credits. The function drops i_data_sem +@@ -636,8 +655,7 @@ int ext4_ext_precache(struct inode *inode) + ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED); + out: + up_read(&ei->i_data_sem); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + return ret; + } + +@@ -724,19 +742,6 @@ static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path, + #define ext4_ext_show_move(inode, path, newblock, level) + #endif + +-void ext4_ext_drop_refs(struct ext4_ext_path *path) +-{ +- int depth, i; +- +- if (!path) +- return; +- depth = path->p_depth; +- for (i = 0; i <= depth; i++, path++) { +- brelse(path->p_bh); +- path->p_bh = NULL; +- } +-} +- + /* + * ext4_ext_binsearch_idx: + * binary search for the closest index of the given block +@@ -955,8 +960,7 @@ ext4_find_extent(struct inode *inode, ext4_lblk_t block, + return path; + + err: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + if (orig_path) + *orig_path = NULL; + return ERR_PTR(ret); +@@ -2174,8 +2178,7 @@ int ext4_ext_insert_extent(handle_t *handle, struct inode *inode, + err = ext4_ext_dirty(handle, inode, path + path->p_depth); + + cleanup: +- ext4_ext_drop_refs(npath); +- kfree(npath); ++ ext4_free_ext_path(npath); + return err; + } + +@@ -3061,8 +3064,7 @@ int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start, + } + } + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + path = NULL; + if (err == -EAGAIN) + goto again; +@@ -4375,8 +4377,7 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, + allocated = map->m_len; + ext4_ext_show_leaf(inode, path); + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + + trace_ext4_ext_map_blocks_exit(inode, flags, map, + err ? err : allocated); +@@ -5245,8 +5246,7 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle, + break; + } + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + return ret; + } + +@@ -5538,15 +5538,13 @@ static int ext4_insert_range(struct file *file, loff_t offset, loff_t len) + EXT4_GET_BLOCKS_METADATA_NOFAIL); + } + +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + if (ret < 0) { + up_write(&EXT4_I(inode)->i_data_sem); + goto out_stop; + } + } else { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + } + + ret = ext4_es_remove_extent(inode, offset_lblk, +@@ -5766,10 +5764,8 @@ ext4_swap_extents(handle_t *handle, struct inode *inode1, + count -= len; + + repeat: +- ext4_ext_drop_refs(path1); +- kfree(path1); +- ext4_ext_drop_refs(path2); +- kfree(path2); ++ ext4_free_ext_path(path1); ++ ext4_free_ext_path(path2); + path1 = path2 = NULL; + } + return replaced_count; +@@ -5848,8 +5844,7 @@ int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu) + } + + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + + return err ? err : mapped; + } +@@ -5916,8 +5911,7 @@ int ext4_ext_replay_update_ex(struct inode *inode, ext4_lblk_t start, + ret = ext4_ext_dirty(NULL, inode, &path[path->p_depth]); + up_write(&EXT4_I(inode)->i_data_sem); + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + ext4_mark_inode_dirty(NULL, inode); + return ret; + } +@@ -5935,8 +5929,7 @@ void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end) + return; + ex = path[path->p_depth].p_ext; + if (!ex) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + ext4_mark_inode_dirty(NULL, inode); + return; + } +@@ -5949,8 +5942,7 @@ void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end) + ext4_ext_dirty(NULL, inode, &path[path->p_depth]); + up_write(&EXT4_I(inode)->i_data_sem); + ext4_mark_inode_dirty(NULL, inode); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + } + } + +@@ -5989,13 +5981,11 @@ int ext4_ext_replay_set_iblocks(struct inode *inode) + return PTR_ERR(path); + ex = path[path->p_depth].p_ext; + if (!ex) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + goto out; + } + end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + + /* Count the number of data blocks */ + cur = 0; +@@ -6025,30 +6015,26 @@ int ext4_ext_replay_set_iblocks(struct inode *inode) + if (IS_ERR(path)) + goto out; + numblks += path->p_depth; +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + while (cur < end) { + path = ext4_find_extent(inode, cur, NULL, 0); + if (IS_ERR(path)) + break; + ex = path[path->p_depth].p_ext; + if (!ex) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + return 0; + } + cur = max(cur + 1, le32_to_cpu(ex->ee_block) + + ext4_ext_get_actual_len(ex)); + ret = skip_hole(inode, &cur); + if (ret < 0) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + break; + } + path2 = ext4_find_extent(inode, cur, NULL, 0); + if (IS_ERR(path2)) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + break; + } + for (i = 0; i <= max(path->p_depth, path2->p_depth); i++) { +@@ -6062,10 +6048,8 @@ int ext4_ext_replay_set_iblocks(struct inode *inode) + if (cmp1 != cmp2 && cmp2 != 0) + numblks++; + } +- ext4_ext_drop_refs(path); +- ext4_ext_drop_refs(path2); +- kfree(path); +- kfree(path2); ++ ext4_free_ext_path(path); ++ ext4_free_ext_path(path2); + } + + out: +@@ -6092,13 +6076,11 @@ int ext4_ext_clear_bb(struct inode *inode) + return PTR_ERR(path); + ex = path[path->p_depth].p_ext; + if (!ex) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + return 0; + } + end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + + cur = 0; + while (cur < end) { +@@ -6117,8 +6099,7 @@ int ext4_ext_clear_bb(struct inode *inode) + ext4_fc_record_regions(inode->i_sb, inode->i_ino, + 0, path[j].p_block, 1, 1); + } +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + } + ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0); + ext4_fc_record_regions(inode->i_sb, inode->i_ino, +diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c +index 23167efda95e..cd0a861853e3 100644 +--- a/fs/ext4/extents_status.c ++++ b/fs/ext4/extents_status.c +@@ -667,8 +667,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode, + } + } + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + } + + static void ext4_es_insert_extent_ind_check(struct inode *inode, +diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c +index 2af962cbb835..ef05bfa87798 100644 +--- a/fs/ext4/fast_commit.c ++++ b/fs/ext4/fast_commit.c +@@ -229,6 +229,12 @@ __releases(&EXT4_SB(inode->i_sb)->s_fc_lock) + finish_wait(wq, &wait.wq_entry); + } + ++static bool ext4_fc_disabled(struct super_block *sb) ++{ ++ return (!test_opt2(sb, JOURNAL_FAST_COMMIT) || ++ (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); ++} ++ + /* + * Inform Ext4's fast about start of an inode update + * +@@ -240,8 +246,7 @@ void ext4_fc_start_update(struct inode *inode) + { + struct ext4_inode_info *ei = EXT4_I(inode); + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + restart: +@@ -265,8 +270,7 @@ void ext4_fc_stop_update(struct inode *inode) + { + struct ext4_inode_info *ei = EXT4_I(inode); + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + if (atomic_dec_and_test(&ei->i_fc_updates)) +@@ -283,8 +287,7 @@ void ext4_fc_del(struct inode *inode) + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_fc_dentry_update *fc_dentry; + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + restart: +@@ -337,8 +340,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl + struct ext4_sb_info *sbi = EXT4_SB(sb); + tid_t tid; + +- if (!test_opt2(sb, JOURNAL_FAST_COMMIT) || +- (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(sb)) + return; + + ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); +@@ -493,10 +495,8 @@ void __ext4_fc_track_unlink(handle_t *handle, + void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry) + { + struct inode *inode = d_inode(dentry); +- struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (sbi->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) +@@ -522,10 +522,8 @@ void __ext4_fc_track_link(handle_t *handle, + void ext4_fc_track_link(handle_t *handle, struct dentry *dentry) + { + struct inode *inode = d_inode(dentry); +- struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (sbi->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) +@@ -551,10 +549,8 @@ void __ext4_fc_track_create(handle_t *handle, struct inode *inode, + void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) + { + struct inode *inode = d_inode(dentry); +- struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (sbi->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) +@@ -576,22 +572,20 @@ static int __track_inode(struct inode *inode, void *arg, bool update) + + void ext4_fc_track_inode(handle_t *handle, struct inode *inode) + { +- struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + int ret; + + if (S_ISDIR(inode->i_mode)) + return; + ++ if (ext4_fc_disabled(inode->i_sb)) ++ return; ++ + if (ext4_should_journal_data(inode)) { + ext4_fc_mark_ineligible(inode->i_sb, + EXT4_FC_REASON_INODE_JOURNAL_DATA, handle); + return; + } + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (sbi->s_mount_state & EXT4_FC_REPLAY)) +- return; +- + if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) + return; + +@@ -634,15 +628,13 @@ static int __track_range(struct inode *inode, void *arg, bool update) + void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start, + ext4_lblk_t end) + { +- struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct __track_range_args args; + int ret; + + if (S_ISDIR(inode->i_mode)) + return; + +- if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || +- (sbi->s_mount_state & EXT4_FC_REPLAY)) ++ if (ext4_fc_disabled(inode->i_sb)) + return; + + if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) +@@ -710,10 +702,10 @@ static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) + * After allocating len, we should have space at least for a 0 byte + * padding. + */ +- if (len + sizeof(struct ext4_fc_tl) > bsize) ++ if (len + EXT4_FC_TAG_BASE_LEN > bsize) + return NULL; + +- if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) { ++ if (bsize - off - 1 > len + EXT4_FC_TAG_BASE_LEN) { + /* + * Only allocate from current buffer if we have enough space for + * this request AND we have space to add a zero byte padding. +@@ -730,10 +722,10 @@ static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) + /* Need to add PAD tag */ + tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off); + tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); +- pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl); ++ pad_len = bsize - off - 1 - EXT4_FC_TAG_BASE_LEN; + tl->fc_len = cpu_to_le16(pad_len); + if (crc) +- *crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl)); ++ *crc = ext4_chksum(sbi, *crc, tl, EXT4_FC_TAG_BASE_LEN); + if (pad_len > 0) + ext4_fc_memzero(sb, tl + 1, pad_len, crc); + ext4_fc_submit_bh(sb, false); +@@ -775,7 +767,7 @@ static int ext4_fc_write_tail(struct super_block *sb, u32 crc) + * ext4_fc_reserve_space takes care of allocating an extra block if + * there's no enough space on this block for accommodating this tail. + */ +- dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc); ++ dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc); + if (!dst) + return -ENOSPC; + +@@ -785,8 +777,8 @@ static int ext4_fc_write_tail(struct super_block *sb, u32 crc) + tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail)); + sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); + +- ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc); +- dst += sizeof(tl); ++ ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, &crc); ++ dst += EXT4_FC_TAG_BASE_LEN; + tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); + ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc); + dst += sizeof(tail.fc_tid); +@@ -808,15 +800,15 @@ static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val, + struct ext4_fc_tl tl; + u8 *dst; + +- dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc); ++ dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc); + if (!dst) + return false; + + tl.fc_tag = cpu_to_le16(tag); + tl.fc_len = cpu_to_le16(len); + +- ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); +- ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc); ++ ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc); ++ ext4_fc_memcpy(sb, dst + EXT4_FC_TAG_BASE_LEN, val, len, crc); + + return true; + } +@@ -828,8 +820,8 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, + struct ext4_fc_dentry_info fcd; + struct ext4_fc_tl tl; + int dlen = fc_dentry->fcd_name.len; +- u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen, +- crc); ++ u8 *dst = ext4_fc_reserve_space(sb, ++ EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc); + + if (!dst) + return false; +@@ -838,8 +830,8 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, + fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino); + tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op); + tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); +- ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); +- dst += sizeof(tl); ++ ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc); ++ dst += EXT4_FC_TAG_BASE_LEN; + ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc); + dst += sizeof(fcd); + ext4_fc_memcpy(sb, dst, fc_dentry->fcd_name.name, dlen, crc); +@@ -874,22 +866,25 @@ static int ext4_fc_write_inode(struct inode *inode, u32 *crc) + tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); + tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino)); + ++ ret = -ECANCELED; + dst = ext4_fc_reserve_space(inode->i_sb, +- sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc); ++ EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc); + if (!dst) +- return -ECANCELED; ++ goto err; + +- if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc)) +- return -ECANCELED; +- dst += sizeof(tl); ++ if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc)) ++ goto err; ++ dst += EXT4_FC_TAG_BASE_LEN; + if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc)) +- return -ECANCELED; ++ goto err; + dst += sizeof(fc_inode); + if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc), + inode_len, crc)) +- return -ECANCELED; +- +- return 0; ++ goto err; ++ ret = 0; ++err: ++ brelse(iloc.bh); ++ return ret; + } + + /* +@@ -1343,7 +1338,7 @@ struct dentry_info_args { + }; + + static inline void tl_to_darg(struct dentry_info_args *darg, +- struct ext4_fc_tl *tl, u8 *val) ++ struct ext4_fc_tl *tl, u8 *val) + { + struct ext4_fc_dentry_info fcd; + +@@ -1352,8 +1347,14 @@ static inline void tl_to_darg(struct dentry_info_args *darg, + darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); + darg->ino = le32_to_cpu(fcd.fc_ino); + darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); +- darg->dname_len = le16_to_cpu(tl->fc_len) - +- sizeof(struct ext4_fc_dentry_info); ++ darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info); ++} ++ ++static inline void ext4_fc_get_tl(struct ext4_fc_tl *tl, u8 *val) ++{ ++ memcpy(tl, val, EXT4_FC_TAG_BASE_LEN); ++ tl->fc_len = le16_to_cpu(tl->fc_len); ++ tl->fc_tag = le16_to_cpu(tl->fc_tag); + } + + /* Unlink replay function */ +@@ -1491,13 +1492,15 @@ static int ext4_fc_record_modified_inode(struct super_block *sb, int ino) + if (state->fc_modified_inodes[i] == ino) + return 0; + if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { +- state->fc_modified_inodes = krealloc( +- state->fc_modified_inodes, ++ int *fc_modified_inodes; ++ ++ fc_modified_inodes = krealloc(state->fc_modified_inodes, + sizeof(int) * (state->fc_modified_inodes_size + + EXT4_FC_REPLAY_REALLOC_INCREMENT), + GFP_KERNEL); +- if (!state->fc_modified_inodes) ++ if (!fc_modified_inodes) + return -ENOMEM; ++ state->fc_modified_inodes = fc_modified_inodes; + state->fc_modified_inodes_size += + EXT4_FC_REPLAY_REALLOC_INCREMENT; + } +@@ -1516,7 +1519,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl, + struct ext4_inode *raw_fc_inode; + struct inode *inode = NULL; + struct ext4_iloc iloc; +- int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag); ++ int inode_len, ino, ret, tag = tl->fc_tag; + struct ext4_extent_header *eh; + + memcpy(&fc_inode, val, sizeof(fc_inode)); +@@ -1541,7 +1544,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl, + if (ret) + goto out; + +- inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode); ++ inode_len = tl->fc_len - sizeof(struct ext4_fc_inode); + raw_inode = ext4_raw_inode(&iloc); + + memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); +@@ -1682,15 +1685,18 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, + if (replay && state->fc_regions_used != state->fc_regions_valid) + state->fc_regions_used = state->fc_regions_valid; + if (state->fc_regions_used == state->fc_regions_size) { ++ struct ext4_fc_alloc_region *fc_regions; ++ ++ fc_regions = krealloc(state->fc_regions, ++ sizeof(struct ext4_fc_alloc_region) * ++ (state->fc_regions_size + ++ EXT4_FC_REPLAY_REALLOC_INCREMENT), ++ GFP_KERNEL); ++ if (!fc_regions) ++ return -ENOMEM; + state->fc_regions_size += + EXT4_FC_REPLAY_REALLOC_INCREMENT; +- state->fc_regions = krealloc( +- state->fc_regions, +- state->fc_regions_size * +- sizeof(struct ext4_fc_alloc_region), +- GFP_KERNEL); +- if (!state->fc_regions) +- return -ENOMEM; ++ state->fc_regions = fc_regions; + } + region = &state->fc_regions[state->fc_regions_used++]; + region->ino = ino; +@@ -1770,8 +1776,7 @@ static int ext4_fc_replay_add_range(struct super_block *sb, + ret = ext4_ext_insert_extent( + NULL, inode, &path, &newex, 0); + up_write((&EXT4_I(inode)->i_data_sem)); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + if (ret) + goto out; + goto next; +@@ -1926,8 +1931,7 @@ static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) + for (j = 0; j < path->p_depth; j++) + ext4_mb_mark_bb(inode->i_sb, + path[j].p_block, 1, 1); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + } + cur += ret; + ext4_mb_mark_bb(inode->i_sb, map.m_pblk, +@@ -1972,6 +1976,34 @@ void ext4_fc_replay_cleanup(struct super_block *sb) + kfree(sbi->s_fc_replay_state.fc_modified_inodes); + } + ++static inline bool ext4_fc_tag_len_isvalid(struct ext4_fc_tl *tl, ++ u8 *val, u8 *end) ++{ ++ if (val + tl->fc_len > end) ++ return false; ++ ++ /* Here only check ADD_RANGE/TAIL/HEAD which will read data when do ++ * journal rescan before do CRC check. Other tags length check will ++ * rely on CRC check. ++ */ ++ switch (tl->fc_tag) { ++ case EXT4_FC_TAG_ADD_RANGE: ++ return (sizeof(struct ext4_fc_add_range) == tl->fc_len); ++ case EXT4_FC_TAG_TAIL: ++ return (sizeof(struct ext4_fc_tail) <= tl->fc_len); ++ case EXT4_FC_TAG_HEAD: ++ return (sizeof(struct ext4_fc_head) == tl->fc_len); ++ case EXT4_FC_TAG_DEL_RANGE: ++ case EXT4_FC_TAG_LINK: ++ case EXT4_FC_TAG_UNLINK: ++ case EXT4_FC_TAG_CREAT: ++ case EXT4_FC_TAG_INODE: ++ case EXT4_FC_TAG_PAD: ++ default: ++ return true; ++ } ++} ++ + /* + * Recovery Scan phase handler + * +@@ -2028,12 +2060,18 @@ static int ext4_fc_replay_scan(journal_t *journal, + } + + state->fc_replay_expected_off++; +- for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) { +- memcpy(&tl, cur, sizeof(tl)); +- val = cur + sizeof(tl); ++ for (cur = start; cur < end - EXT4_FC_TAG_BASE_LEN; ++ cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { ++ ext4_fc_get_tl(&tl, cur); ++ val = cur + EXT4_FC_TAG_BASE_LEN; ++ if (!ext4_fc_tag_len_isvalid(&tl, val, end)) { ++ ret = state->fc_replay_num_tags ? ++ JBD2_FC_REPLAY_STOP : -ECANCELED; ++ goto out_err; ++ } + ext4_debug("Scan phase, tag:%s, blk %lld\n", +- tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr); +- switch (le16_to_cpu(tl.fc_tag)) { ++ tag2str(tl.fc_tag), bh->b_blocknr); ++ switch (tl.fc_tag) { + case EXT4_FC_TAG_ADD_RANGE: + memcpy(&ext, val, sizeof(ext)); + ex = (struct ext4_extent *)&ext.fc_ex; +@@ -2053,13 +2091,13 @@ static int ext4_fc_replay_scan(journal_t *journal, + case EXT4_FC_TAG_PAD: + state->fc_cur_tag++; + state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, +- sizeof(tl) + le16_to_cpu(tl.fc_len)); ++ EXT4_FC_TAG_BASE_LEN + tl.fc_len); + break; + case EXT4_FC_TAG_TAIL: + state->fc_cur_tag++; + memcpy(&tail, val, sizeof(tail)); + state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, +- sizeof(tl) + ++ EXT4_FC_TAG_BASE_LEN + + offsetof(struct ext4_fc_tail, + fc_crc)); + if (le32_to_cpu(tail.fc_tid) == expected_tid && +@@ -2086,7 +2124,7 @@ static int ext4_fc_replay_scan(journal_t *journal, + } + state->fc_cur_tag++; + state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, +- sizeof(tl) + le16_to_cpu(tl.fc_len)); ++ EXT4_FC_TAG_BASE_LEN + tl.fc_len); + break; + default: + ret = state->fc_replay_num_tags ? +@@ -2141,19 +2179,20 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, + start = (u8 *)bh->b_data; + end = (__u8 *)bh->b_data + journal->j_blocksize - 1; + +- for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) { +- memcpy(&tl, cur, sizeof(tl)); +- val = cur + sizeof(tl); ++ for (cur = start; cur < end - EXT4_FC_TAG_BASE_LEN; ++ cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { ++ ext4_fc_get_tl(&tl, cur); ++ val = cur + EXT4_FC_TAG_BASE_LEN; + + if (state->fc_replay_num_tags == 0) { + ret = JBD2_FC_REPLAY_STOP; + ext4_fc_set_bitmaps_and_counters(sb); + break; + } +- ext4_debug("Replay phase, tag:%s\n", +- tag2str(le16_to_cpu(tl.fc_tag))); ++ ++ ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag)); + state->fc_replay_num_tags--; +- switch (le16_to_cpu(tl.fc_tag)) { ++ switch (tl.fc_tag) { + case EXT4_FC_TAG_LINK: + ret = ext4_fc_replay_link(sb, &tl, val); + break; +@@ -2174,19 +2213,18 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, + break; + case EXT4_FC_TAG_PAD: + trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0, +- le16_to_cpu(tl.fc_len), 0); ++ tl.fc_len, 0); + break; + case EXT4_FC_TAG_TAIL: +- trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0, +- le16_to_cpu(tl.fc_len), 0); ++ trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, ++ 0, tl.fc_len, 0); + memcpy(&tail, val, sizeof(tail)); + WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid); + break; + case EXT4_FC_TAG_HEAD: + break; + default: +- trace_ext4_fc_replay(sb, le16_to_cpu(tl.fc_tag), 0, +- le16_to_cpu(tl.fc_len), 0); ++ trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0); + ret = -ECANCELED; + break; + } +diff --git a/fs/ext4/fast_commit.h b/fs/ext4/fast_commit.h +index 1db12847a83b..a6154c3ed135 100644 +--- a/fs/ext4/fast_commit.h ++++ b/fs/ext4/fast_commit.h +@@ -70,6 +70,9 @@ struct ext4_fc_tail { + __le32 fc_crc; + }; + ++/* Tag base length */ ++#define EXT4_FC_TAG_BASE_LEN (sizeof(struct ext4_fc_tl)) ++ + /* + * Fast commit status codes + */ +diff --git a/fs/ext4/file.c b/fs/ext4/file.c +index 109d07629f81..847a2f806b8f 100644 +--- a/fs/ext4/file.c ++++ b/fs/ext4/file.c +@@ -528,6 +528,12 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) + ret = -EAGAIN; + goto out; + } ++ /* ++ * Make sure inline data cannot be created anymore since we are going ++ * to allocate blocks for DIO. We know the inode does not have any ++ * inline data now because ext4_dio_supported() checked for that. ++ */ ++ ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + + offset = iocb->ki_pos; + count = ret; +diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c +index fbd876e10a85..978cd21d7c0f 100644 +--- a/fs/ext4/inode.c ++++ b/fs/ext4/inode.c +@@ -1188,6 +1188,13 @@ static int ext4_write_begin(struct file *file, struct address_space *mapping, + page = grab_cache_page_write_begin(mapping, index); + if (!page) + return -ENOMEM; ++ /* ++ * The same as page allocation, we prealloc buffer heads before ++ * starting the handle. ++ */ ++ if (!page_has_buffers(page)) ++ create_empty_buffers(page, inode->i_sb->s_blocksize, 0); ++ + unlock_page(page); + + retry_journal: +@@ -5341,6 +5348,7 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + int error, rc = 0; + int orphan = 0; + const unsigned int ia_valid = attr->ia_valid; ++ bool inc_ivers = true; + + if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) + return -EIO; +@@ -5424,8 +5432,8 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + return -EINVAL; + } + +- if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size) +- inode_inc_iversion(inode); ++ if (attr->ia_size == inode->i_size) ++ inc_ivers = false; + + if (shrink) { + if (ext4_should_order_data(inode)) { +@@ -5527,6 +5535,8 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + } + + if (!error) { ++ if (inc_ivers) ++ inode_inc_iversion(inode); + setattr_copy(mnt_userns, inode, attr); + mark_inode_dirty(inode); + } +@@ -5730,9 +5740,6 @@ int ext4_mark_iloc_dirty(handle_t *handle, + } + ext4_fc_track_inode(handle, inode); + +- if (IS_I_VERSION(inode)) +- inode_inc_iversion(inode); +- + /* the do_update_inode consumes one bh->b_count */ + get_bh(iloc->bh); + +diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c +index 3cf3ec4b1c21..4d49c5cfb690 100644 +--- a/fs/ext4/ioctl.c ++++ b/fs/ext4/ioctl.c +@@ -452,6 +452,7 @@ static long swap_inode_boot_loader(struct super_block *sb, + swap_inode_data(inode, inode_bl); + + inode->i_ctime = inode_bl->i_ctime = current_time(inode); ++ inode_inc_iversion(inode); + + inode->i_generation = prandom_u32(); + inode_bl->i_generation = prandom_u32(); +@@ -665,6 +666,7 @@ static int ext4_ioctl_setflags(struct inode *inode, + ext4_set_inode_flags(inode, false); + + inode->i_ctime = current_time(inode); ++ inode_inc_iversion(inode); + + err = ext4_mark_iloc_dirty(handle, inode, &iloc); + flags_err: +@@ -775,6 +777,7 @@ static int ext4_ioctl_setproject(struct inode *inode, __u32 projid) + + EXT4_I(inode)->i_projid = kprojid; + inode->i_ctime = current_time(inode); ++ inode_inc_iversion(inode); + out_dirty: + rc = ext4_mark_iloc_dirty(handle, inode, &iloc); + if (!err) +@@ -1060,9 +1063,6 @@ static int ext4_ioctl_checkpoint(struct file *filp, unsigned long arg) + if (!EXT4_SB(sb)->s_journal) + return -ENODEV; + +- if (flags & ~EXT4_IOC_CHECKPOINT_FLAG_VALID) +- return -EINVAL; +- + if ((flags & JBD2_JOURNAL_FLUSH_DISCARD) && + !bdev_max_discard_sectors(EXT4_SB(sb)->s_journal->j_dev)) + return -EOPNOTSUPP; +@@ -1257,6 +1257,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err == 0) { + inode->i_ctime = current_time(inode); ++ inode_inc_iversion(inode); + inode->i_generation = generation; + err = ext4_mark_iloc_dirty(handle, inode, &iloc); + } +diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c +index 9dad93059945..9c446d87a61c 100644 +--- a/fs/ext4/mballoc.c ++++ b/fs/ext4/mballoc.c +@@ -3419,7 +3419,6 @@ int ext4_mb_init(struct super_block *sb) + sbi->s_mb_stats = MB_DEFAULT_STATS; + sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; + sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; +- sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC; + /* + * The default group preallocation is 512, which for 4k block + * sizes translates to 2 megabytes. However for bigalloc file +@@ -3984,6 +3983,128 @@ static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) + mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); + } + ++/* ++ * This function returns the next element to look at during inode ++ * PA rbtree walk. We assume that we have held the inode PA rbtree lock ++ * (ei->i_prealloc_lock) ++ * ++ * new_start The start of the range we want to compare ++ * cur_start The existing start that we are comparing against ++ * node The node of the rb_tree ++ */ ++static inline struct rb_node* ++ext4_mb_pa_rb_next_iter(ext4_lblk_t new_start, ext4_lblk_t cur_start, struct rb_node *node) ++{ ++ if (new_start < cur_start) ++ return node->rb_left; ++ else ++ return node->rb_right; ++} ++ ++static inline void ++ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac, ++ ext4_lblk_t start, ext4_lblk_t end) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); ++ struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); ++ struct ext4_prealloc_space *tmp_pa; ++ ext4_lblk_t tmp_pa_start, tmp_pa_end; ++ struct rb_node *iter; ++ ++ read_lock(&ei->i_prealloc_lock); ++ iter = ei->i_prealloc_node.rb_node; ++ while (iter) { ++ tmp_pa = rb_entry(iter, struct ext4_prealloc_space, ++ pa_node.inode_node); ++ tmp_pa_start = tmp_pa->pa_lstart; ++ tmp_pa_end = tmp_pa->pa_lstart + EXT4_C2B(sbi, tmp_pa->pa_len); ++ ++ spin_lock(&tmp_pa->pa_lock); ++ if (tmp_pa->pa_deleted == 0) ++ BUG_ON(!(start >= tmp_pa_end || end <= tmp_pa_start)); ++ spin_unlock(&tmp_pa->pa_lock); ++ ++ iter = ext4_mb_pa_rb_next_iter(start, tmp_pa_start, iter); ++ } ++ read_unlock(&ei->i_prealloc_lock); ++} ++/* ++ * Given an allocation context "ac" and a range "start", "end", check ++ * and adjust boundaries if the range overlaps with any of the existing ++ * preallocatoins stored in the corresponding inode of the allocation context. ++ * ++ * Parameters: ++ * ac allocation context ++ * start start of the new range ++ * end end of the new range ++ */ ++static inline void ++ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac, ++ ext4_lblk_t *start, ext4_lblk_t *end) ++{ ++ struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); ++ struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); ++ struct ext4_prealloc_space *tmp_pa; ++ struct rb_node *iter; ++ ext4_lblk_t new_start, new_end; ++ ext4_lblk_t tmp_pa_start, tmp_pa_end; ++ ++ new_start = *start; ++ new_end = *end; ++ ++ /* check we don't cross already preallocated blocks */ ++ read_lock(&ei->i_prealloc_lock); ++ for (iter = ei->i_prealloc_node.rb_node; iter; ++ iter = ext4_mb_pa_rb_next_iter(new_start, tmp_pa_start, iter)) { ++ tmp_pa = rb_entry(iter, struct ext4_prealloc_space, ++ pa_node.inode_node); ++ tmp_pa_start = tmp_pa->pa_lstart; ++ tmp_pa_end = tmp_pa->pa_lstart + EXT4_C2B(sbi, tmp_pa->pa_len); ++ ++ /* ++ * If pa is deleted, ignore overlaps and just iterate in rbtree ++ * based on tmp_pa_start ++ */ ++ if (tmp_pa->pa_deleted) { ++ continue; ++ } ++ spin_lock(&tmp_pa->pa_lock); ++ if (tmp_pa->pa_deleted) { ++ spin_unlock(&tmp_pa->pa_lock); ++ continue; ++ } ++ ++ /* PA must not overlap original request */ ++ BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end || ++ ac->ac_o_ex.fe_logical < tmp_pa_start)); ++ ++ /* skip PAs this normalized request doesn't overlap with */ ++ if (tmp_pa_start >= new_end || tmp_pa_end <= new_start) { ++ spin_unlock(&tmp_pa->pa_lock); ++ continue; ++ } ++ BUG_ON(tmp_pa_start <= new_start && tmp_pa_end >= new_end); ++ ++ /* adjust start or end to be adjacent to this pa */ ++ if (tmp_pa_end <= ac->ac_o_ex.fe_logical) { ++ BUG_ON(tmp_pa_end < new_start); ++ new_start = tmp_pa_end; ++ } else if (tmp_pa_start > ac->ac_o_ex.fe_logical) { ++ BUG_ON(tmp_pa_start > new_end); ++ new_end = tmp_pa_start; ++ } ++ spin_unlock(&tmp_pa->pa_lock); ++ } ++ read_unlock(&ei->i_prealloc_lock); ++ ++ /* XXX: extra loop to check we really don't overlap preallocations */ ++ ext4_mb_pa_assert_overlap(ac, new_start, new_end); ++ ++ *start = new_start; ++ *end = new_end; ++ return; ++} ++ + /* + * Normalization means making request better in terms of + * size and alignment +@@ -3998,8 +4119,6 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, + loff_t size, start_off; + loff_t orig_size __maybe_unused; + ext4_lblk_t start; +- struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); +- struct ext4_prealloc_space *pa; + + /* do normalize only data requests, metadata requests + do not need preallocation */ +@@ -4100,61 +4219,9 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac, + + end = start + size; + +- /* check we don't cross already preallocated blocks */ +- rcu_read_lock(); +- list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { +- ext4_lblk_t pa_end; ++ ext4_mb_pa_adjust_overlap(ac, &start, &end); + +- if (pa->pa_deleted) +- continue; +- spin_lock(&pa->pa_lock); +- if (pa->pa_deleted) { +- spin_unlock(&pa->pa_lock); +- continue; +- } +- +- pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), +- pa->pa_len); +- +- /* PA must not overlap original request */ +- BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || +- ac->ac_o_ex.fe_logical < pa->pa_lstart)); +- +- /* skip PAs this normalized request doesn't overlap with */ +- if (pa->pa_lstart >= end || pa_end <= start) { +- spin_unlock(&pa->pa_lock); +- continue; +- } +- BUG_ON(pa->pa_lstart <= start && pa_end >= end); +- +- /* adjust start or end to be adjacent to this pa */ +- if (pa_end <= ac->ac_o_ex.fe_logical) { +- BUG_ON(pa_end < start); +- start = pa_end; +- } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { +- BUG_ON(pa->pa_lstart > end); +- end = pa->pa_lstart; +- } +- spin_unlock(&pa->pa_lock); +- } +- rcu_read_unlock(); + size = end - start; +- +- /* XXX: extra loop to check we really don't overlap preallocations */ +- rcu_read_lock(); +- list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { +- ext4_lblk_t pa_end; +- +- spin_lock(&pa->pa_lock); +- if (pa->pa_deleted == 0) { +- pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), +- pa->pa_len); +- BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); +- } +- spin_unlock(&pa->pa_lock); +- } +- rcu_read_unlock(); +- + /* + * In this function "start" and "size" are normalized for better + * alignment and length such that we could preallocate more blocks. +@@ -4361,7 +4428,9 @@ ext4_mb_use_preallocated(struct ext4_allocation_context *ac) + int order, i; + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + struct ext4_locality_group *lg; +- struct ext4_prealloc_space *pa, *cpa = NULL; ++ struct ext4_prealloc_space *tmp_pa, *cpa = NULL; ++ ext4_lblk_t tmp_pa_start, tmp_pa_end; ++ struct rb_node *iter; + ext4_fsblk_t goal_block; + + /* only data can be preallocated */ +@@ -4369,35 +4438,46 @@ ext4_mb_use_preallocated(struct ext4_allocation_context *ac) + return false; + + /* first, try per-file preallocation */ +- rcu_read_lock(); +- list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { ++ read_lock(&ei->i_prealloc_lock); ++ for (iter = ei->i_prealloc_node.rb_node; iter; ++ iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, tmp_pa_start, iter)) { ++ tmp_pa = rb_entry(iter, struct ext4_prealloc_space, pa_node.inode_node); + + /* all fields in this condition don't change, + * so we can skip locking for them */ +- if (ac->ac_o_ex.fe_logical < pa->pa_lstart || +- ac->ac_o_ex.fe_logical >= (pa->pa_lstart + +- EXT4_C2B(sbi, pa->pa_len))) ++ tmp_pa_start = tmp_pa->pa_lstart; ++ tmp_pa_end = tmp_pa->pa_lstart + EXT4_C2B(sbi, tmp_pa->pa_len); ++ ++ /* original request start doesn't lie in this PA */ ++ if (ac->ac_o_ex.fe_logical < tmp_pa_start || ++ ac->ac_o_ex.fe_logical >= tmp_pa_end) { + continue; ++ } + + /* non-extent files can't have physical blocks past 2^32 */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && +- (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > +- EXT4_MAX_BLOCK_FILE_PHYS)) +- continue; ++ (tmp_pa->pa_pstart + EXT4_C2B(sbi, tmp_pa->pa_len) > ++ EXT4_MAX_BLOCK_FILE_PHYS)) { ++ /* ++ * Since PAs don't overlap, we won't find any ++ * other PA to satisfy this. ++ */ ++ break; ++ } + + /* found preallocated blocks, use them */ +- spin_lock(&pa->pa_lock); +- if (pa->pa_deleted == 0 && pa->pa_free) { +- atomic_inc(&pa->pa_count); +- ext4_mb_use_inode_pa(ac, pa); +- spin_unlock(&pa->pa_lock); ++ spin_lock(&tmp_pa->pa_lock); ++ if (tmp_pa->pa_deleted == 0 && tmp_pa->pa_free) { ++ atomic_inc(&tmp_pa->pa_count); ++ ext4_mb_use_inode_pa(ac, tmp_pa); ++ spin_unlock(&tmp_pa->pa_lock); + ac->ac_criteria = 10; +- rcu_read_unlock(); ++ read_unlock(&ei->i_prealloc_lock); + return true; + } +- spin_unlock(&pa->pa_lock); ++ spin_unlock(&tmp_pa->pa_lock); + } +- rcu_read_unlock(); ++ read_unlock(&ei->i_prealloc_lock); + + /* can we use group allocation? */ + if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) +@@ -4419,16 +4499,16 @@ ext4_mb_use_preallocated(struct ext4_allocation_context *ac) + */ + for (i = order; i < PREALLOC_TB_SIZE; i++) { + rcu_read_lock(); +- list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], +- pa_inode_list) { +- spin_lock(&pa->pa_lock); +- if (pa->pa_deleted == 0 && +- pa->pa_free >= ac->ac_o_ex.fe_len) { ++ list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[i], ++ pa_node.lg_list) { ++ spin_lock(&tmp_pa->pa_lock); ++ if (tmp_pa->pa_deleted == 0 && ++ tmp_pa->pa_free >= ac->ac_o_ex.fe_len) { + + cpa = ext4_mb_check_group_pa(goal_block, +- pa, cpa); ++ tmp_pa, cpa); + } +- spin_unlock(&pa->pa_lock); ++ spin_unlock(&tmp_pa->pa_lock); + } + rcu_read_unlock(); + } +@@ -4525,16 +4605,21 @@ static void ext4_mb_mark_pa_deleted(struct super_block *sb, + } + } + +-static void ext4_mb_pa_callback(struct rcu_head *head) ++static void inline ext4_mb_pa_free(struct ext4_prealloc_space *pa) + { +- struct ext4_prealloc_space *pa; +- pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); +- ++ BUG_ON(!pa); + BUG_ON(atomic_read(&pa->pa_count)); + BUG_ON(pa->pa_deleted == 0); + kmem_cache_free(ext4_pspace_cachep, pa); + } + ++static void ext4_mb_pa_callback(struct rcu_head *head) ++{ ++ struct ext4_prealloc_space *pa; ++ pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); ++ ext4_mb_pa_free(pa); ++} ++ + /* + * drops a reference to preallocated space descriptor + * if this was the last reference and the space is consumed +@@ -4544,6 +4629,7 @@ static void ext4_mb_put_pa(struct ext4_allocation_context *ac, + { + ext4_group_t grp; + ext4_fsblk_t grp_blk; ++ struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + + /* in this short window concurrent discard can set pa_deleted */ + spin_lock(&pa->pa_lock); +@@ -4588,11 +4674,52 @@ static void ext4_mb_put_pa(struct ext4_allocation_context *ac, + list_del(&pa->pa_group_list); + ext4_unlock_group(sb, grp); + +- spin_lock(pa->pa_obj_lock); +- list_del_rcu(&pa->pa_inode_list); +- spin_unlock(pa->pa_obj_lock); ++ if (pa->pa_type == MB_INODE_PA) { ++ write_lock(pa->pa_node_lock.inode_lock); ++ rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); ++ write_unlock(pa->pa_node_lock.inode_lock); ++ ext4_mb_pa_free(pa); ++ } else { ++ spin_lock(pa->pa_node_lock.lg_lock); ++ list_del_rcu(&pa->pa_node.lg_list); ++ spin_unlock(pa->pa_node_lock.lg_lock); ++ call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); ++ } ++} + +- call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); ++static void ext4_mb_rb_insert(struct rb_root *root, struct rb_node *new, ++ int (*cmp)(struct rb_node *, struct rb_node *)) ++{ ++ struct rb_node **iter = &root->rb_node, *parent = NULL; ++ ++ while (*iter) { ++ parent = *iter; ++ if (cmp(new, *iter) < 0) ++ iter = &((*iter)->rb_left); ++ else ++ iter = &((*iter)->rb_right); ++ } ++ ++ rb_link_node(new, parent, iter); ++ rb_insert_color(new, root); ++} ++ ++static int ext4_mb_pa_cmp(struct rb_node *new, struct rb_node *cur) ++{ ++ ext4_lblk_t cur_start, new_start; ++ struct ext4_prealloc_space *cur_pa = rb_entry(cur, ++ struct ext4_prealloc_space, ++ pa_node.inode_node); ++ struct ext4_prealloc_space *new_pa = rb_entry(new, ++ struct ext4_prealloc_space, ++ pa_node.inode_node); ++ cur_start = cur_pa->pa_lstart; ++ new_start = new_pa->pa_lstart; ++ ++ if (new_start < cur_start) ++ return -1; ++ else ++ return 1; + } + + /* +@@ -4658,7 +4785,6 @@ ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) + pa->pa_len = ac->ac_b_ex.fe_len; + pa->pa_free = pa->pa_len; + spin_lock_init(&pa->pa_lock); +- INIT_LIST_HEAD(&pa->pa_inode_list); + INIT_LIST_HEAD(&pa->pa_group_list); + pa->pa_deleted = 0; + pa->pa_type = MB_INODE_PA; +@@ -4673,14 +4799,15 @@ ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) + ei = EXT4_I(ac->ac_inode); + grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); + +- pa->pa_obj_lock = &ei->i_prealloc_lock; ++ pa->pa_node_lock.inode_lock = &ei->i_prealloc_lock; + pa->pa_inode = ac->ac_inode; + + list_add(&pa->pa_group_list, &grp->bb_prealloc_list); + +- spin_lock(pa->pa_obj_lock); +- list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); +- spin_unlock(pa->pa_obj_lock); ++ write_lock(pa->pa_node_lock.inode_lock); ++ ext4_mb_rb_insert(&ei->i_prealloc_node, &pa->pa_node.inode_node, ++ ext4_mb_pa_cmp); ++ write_unlock(pa->pa_node_lock.inode_lock); + atomic_inc(&ei->i_prealloc_active); + } + +@@ -4712,7 +4839,7 @@ ext4_mb_new_group_pa(struct ext4_allocation_context *ac) + pa->pa_len = ac->ac_b_ex.fe_len; + pa->pa_free = pa->pa_len; + spin_lock_init(&pa->pa_lock); +- INIT_LIST_HEAD(&pa->pa_inode_list); ++ INIT_LIST_HEAD(&pa->pa_node.lg_list); + INIT_LIST_HEAD(&pa->pa_group_list); + pa->pa_deleted = 0; + pa->pa_type = MB_GROUP_PA; +@@ -4728,7 +4855,7 @@ ext4_mb_new_group_pa(struct ext4_allocation_context *ac) + lg = ac->ac_lg; + BUG_ON(lg == NULL); + +- pa->pa_obj_lock = &lg->lg_prealloc_lock; ++ pa->pa_node_lock.lg_lock = &lg->lg_prealloc_lock; + pa->pa_inode = NULL; + + list_add(&pa->pa_group_list, &grp->bb_prealloc_list); +@@ -4846,6 +4973,7 @@ ext4_mb_discard_group_preallocations(struct super_block *sb, + struct ext4_prealloc_space *pa, *tmp; + struct list_head list; + struct ext4_buddy e4b; ++ struct ext4_inode_info *ei; + int err; + int free = 0; + +@@ -4904,17 +5032,26 @@ ext4_mb_discard_group_preallocations(struct super_block *sb, + list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { + + /* remove from object (inode or locality group) */ +- spin_lock(pa->pa_obj_lock); +- list_del_rcu(&pa->pa_inode_list); +- spin_unlock(pa->pa_obj_lock); ++ if (pa->pa_type == MB_GROUP_PA) { ++ spin_lock(pa->pa_node_lock.lg_lock); ++ list_del_rcu(&pa->pa_node.lg_list); ++ spin_unlock(pa->pa_node_lock.lg_lock); ++ } else { ++ write_lock(pa->pa_node_lock.inode_lock); ++ ei = EXT4_I(pa->pa_inode); ++ rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); ++ write_unlock(pa->pa_node_lock.inode_lock); ++ } + +- if (pa->pa_type == MB_GROUP_PA) ++ list_del(&pa->u.pa_tmp_list); ++ ++ if (pa->pa_type == MB_GROUP_PA) { + ext4_mb_release_group_pa(&e4b, pa); +- else ++ call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); ++ } else { + ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); +- +- list_del(&pa->u.pa_tmp_list); +- call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); ++ ext4_mb_pa_free(pa); ++ } + } + + ext4_unlock_group(sb, group); +@@ -4944,6 +5081,7 @@ void ext4_discard_preallocations(struct inode *inode, unsigned int needed) + ext4_group_t group = 0; + struct list_head list; + struct ext4_buddy e4b; ++ struct rb_node *iter; + int err; + + if (!S_ISREG(inode->i_mode)) { +@@ -4966,17 +5104,18 @@ void ext4_discard_preallocations(struct inode *inode, unsigned int needed) + + repeat: + /* first, collect all pa's in the inode */ +- spin_lock(&ei->i_prealloc_lock); +- while (!list_empty(&ei->i_prealloc_list) && needed) { +- pa = list_entry(ei->i_prealloc_list.prev, +- struct ext4_prealloc_space, pa_inode_list); +- BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); ++ write_lock(&ei->i_prealloc_lock); ++ for (iter = rb_first(&ei->i_prealloc_node); iter && needed; iter = rb_next(iter)) { ++ pa = rb_entry(iter, struct ext4_prealloc_space, ++ pa_node.inode_node); ++ BUG_ON(pa->pa_node_lock.inode_lock != &ei->i_prealloc_lock); ++ + spin_lock(&pa->pa_lock); + if (atomic_read(&pa->pa_count)) { + /* this shouldn't happen often - nobody should + * use preallocation while we're discarding it */ + spin_unlock(&pa->pa_lock); +- spin_unlock(&ei->i_prealloc_lock); ++ write_unlock(&ei->i_prealloc_lock); + ext4_msg(sb, KERN_ERR, + "uh-oh! used pa while discarding"); + WARN_ON(1); +@@ -4987,7 +5126,7 @@ void ext4_discard_preallocations(struct inode *inode, unsigned int needed) + if (pa->pa_deleted == 0) { + ext4_mb_mark_pa_deleted(sb, pa); + spin_unlock(&pa->pa_lock); +- list_del_rcu(&pa->pa_inode_list); ++ rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); + list_add(&pa->u.pa_tmp_list, &list); + needed--; + continue; +@@ -4995,7 +5134,7 @@ void ext4_discard_preallocations(struct inode *inode, unsigned int needed) + + /* someone is deleting pa right now */ + spin_unlock(&pa->pa_lock); +- spin_unlock(&ei->i_prealloc_lock); ++ write_unlock(&ei->i_prealloc_lock); + + /* we have to wait here because pa_deleted + * doesn't mean pa is already unlinked from +@@ -5012,7 +5151,7 @@ void ext4_discard_preallocations(struct inode *inode, unsigned int needed) + schedule_timeout_uninterruptible(HZ); + goto repeat; + } +- spin_unlock(&ei->i_prealloc_lock); ++ write_unlock(&ei->i_prealloc_lock); + + list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { + BUG_ON(pa->pa_type != MB_INODE_PA); +@@ -5044,7 +5183,7 @@ void ext4_discard_preallocations(struct inode *inode, unsigned int needed) + put_bh(bitmap_bh); + + list_del(&pa->u.pa_tmp_list); +- call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); ++ ext4_mb_pa_free(pa); + } + } + +@@ -5061,14 +5200,20 @@ static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) + return 0; + } + +-static void ext4_mb_pa_free(struct ext4_allocation_context *ac) ++static void ext4_mb_pa_put_free(struct ext4_allocation_context *ac) + { + struct ext4_prealloc_space *pa = ac->ac_pa; + + BUG_ON(!pa); + ac->ac_pa = NULL; + WARN_ON(!atomic_dec_and_test(&pa->pa_count)); +- kmem_cache_free(ext4_pspace_cachep, pa); ++ /* ++ * current function is only called due to an error or due to ++ * len of found blocks < len of requested blocks hence the PA has not ++ * been added to grp->bb_prealloc_list. So we don't need to lock it ++ */ ++ pa->pa_deleted = 1; ++ ext4_mb_pa_free(pa); + } + + #ifdef CONFIG_EXT4_DEBUG +@@ -5273,7 +5418,7 @@ ext4_mb_discard_lg_preallocations(struct super_block *sb, + + spin_lock(&lg->lg_prealloc_lock); + list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], +- pa_inode_list, ++ pa_node.lg_list, + lockdep_is_held(&lg->lg_prealloc_lock)) { + spin_lock(&pa->pa_lock); + if (atomic_read(&pa->pa_count)) { +@@ -5296,7 +5441,7 @@ ext4_mb_discard_lg_preallocations(struct super_block *sb, + ext4_mb_mark_pa_deleted(sb, pa); + spin_unlock(&pa->pa_lock); + +- list_del_rcu(&pa->pa_inode_list); ++ list_del_rcu(&pa->pa_node.lg_list); + list_add(&pa->u.pa_tmp_list, &discard_list); + + total_entries--; +@@ -5357,7 +5502,7 @@ static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) + /* Add the prealloc space to lg */ + spin_lock(&lg->lg_prealloc_lock); + list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], +- pa_inode_list, ++ pa_node.lg_list, + lockdep_is_held(&lg->lg_prealloc_lock)) { + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted) { +@@ -5366,8 +5511,8 @@ static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) + } + if (!added && pa->pa_free < tmp_pa->pa_free) { + /* Add to the tail of the previous entry */ +- list_add_tail_rcu(&pa->pa_inode_list, +- &tmp_pa->pa_inode_list); ++ list_add_tail_rcu(&pa->pa_node.lg_list, ++ &tmp_pa->pa_node.lg_list); + added = 1; + /* + * we want to count the total +@@ -5378,7 +5523,7 @@ static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) + lg_prealloc_count++; + } + if (!added) +- list_add_tail_rcu(&pa->pa_inode_list, ++ list_add_tail_rcu(&pa->pa_node.lg_list, + &lg->lg_prealloc_list[order]); + spin_unlock(&lg->lg_prealloc_lock); + +@@ -5391,30 +5536,11 @@ static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) + return ; + } + +-/* +- * if per-inode prealloc list is too long, trim some PA +- */ +-static void ext4_mb_trim_inode_pa(struct inode *inode) +-{ +- struct ext4_inode_info *ei = EXT4_I(inode); +- struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); +- int count, delta; +- +- count = atomic_read(&ei->i_prealloc_active); +- delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1; +- if (count > sbi->s_mb_max_inode_prealloc + delta) { +- count -= sbi->s_mb_max_inode_prealloc; +- ext4_discard_preallocations(inode, count); +- } +-} +- + /* + * release all resource we used in allocation + */ + static int ext4_mb_release_context(struct ext4_allocation_context *ac) + { +- struct inode *inode = ac->ac_inode; +- struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_prealloc_space *pa = ac->ac_pa; + if (pa) { +@@ -5434,23 +5560,13 @@ static int ext4_mb_release_context(struct ext4_allocation_context *ac) + * doesn't grow big. + */ + if (likely(pa->pa_free)) { +- spin_lock(pa->pa_obj_lock); +- list_del_rcu(&pa->pa_inode_list); +- spin_unlock(pa->pa_obj_lock); ++ spin_lock(pa->pa_node_lock.lg_lock); ++ list_del_rcu(&pa->pa_node.lg_list); ++ spin_unlock(pa->pa_node_lock.lg_lock); + ext4_mb_add_n_trim(ac); + } + } + +- if (pa->pa_type == MB_INODE_PA) { +- /* +- * treat per-inode prealloc list as a lru list, then try +- * to trim the least recently used PA. +- */ +- spin_lock(pa->pa_obj_lock); +- list_move(&pa->pa_inode_list, &ei->i_prealloc_list); +- spin_unlock(pa->pa_obj_lock); +- } +- + ext4_mb_put_pa(ac, ac->ac_sb, pa); + } + if (ac->ac_bitmap_page) +@@ -5460,7 +5576,6 @@ static int ext4_mb_release_context(struct ext4_allocation_context *ac) + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) + mutex_unlock(&ac->ac_lg->lg_mutex); + ext4_mb_collect_stats(ac); +- ext4_mb_trim_inode_pa(inode); + return 0; + } + +@@ -5617,13 +5732,13 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, + * So we have to free this pa here itself. + */ + if (*errp) { +- ext4_mb_pa_free(ac); ++ ext4_mb_pa_put_free(ac); + ext4_discard_allocated_blocks(ac); + goto errout; + } + if (ac->ac_status == AC_STATUS_FOUND && + ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) +- ext4_mb_pa_free(ac); ++ ext4_mb_pa_put_free(ac); + } + if (likely(ac->ac_status == AC_STATUS_FOUND)) { + *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); +@@ -5642,7 +5757,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, + * If block allocation fails then the pa allocated above + * needs to be freed here itself. + */ +- ext4_mb_pa_free(ac); ++ ext4_mb_pa_put_free(ac); + *errp = -ENOSPC; + } + +diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h +index dcda2a943cee..6d85ee8674a6 100644 +--- a/fs/ext4/mballoc.h ++++ b/fs/ext4/mballoc.h +@@ -73,11 +73,6 @@ + */ + #define MB_DEFAULT_GROUP_PREALLOC 512 + +-/* +- * maximum length of inode prealloc list +- */ +-#define MB_DEFAULT_MAX_INODE_PREALLOC 512 +- + /* + * Number of groups to search linearly before performing group scanning + * optimization. +@@ -114,7 +109,10 @@ struct ext4_free_data { + }; + + struct ext4_prealloc_space { +- struct list_head pa_inode_list; ++ union { ++ struct rb_node inode_node; /* for inode PA rbtree */ ++ struct list_head lg_list; /* for lg PAs */ ++ } pa_node; + struct list_head pa_group_list; + union { + struct list_head pa_tmp_list; +@@ -128,8 +126,11 @@ struct ext4_prealloc_space { + ext4_grpblk_t pa_len; /* len of preallocated chunk */ + ext4_grpblk_t pa_free; /* how many blocks are free */ + unsigned short pa_type; /* pa type. inode or group */ +- spinlock_t *pa_obj_lock; +- struct inode *pa_inode; /* hack, for history only */ ++ union { ++ rwlock_t *inode_lock; /* locks the rbtree holding this PA */ ++ spinlock_t *lg_lock; /* locks the lg list holding this PA */ ++ } pa_node_lock; ++ struct inode *pa_inode; /* used to get the inode during group discard */ + }; + + enum { +diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c +index 54e7d3c95fd7..0a220ec9862d 100644 +--- a/fs/ext4/migrate.c ++++ b/fs/ext4/migrate.c +@@ -56,8 +56,7 @@ static int finish_range(handle_t *handle, struct inode *inode, + retval = ext4_ext_insert_extent(handle, inode, &path, &newext, 0); + err_out: + up_write((&EXT4_I(inode)->i_data_sem)); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + lb->first_pblock = 0; + return retval; + } +diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c +index 701f1d6a217f..044e34cd835c 100644 +--- a/fs/ext4/move_extent.c ++++ b/fs/ext4/move_extent.c +@@ -32,8 +32,7 @@ get_ext_path(struct inode *inode, ext4_lblk_t lblock, + if (IS_ERR(path)) + return PTR_ERR(path); + if (path[ext_depth(inode)].p_ext == NULL) { +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + *ppath = NULL; + return -ENODATA; + } +@@ -103,12 +102,10 @@ mext_check_coverage(struct inode *inode, ext4_lblk_t from, ext4_lblk_t count, + if (unwritten != ext4_ext_is_unwritten(ext)) + goto out; + from += ext4_ext_get_actual_len(ext); +- ext4_ext_drop_refs(path); + } + ret = 1; + out: +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + return ret; + } + +@@ -472,19 +469,17 @@ mext_check_arguments(struct inode *orig_inode, + if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode)) + return -EPERM; + +- /* Ext4 move extent does not support swapfile */ ++ /* Ext4 move extent does not support swap files */ + if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) { +- ext4_debug("ext4 move extent: The argument files should " +- "not be swapfile [ino:orig %lu, donor %lu]\n", ++ ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %lu, donor %lu]\n", + orig_inode->i_ino, donor_inode->i_ino); +- return -EBUSY; ++ return -ETXTBSY; + } + + if (ext4_is_quota_file(orig_inode) && ext4_is_quota_file(donor_inode)) { +- ext4_debug("ext4 move extent: The argument files should " +- "not be quota files [ino:orig %lu, donor %lu]\n", ++ ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %lu, donor %lu]\n", + orig_inode->i_ino, donor_inode->i_ino); +- return -EBUSY; ++ return -EOPNOTSUPP; + } + + /* Ext4 move extent supports only extent based file */ +@@ -631,11 +626,11 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk, + if (ret) + goto out; + ex = path[path->p_depth].p_ext; +- next_blk = ext4_ext_next_allocated_block(path); + cur_blk = le32_to_cpu(ex->ee_block); + cur_len = ext4_ext_get_actual_len(ex); + /* Check hole before the start pos */ + if (cur_blk + cur_len - 1 < o_start) { ++ next_blk = ext4_ext_next_allocated_block(path); + if (next_blk == EXT_MAX_BLOCKS) { + ret = -ENODATA; + goto out; +@@ -663,7 +658,7 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk, + donor_page_index = d_start >> (PAGE_SHIFT - + donor_inode->i_blkbits); + offset_in_page = o_start % blocks_per_page; +- if (cur_len > blocks_per_page- offset_in_page) ++ if (cur_len > blocks_per_page - offset_in_page) + cur_len = blocks_per_page - offset_in_page; + /* + * Up semaphore to avoid following problems: +@@ -694,8 +689,7 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk, + ext4_discard_preallocations(donor_inode, 0); + } + +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + ext4_double_up_write_data_sem(orig_inode, donor_inode); + unlock_two_nondirectories(orig_inode, donor_inode); + +diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c +index 3a31b662f661..4183a4cb4a21 100644 +--- a/fs/ext4/namei.c ++++ b/fs/ext4/namei.c +@@ -85,15 +85,20 @@ static struct buffer_head *ext4_append(handle_t *handle, + return bh; + inode->i_size += inode->i_sb->s_blocksize; + EXT4_I(inode)->i_disksize = inode->i_size; ++ err = ext4_mark_inode_dirty(handle, inode); ++ if (err) ++ goto out; + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); +- if (err) { +- brelse(bh); +- ext4_std_error(inode->i_sb, err); +- return ERR_PTR(err); +- } ++ if (err) ++ goto out; + return bh; ++ ++out: ++ brelse(bh); ++ ext4_std_error(inode->i_sb, err); ++ return ERR_PTR(err); + } + + static int ext4_dx_csum_verify(struct inode *inode, +@@ -126,7 +131,7 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode, + struct ext4_dir_entry *dirent; + int is_dx_block = 0; + +- if (block >= inode->i_size) { ++ if (block >= inode->i_size >> inode->i_blkbits) { + ext4_error_inode(inode, func, line, block, + "Attempting to read directory block (%u) that is past i_size (%llu)", + block, inode->i_size); +diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c +index fea2a68d067b..6dfe9ccae0c5 100644 +--- a/fs/ext4/resize.c ++++ b/fs/ext4/resize.c +@@ -2122,7 +2122,7 @@ int ext4_resize_fs(struct super_block *sb, ext4_fsblk_t n_blocks_count) + goto out; + } + +- if (ext4_blocks_count(es) == n_blocks_count) ++ if (ext4_blocks_count(es) == n_blocks_count && n_blocks_count_retry == 0) + goto out; + + err = ext4_alloc_flex_bg_array(sb, n_group + 1); +diff --git a/fs/ext4/super.c b/fs/ext4/super.c +index 9a66abcca1a8..c75869419621 100644 +--- a/fs/ext4/super.c ++++ b/fs/ext4/super.c +@@ -205,19 +205,12 @@ int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, bh_end_io_t *end_io + + int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait) + { +- if (trylock_buffer(bh)) { +- if (wait) +- return ext4_read_bh(bh, op_flags, NULL); ++ lock_buffer(bh); ++ if (!wait) { + ext4_read_bh_nowait(bh, op_flags, NULL); + return 0; + } +- if (wait) { +- wait_on_buffer(bh); +- if (buffer_uptodate(bh)) +- return 0; +- return -EIO; +- } +- return 0; ++ return ext4_read_bh(bh, op_flags, NULL); + } + + /* +@@ -264,7 +257,8 @@ void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block) + struct buffer_head *bh = sb_getblk_gfp(sb, block, 0); + + if (likely(bh)) { +- ext4_read_bh_lock(bh, REQ_RAHEAD, false); ++ if (trylock_buffer(bh)) ++ ext4_read_bh_nowait(bh, REQ_RAHEAD, NULL); + brelse(bh); + } + } +@@ -1330,9 +1324,9 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) + + inode_set_iversion(&ei->vfs_inode, 1); + spin_lock_init(&ei->i_raw_lock); +- INIT_LIST_HEAD(&ei->i_prealloc_list); ++ ei->i_prealloc_node = RB_ROOT; + atomic_set(&ei->i_prealloc_active, 0); +- spin_lock_init(&ei->i_prealloc_lock); ++ rwlock_init(&ei->i_prealloc_lock); + ext4_es_init_tree(&ei->i_es_tree); + rwlock_init(&ei->i_es_lock); + INIT_LIST_HEAD(&ei->i_es_list); +@@ -1576,7 +1570,7 @@ enum { + Opt_bsd_df, Opt_minix_df, Opt_grpid, Opt_nogrpid, + Opt_resgid, Opt_resuid, Opt_sb, + Opt_nouid32, Opt_debug, Opt_removed, +- Opt_user_xattr, Opt_nouser_xattr, Opt_acl, Opt_noacl, ++ Opt_user_xattr, Opt_acl, + Opt_auto_da_alloc, Opt_noauto_da_alloc, Opt_noload, + Opt_commit, Opt_min_batch_time, Opt_max_batch_time, Opt_journal_dev, + Opt_journal_path, Opt_journal_checksum, Opt_journal_async_commit, +@@ -1585,7 +1579,7 @@ enum { + Opt_inlinecrypt, + Opt_usrjquota, Opt_grpjquota, Opt_quota, + Opt_noquota, Opt_barrier, Opt_nobarrier, Opt_err, +- Opt_usrquota, Opt_grpquota, Opt_prjquota, Opt_i_version, ++ Opt_usrquota, Opt_grpquota, Opt_prjquota, + Opt_dax, Opt_dax_always, Opt_dax_inode, Opt_dax_never, + Opt_stripe, Opt_delalloc, Opt_nodelalloc, Opt_warn_on_error, + Opt_nowarn_on_error, Opt_mblk_io_submit, Opt_debug_want_extra_isize, +@@ -1662,9 +1656,7 @@ static const struct fs_parameter_spec ext4_param_specs[] = { + fsparam_flag ("oldalloc", Opt_removed), + fsparam_flag ("orlov", Opt_removed), + fsparam_flag ("user_xattr", Opt_user_xattr), +- fsparam_flag ("nouser_xattr", Opt_nouser_xattr), + fsparam_flag ("acl", Opt_acl), +- fsparam_flag ("noacl", Opt_noacl), + fsparam_flag ("norecovery", Opt_noload), + fsparam_flag ("noload", Opt_noload), + fsparam_flag ("bh", Opt_removed), +@@ -1694,7 +1686,7 @@ static const struct fs_parameter_spec ext4_param_specs[] = { + fsparam_flag ("barrier", Opt_barrier), + fsparam_u32 ("barrier", Opt_barrier), + fsparam_flag ("nobarrier", Opt_nobarrier), +- fsparam_flag ("i_version", Opt_i_version), ++ fsparam_flag ("i_version", Opt_removed), + fsparam_flag ("dax", Opt_dax), + fsparam_enum ("dax", Opt_dax_type, ext4_param_dax), + fsparam_u32 ("stripe", Opt_stripe), +@@ -1814,13 +1806,10 @@ static const struct mount_opts { + {Opt_journal_ioprio, 0, MOPT_NO_EXT2}, + {Opt_data, 0, MOPT_NO_EXT2}, + {Opt_user_xattr, EXT4_MOUNT_XATTR_USER, MOPT_SET}, +- {Opt_nouser_xattr, EXT4_MOUNT_XATTR_USER, MOPT_CLEAR}, + #ifdef CONFIG_EXT4_FS_POSIX_ACL + {Opt_acl, EXT4_MOUNT_POSIX_ACL, MOPT_SET}, +- {Opt_noacl, EXT4_MOUNT_POSIX_ACL, MOPT_CLEAR}, + #else + {Opt_acl, 0, MOPT_NOSUPPORT}, +- {Opt_noacl, 0, MOPT_NOSUPPORT}, + #endif + {Opt_nouid32, EXT4_MOUNT_NO_UID32, MOPT_SET}, + {Opt_debug, EXT4_MOUNT_DEBUG, MOPT_SET}, +@@ -2120,10 +2109,6 @@ static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param) + else + return note_qf_name(fc, GRPQUOTA, param); + #endif +- case Opt_noacl: +- case Opt_nouser_xattr: +- ext4_msg(NULL, KERN_WARNING, deprecated_msg, param->key, "3.5"); +- break; + case Opt_sb: + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + ext4_msg(NULL, KERN_WARNING, +@@ -2140,11 +2125,6 @@ static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param) + case Opt_abort: + ctx_set_mount_flag(ctx, EXT4_MF_FS_ABORTED); + return 0; +- case Opt_i_version: +- ext4_msg(NULL, KERN_WARNING, deprecated_msg, param->key, "5.20"); +- ext4_msg(NULL, KERN_WARNING, "Use iversion instead\n"); +- ctx_set_flags(ctx, SB_I_VERSION); +- return 0; + case Opt_inlinecrypt: + #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + ctx_set_flags(ctx, SB_INLINECRYPT); +@@ -2814,14 +2794,6 @@ static void ext4_apply_options(struct fs_context *fc, struct super_block *sb) + sb->s_flags &= ~ctx->mask_s_flags; + sb->s_flags |= ctx->vals_s_flags; + +- /* +- * i_version differs from common mount option iversion so we have +- * to let vfs know that it was set, otherwise it would get cleared +- * on remount +- */ +- if (ctx->mask_s_flags & SB_I_VERSION) +- fc->sb_flags |= SB_I_VERSION; +- + #define APPLY(X) ({ if (ctx->spec & EXT4_SPEC_##X) sbi->X = ctx->X; }) + APPLY(s_commit_interval); + APPLY(s_stripe); +@@ -2970,8 +2942,6 @@ static int _ext4_show_options(struct seq_file *seq, struct super_block *sb, + SEQ_OPTS_PRINT("min_batch_time=%u", sbi->s_min_batch_time); + if (nodefs || sbi->s_max_batch_time != EXT4_DEF_MAX_BATCH_TIME) + SEQ_OPTS_PRINT("max_batch_time=%u", sbi->s_max_batch_time); +- if (sb->s_flags & SB_I_VERSION) +- SEQ_OPTS_PUTS("i_version"); + if (nodefs || sbi->s_stripe) + SEQ_OPTS_PRINT("stripe=%lu", sbi->s_stripe); + if (nodefs || EXT4_MOUNT_DATA_FLAGS & +@@ -3767,6 +3737,7 @@ static int ext4_lazyinit_thread(void *arg) + unsigned long next_wakeup, cur; + + BUG_ON(NULL == eli); ++ set_freezable(); + + cont_thread: + while (true) { +@@ -3982,9 +3953,9 @@ int ext4_register_li_request(struct super_block *sb, + goto out; + } + +- if (test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS) && +- (first_not_zeroed == ngroups || sb_rdonly(sb) || +- !test_opt(sb, INIT_INODE_TABLE))) ++ if (sb_rdonly(sb) || ++ (test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS) && ++ (first_not_zeroed == ngroups || !test_opt(sb, INIT_INODE_TABLE)))) + goto out; + + elr = ext4_li_request_new(sb, first_not_zeroed); +@@ -4311,112 +4282,10 @@ static struct ext4_sb_info *ext4_alloc_sbi(struct super_block *sb) + return NULL; + } + +-static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ++static void ext4_set_def_opts(struct super_block *sb, ++ struct ext4_super_block *es) + { +- struct buffer_head *bh, **group_desc; +- struct ext4_super_block *es = NULL; +- struct ext4_sb_info *sbi = EXT4_SB(sb); +- struct flex_groups **flex_groups; +- ext4_fsblk_t block; +- ext4_fsblk_t logical_sb_block; +- unsigned long offset = 0; + unsigned long def_mount_opts; +- struct inode *root; +- int ret = -ENOMEM; +- int blocksize, clustersize; +- unsigned int db_count; +- unsigned int i; +- int needs_recovery, has_huge_files; +- __u64 blocks_count; +- int err = 0; +- ext4_group_t first_not_zeroed; +- struct ext4_fs_context *ctx = fc->fs_private; +- int silent = fc->sb_flags & SB_SILENT; +- +- /* Set defaults for the variables that will be set during parsing */ +- if (!(ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO)) +- ctx->journal_ioprio = DEFAULT_JOURNAL_IOPRIO; +- +- sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; +- sbi->s_sectors_written_start = +- part_stat_read(sb->s_bdev, sectors[STAT_WRITE]); +- +- /* -EINVAL is default */ +- ret = -EINVAL; +- blocksize = sb_min_blocksize(sb, EXT4_MIN_BLOCK_SIZE); +- if (!blocksize) { +- ext4_msg(sb, KERN_ERR, "unable to set blocksize"); +- goto out_fail; +- } +- +- /* +- * The ext4 superblock will not be buffer aligned for other than 1kB +- * block sizes. We need to calculate the offset from buffer start. +- */ +- if (blocksize != EXT4_MIN_BLOCK_SIZE) { +- logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; +- offset = do_div(logical_sb_block, blocksize); +- } else { +- logical_sb_block = sbi->s_sb_block; +- } +- +- bh = ext4_sb_bread_unmovable(sb, logical_sb_block); +- if (IS_ERR(bh)) { +- ext4_msg(sb, KERN_ERR, "unable to read superblock"); +- ret = PTR_ERR(bh); +- goto out_fail; +- } +- /* +- * Note: s_es must be initialized as soon as possible because +- * some ext4 macro-instructions depend on its value +- */ +- es = (struct ext4_super_block *) (bh->b_data + offset); +- sbi->s_es = es; +- sb->s_magic = le16_to_cpu(es->s_magic); +- if (sb->s_magic != EXT4_SUPER_MAGIC) +- goto cantfind_ext4; +- sbi->s_kbytes_written = le64_to_cpu(es->s_kbytes_written); +- +- /* Warn if metadata_csum and gdt_csum are both set. */ +- if (ext4_has_feature_metadata_csum(sb) && +- ext4_has_feature_gdt_csum(sb)) +- ext4_warning(sb, "metadata_csum and uninit_bg are " +- "redundant flags; please run fsck."); +- +- /* Check for a known checksum algorithm */ +- if (!ext4_verify_csum_type(sb, es)) { +- ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " +- "unknown checksum algorithm."); +- silent = 1; +- goto cantfind_ext4; +- } +- ext4_setup_csum_trigger(sb, EXT4_JTR_ORPHAN_FILE, +- ext4_orphan_file_block_trigger); +- +- /* Load the checksum driver */ +- sbi->s_chksum_driver = crypto_alloc_shash("crc32c", 0, 0); +- if (IS_ERR(sbi->s_chksum_driver)) { +- ext4_msg(sb, KERN_ERR, "Cannot load crc32c driver."); +- ret = PTR_ERR(sbi->s_chksum_driver); +- sbi->s_chksum_driver = NULL; +- goto failed_mount; +- } +- +- /* Check superblock checksum */ +- if (!ext4_superblock_csum_verify(sb, es)) { +- ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " +- "invalid superblock checksum. Run e2fsck?"); +- silent = 1; +- ret = -EFSBADCRC; +- goto cantfind_ext4; +- } +- +- /* Precompute checksum seed for all metadata */ +- if (ext4_has_feature_csum_seed(sb)) +- sbi->s_csum_seed = le32_to_cpu(es->s_checksum_seed); +- else if (ext4_has_metadata_csum(sb) || ext4_has_feature_ea_inode(sb)) +- sbi->s_csum_seed = ext4_chksum(sbi, ~0, es->s_uuid, +- sizeof(es->s_uuid)); + + /* Set defaults before we parse the mount options */ + def_mount_opts = le32_to_cpu(es->s_default_mount_opts); +@@ -4445,9 +4314,9 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + else if ((def_mount_opts & EXT4_DEFM_JMODE) == EXT4_DEFM_JMODE_WBACK) + set_opt(sb, WRITEBACK_DATA); + +- if (le16_to_cpu(sbi->s_es->s_errors) == EXT4_ERRORS_PANIC) ++ if (le16_to_cpu(es->s_errors) == EXT4_ERRORS_PANIC) + set_opt(sb, ERRORS_PANIC); +- else if (le16_to_cpu(sbi->s_es->s_errors) == EXT4_ERRORS_CONTINUE) ++ else if (le16_to_cpu(es->s_errors) == EXT4_ERRORS_CONTINUE) + set_opt(sb, ERRORS_CONT); + else + set_opt(sb, ERRORS_RO); +@@ -4456,12 +4325,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + if (def_mount_opts & EXT4_DEFM_DISCARD) + set_opt(sb, DISCARD); + +- sbi->s_resuid = make_kuid(&init_user_ns, le16_to_cpu(es->s_def_resuid)); +- sbi->s_resgid = make_kgid(&init_user_ns, le16_to_cpu(es->s_def_resgid)); +- sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; +- sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; +- sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; +- + if ((def_mount_opts & EXT4_DEFM_NOBARRIER) == 0) + set_opt(sb, BARRIER); + +@@ -4473,31 +4336,96 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + ((def_mount_opts & EXT4_DEFM_NODELALLOC) == 0)) + set_opt(sb, DELALLOC); + +- /* +- * set default s_li_wait_mult for lazyinit, for the case there is +- * no mount option specified. +- */ +- sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; ++ if (sb->s_blocksize == PAGE_SIZE) ++ set_opt(sb, DIOREAD_NOLOCK); ++} + +- if (le32_to_cpu(es->s_log_block_size) > +- (EXT4_MAX_BLOCK_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { +- ext4_msg(sb, KERN_ERR, +- "Invalid log block size: %u", +- le32_to_cpu(es->s_log_block_size)); +- goto failed_mount; +- } +- if (le32_to_cpu(es->s_log_cluster_size) > +- (EXT4_MAX_CLUSTER_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { +- ext4_msg(sb, KERN_ERR, +- "Invalid log cluster size: %u", +- le32_to_cpu(es->s_log_cluster_size)); +- goto failed_mount; ++static int ext4_handle_clustersize(struct super_block *sb) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ struct ext4_super_block *es = sbi->s_es; ++ int clustersize; ++ ++ /* Handle clustersize */ ++ clustersize = BLOCK_SIZE << le32_to_cpu(es->s_log_cluster_size); ++ if (ext4_has_feature_bigalloc(sb)) { ++ if (clustersize < sb->s_blocksize) { ++ ext4_msg(sb, KERN_ERR, ++ "cluster size (%d) smaller than " ++ "block size (%lu)", clustersize, sb->s_blocksize); ++ return -EINVAL; ++ } ++ sbi->s_cluster_bits = le32_to_cpu(es->s_log_cluster_size) - ++ le32_to_cpu(es->s_log_block_size); ++ sbi->s_clusters_per_group = ++ le32_to_cpu(es->s_clusters_per_group); ++ if (sbi->s_clusters_per_group > sb->s_blocksize * 8) { ++ ext4_msg(sb, KERN_ERR, ++ "#clusters per group too big: %lu", ++ sbi->s_clusters_per_group); ++ return -EINVAL; ++ } ++ if (sbi->s_blocks_per_group != ++ (sbi->s_clusters_per_group * (clustersize / sb->s_blocksize))) { ++ ext4_msg(sb, KERN_ERR, "blocks per group (%lu) and " ++ "clusters per group (%lu) inconsistent", ++ sbi->s_blocks_per_group, ++ sbi->s_clusters_per_group); ++ return -EINVAL; ++ } ++ } else { ++ if (clustersize != sb->s_blocksize) { ++ ext4_msg(sb, KERN_ERR, ++ "fragment/cluster size (%d) != " ++ "block size (%lu)", clustersize, sb->s_blocksize); ++ return -EINVAL; ++ } ++ if (sbi->s_blocks_per_group > sb->s_blocksize * 8) { ++ ext4_msg(sb, KERN_ERR, ++ "#blocks per group too big: %lu", ++ sbi->s_blocks_per_group); ++ return -EINVAL; ++ } ++ sbi->s_clusters_per_group = sbi->s_blocks_per_group; ++ sbi->s_cluster_bits = 0; + } ++ sbi->s_cluster_ratio = clustersize / sb->s_blocksize; + +- blocksize = EXT4_MIN_BLOCK_SIZE << le32_to_cpu(es->s_log_block_size); ++ /* Do we have standard group size of clustersize * 8 blocks ? */ ++ if (sbi->s_blocks_per_group == clustersize << 3) ++ set_opt2(sb, STD_GROUP_SIZE); + +- if (blocksize == PAGE_SIZE) +- set_opt(sb, DIOREAD_NOLOCK); ++ return 0; ++} ++ ++static void ext4_fast_commit_init(struct super_block *sb) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ ++ /* Initialize fast commit stuff */ ++ atomic_set(&sbi->s_fc_subtid, 0); ++ INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_MAIN]); ++ INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_STAGING]); ++ INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_MAIN]); ++ INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_STAGING]); ++ sbi->s_fc_bytes = 0; ++ ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); ++ sbi->s_fc_ineligible_tid = 0; ++ spin_lock_init(&sbi->s_fc_lock); ++ memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); ++ sbi->s_fc_replay_state.fc_regions = NULL; ++ sbi->s_fc_replay_state.fc_regions_size = 0; ++ sbi->s_fc_replay_state.fc_regions_used = 0; ++ sbi->s_fc_replay_state.fc_regions_valid = 0; ++ sbi->s_fc_replay_state.fc_modified_inodes = NULL; ++ sbi->s_fc_replay_state.fc_modified_inodes_size = 0; ++ sbi->s_fc_replay_state.fc_modified_inodes_used = 0; ++} ++ ++static int ext4_inode_info_init(struct super_block *sb, ++ struct ext4_super_block *es) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (le32_to_cpu(es->s_rev_level) == EXT4_GOOD_OLD_REV) { + sbi->s_inode_size = EXT4_GOOD_OLD_INODE_SIZE; +@@ -4508,16 +4436,16 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + if (sbi->s_first_ino < EXT4_GOOD_OLD_FIRST_INO) { + ext4_msg(sb, KERN_ERR, "invalid first ino: %u", + sbi->s_first_ino); +- goto failed_mount; ++ return -EINVAL; + } + if ((sbi->s_inode_size < EXT4_GOOD_OLD_INODE_SIZE) || + (!is_power_of_2(sbi->s_inode_size)) || +- (sbi->s_inode_size > blocksize)) { ++ (sbi->s_inode_size > sb->s_blocksize)) { + ext4_msg(sb, KERN_ERR, + "unsupported inode size: %d", + sbi->s_inode_size); +- ext4_msg(sb, KERN_ERR, "blocksize: %d", blocksize); +- goto failed_mount; ++ ext4_msg(sb, KERN_ERR, "blocksize: %lu", sb->s_blocksize); ++ return -EINVAL; + } + /* + * i_atime_extra is the last extra field available for +@@ -4535,6 +4463,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + } + sb->s_time_min = EXT4_TIMESTAMP_MIN; + } ++ + if (sbi->s_inode_size > EXT4_GOOD_OLD_INODE_SIZE) { + sbi->s_want_extra_isize = sizeof(struct ext4_inode) - + EXT4_GOOD_OLD_INODE_SIZE; +@@ -4546,7 +4475,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + if (v > max) { + ext4_msg(sb, KERN_ERR, + "bad s_want_extra_isize: %d", v); +- goto failed_mount; ++ return -EINVAL; + } + if (sbi->s_want_extra_isize < v) + sbi->s_want_extra_isize = v; +@@ -4555,91 +4484,112 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + if (v > max) { + ext4_msg(sb, KERN_ERR, + "bad s_min_extra_isize: %d", v); +- goto failed_mount; ++ return -EINVAL; + } + if (sbi->s_want_extra_isize < v) + sbi->s_want_extra_isize = v; + } + } + +- err = parse_apply_sb_mount_options(sb, ctx); +- if (err < 0) +- goto failed_mount; +- +- sbi->s_def_mount_opt = sbi->s_mount_opt; +- +- err = ext4_check_opt_consistency(fc, sb); +- if (err < 0) +- goto failed_mount; +- +- ext4_apply_options(fc, sb); ++ return 0; ++} + + #if IS_ENABLED(CONFIG_UNICODE) +- if (ext4_has_feature_casefold(sb) && !sb->s_encoding) { +- const struct ext4_sb_encodings *encoding_info; +- struct unicode_map *encoding; +- __u16 encoding_flags = le16_to_cpu(es->s_encoding_flags); ++static int ext4_encoding_init(struct super_block *sb, struct ext4_super_block *es) ++{ ++ const struct ext4_sb_encodings *encoding_info; ++ struct unicode_map *encoding; ++ __u16 encoding_flags = le16_to_cpu(es->s_encoding_flags); + +- encoding_info = ext4_sb_read_encoding(es); +- if (!encoding_info) { +- ext4_msg(sb, KERN_ERR, +- "Encoding requested by superblock is unknown"); +- goto failed_mount; +- } ++ if (!ext4_has_feature_casefold(sb) || sb->s_encoding) ++ return 0; + +- encoding = utf8_load(encoding_info->version); +- if (IS_ERR(encoding)) { +- ext4_msg(sb, KERN_ERR, +- "can't mount with superblock charset: %s-%u.%u.%u " +- "not supported by the kernel. flags: 0x%x.", +- encoding_info->name, +- unicode_major(encoding_info->version), +- unicode_minor(encoding_info->version), +- unicode_rev(encoding_info->version), +- encoding_flags); +- goto failed_mount; +- } +- ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: " +- "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, +- unicode_major(encoding_info->version), +- unicode_minor(encoding_info->version), +- unicode_rev(encoding_info->version), +- encoding_flags); ++ encoding_info = ext4_sb_read_encoding(es); ++ if (!encoding_info) { ++ ext4_msg(sb, KERN_ERR, ++ "Encoding requested by superblock is unknown"); ++ return -EINVAL; ++ } + +- sb->s_encoding = encoding; +- sb->s_encoding_flags = encoding_flags; ++ encoding = utf8_load(encoding_info->version); ++ if (IS_ERR(encoding)) { ++ ext4_msg(sb, KERN_ERR, ++ "can't mount with superblock charset: %s-%u.%u.%u " ++ "not supported by the kernel. flags: 0x%x.", ++ encoding_info->name, ++ unicode_major(encoding_info->version), ++ unicode_minor(encoding_info->version), ++ unicode_rev(encoding_info->version), ++ encoding_flags); ++ return -EINVAL; + } ++ ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: " ++ "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, ++ unicode_major(encoding_info->version), ++ unicode_minor(encoding_info->version), ++ unicode_rev(encoding_info->version), ++ encoding_flags); ++ ++ sb->s_encoding = encoding; ++ sb->s_encoding_flags = encoding_flags; ++ ++ return 0; ++} ++#else ++static inline int ext4_encoding_init(struct super_block *sb, struct ext4_super_block *es) ++{ ++ return 0; ++} + #endif + +- if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) { +- printk_once(KERN_WARNING "EXT4-fs: Warning: mounting with data=journal disables delayed allocation, dioread_nolock, O_DIRECT and fast_commit support!\n"); +- /* can't mount with both data=journal and dioread_nolock. */ +- clear_opt(sb, DIOREAD_NOLOCK); +- clear_opt2(sb, JOURNAL_FAST_COMMIT); +- if (test_opt2(sb, EXPLICIT_DELALLOC)) { +- ext4_msg(sb, KERN_ERR, "can't mount with " +- "both data=journal and delalloc"); +- goto failed_mount; +- } +- if (test_opt(sb, DAX_ALWAYS)) { +- ext4_msg(sb, KERN_ERR, "can't mount with " +- "both data=journal and dax"); +- goto failed_mount; +- } +- if (ext4_has_feature_encrypt(sb)) { +- ext4_msg(sb, KERN_WARNING, +- "encrypted files will use data=ordered " +- "instead of data journaling mode"); +- } +- if (test_opt(sb, DELALLOC)) +- clear_opt(sb, DELALLOC); +- } else { +- sb->s_iflags |= SB_I_CGROUPWB; ++static int ext4_init_metadata_csum(struct super_block *sb, struct ext4_super_block *es) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ ++ /* Warn if metadata_csum and gdt_csum are both set. */ ++ if (ext4_has_feature_metadata_csum(sb) && ++ ext4_has_feature_gdt_csum(sb)) ++ ext4_warning(sb, "metadata_csum and uninit_bg are " ++ "redundant flags; please run fsck."); ++ ++ /* Check for a known checksum algorithm */ ++ if (!ext4_verify_csum_type(sb, es)) { ++ ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " ++ "unknown checksum algorithm."); ++ return -EINVAL; + } ++ ext4_setup_csum_trigger(sb, EXT4_JTR_ORPHAN_FILE, ++ ext4_orphan_file_block_trigger); + +- sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | +- (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); ++ /* Load the checksum driver */ ++ sbi->s_chksum_driver = crypto_alloc_shash("crc32c", 0, 0); ++ if (IS_ERR(sbi->s_chksum_driver)) { ++ int ret = PTR_ERR(sbi->s_chksum_driver); ++ ext4_msg(sb, KERN_ERR, "Cannot load crc32c driver."); ++ sbi->s_chksum_driver = NULL; ++ return ret; ++ } ++ ++ /* Check superblock checksum */ ++ if (!ext4_superblock_csum_verify(sb, es)) { ++ ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " ++ "invalid superblock checksum. Run e2fsck?"); ++ return -EFSBADCRC; ++ } ++ ++ /* Precompute checksum seed for all metadata */ ++ if (ext4_has_feature_csum_seed(sb)) ++ sbi->s_csum_seed = le32_to_cpu(es->s_checksum_seed); ++ else if (ext4_has_metadata_csum(sb) || ext4_has_feature_ea_inode(sb)) ++ sbi->s_csum_seed = ext4_chksum(sbi, ~0, es->s_uuid, ++ sizeof(es->s_uuid)); ++ return 0; ++} + ++static int ext4_check_feature_compatibility(struct super_block *sb, ++ struct ext4_super_block *es, ++ int silent) ++{ + if (le32_to_cpu(es->s_rev_level) == EXT4_GOOD_OLD_REV && + (ext4_has_compat_features(sb) || + ext4_has_ro_compat_features(sb) || +@@ -4653,7 +4603,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + if (ext4_has_feature_64bit(sb)) { + ext4_msg(sb, KERN_ERR, + "The Hurd can't support 64-bit file systems"); +- goto failed_mount; ++ return -EINVAL; + } + + /* +@@ -4663,7 +4613,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + if (ext4_has_feature_ea_inode(sb)) { + ext4_msg(sb, KERN_ERR, + "ea_inode feature is not supported for Hurd"); +- goto failed_mount; ++ return -EINVAL; + } + } + +@@ -4677,10 +4627,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + * it's actually an ext[34] filesystem. + */ + if (silent && ext4_feature_set_ok(sb, sb_rdonly(sb))) +- goto failed_mount; ++ return -EINVAL; + ext4_msg(sb, KERN_ERR, "couldn't mount as ext2 due " + "to feature incompatibilities"); +- goto failed_mount; ++ return -EINVAL; + } + } + +@@ -4694,10 +4644,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + * it's actually an ext4 filesystem. + */ + if (silent && ext4_feature_set_ok(sb, sb_rdonly(sb))) +- goto failed_mount; ++ return -EINVAL; + ext4_msg(sb, KERN_ERR, "couldn't mount as ext3 due " + "to feature incompatibilities"); +- goto failed_mount; ++ return -EINVAL; + } + } + +@@ -4707,75 +4657,495 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + * so there is a chance incompat flags are set on a rev 0 filesystem. + */ + if (!ext4_feature_set_ok(sb, (sb_rdonly(sb)))) +- goto failed_mount; ++ return -EINVAL; + +- if (le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) > (blocksize / 4)) { +- ext4_msg(sb, KERN_ERR, +- "Number of reserved GDT blocks insanely large: %d", +- le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks)); +- goto failed_mount; ++ return 0; ++} ++ ++static int ext4_geometry_check(struct super_block *sb, ++ struct ext4_super_block *es) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ __u64 blocks_count; ++ ++ /* check blocks count against device size */ ++ blocks_count = sb_bdev_nr_blocks(sb); ++ if (blocks_count && ext4_blocks_count(es) > blocks_count) { ++ ext4_msg(sb, KERN_WARNING, "bad geometry: block count %llu " ++ "exceeds size of device (%llu blocks)", ++ ext4_blocks_count(es), blocks_count); ++ return -EINVAL; + } + +- if (sbi->s_daxdev) { +- if (blocksize == PAGE_SIZE) +- set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); +- else +- ext4_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); ++ /* ++ * It makes no sense for the first data block to be beyond the end ++ * of the filesystem. ++ */ ++ if (le32_to_cpu(es->s_first_data_block) >= ext4_blocks_count(es)) { ++ ext4_msg(sb, KERN_WARNING, "bad geometry: first data " ++ "block %u is beyond end of filesystem (%llu)", ++ le32_to_cpu(es->s_first_data_block), ++ ext4_blocks_count(es)); ++ return -EINVAL; ++ } ++ if ((es->s_first_data_block == 0) && (es->s_log_block_size == 0) && ++ (sbi->s_cluster_ratio == 1)) { ++ ext4_msg(sb, KERN_WARNING, "bad geometry: first data " ++ "block is 0 with a 1k block and cluster size"); ++ return -EINVAL; + } + +- if (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) { +- if (ext4_has_feature_inline_data(sb)) { +- ext4_msg(sb, KERN_ERR, "Cannot use DAX on a filesystem" +- " that may contain inline data"); +- goto failed_mount; ++ blocks_count = (ext4_blocks_count(es) - ++ le32_to_cpu(es->s_first_data_block) + ++ EXT4_BLOCKS_PER_GROUP(sb) - 1); ++ do_div(blocks_count, EXT4_BLOCKS_PER_GROUP(sb)); ++ if (blocks_count > ((uint64_t)1<<32) - EXT4_DESC_PER_BLOCK(sb)) { ++ ext4_msg(sb, KERN_WARNING, "groups count too large: %llu " ++ "(block count %llu, first data block %u, " ++ "blocks per group %lu)", blocks_count, ++ ext4_blocks_count(es), ++ le32_to_cpu(es->s_first_data_block), ++ EXT4_BLOCKS_PER_GROUP(sb)); ++ return -EINVAL; ++ } ++ sbi->s_groups_count = blocks_count; ++ sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count, ++ (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); ++ if (((u64)sbi->s_groups_count * sbi->s_inodes_per_group) != ++ le32_to_cpu(es->s_inodes_count)) { ++ ext4_msg(sb, KERN_ERR, "inodes count not valid: %u vs %llu", ++ le32_to_cpu(es->s_inodes_count), ++ ((u64)sbi->s_groups_count * sbi->s_inodes_per_group)); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static void ext4_group_desc_free(struct ext4_sb_info *sbi) ++{ ++ struct buffer_head **group_desc; ++ int i; ++ ++ rcu_read_lock(); ++ group_desc = rcu_dereference(sbi->s_group_desc); ++ for (i = 0; i < sbi->s_gdb_count; i++) ++ brelse(group_desc[i]); ++ kvfree(group_desc); ++ rcu_read_unlock(); ++} ++ ++static int ext4_group_desc_init(struct super_block *sb, ++ struct ext4_super_block *es, ++ ext4_fsblk_t logical_sb_block, ++ ext4_group_t *first_not_zeroed) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ unsigned int db_count; ++ ext4_fsblk_t block; ++ int ret; ++ int i; ++ ++ db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / ++ EXT4_DESC_PER_BLOCK(sb); ++ if (ext4_has_feature_meta_bg(sb)) { ++ if (le32_to_cpu(es->s_first_meta_bg) > db_count) { ++ ext4_msg(sb, KERN_WARNING, ++ "first meta block group too large: %u " ++ "(group descriptor block count %u)", ++ le32_to_cpu(es->s_first_meta_bg), db_count); ++ return -EINVAL; + } +- if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) { ++ } ++ rcu_assign_pointer(sbi->s_group_desc, ++ kvmalloc_array(db_count, ++ sizeof(struct buffer_head *), ++ GFP_KERNEL)); ++ if (sbi->s_group_desc == NULL) { ++ ext4_msg(sb, KERN_ERR, "not enough memory"); ++ return -ENOMEM; ++ } ++ ++ bgl_lock_init(sbi->s_blockgroup_lock); ++ ++ /* Pre-read the descriptors into the buffer cache */ ++ for (i = 0; i < db_count; i++) { ++ block = descriptor_loc(sb, logical_sb_block, i); ++ ext4_sb_breadahead_unmovable(sb, block); ++ } ++ ++ for (i = 0; i < db_count; i++) { ++ struct buffer_head *bh; ++ ++ block = descriptor_loc(sb, logical_sb_block, i); ++ bh = ext4_sb_bread_unmovable(sb, block); ++ if (IS_ERR(bh)) { + ext4_msg(sb, KERN_ERR, +- "DAX unsupported by block device."); +- goto failed_mount; ++ "can't read group descriptor %d", i); ++ sbi->s_gdb_count = i; ++ ret = PTR_ERR(bh); ++ goto out; + } ++ rcu_read_lock(); ++ rcu_dereference(sbi->s_group_desc)[i] = bh; ++ rcu_read_unlock(); ++ } ++ sbi->s_gdb_count = db_count; ++ if (!ext4_check_descriptors(sb, logical_sb_block, first_not_zeroed)) { ++ ext4_msg(sb, KERN_ERR, "group descriptors corrupted!"); ++ ret = -EFSCORRUPTED; ++ goto out; + } ++ return 0; ++out: ++ ext4_group_desc_free(sbi); ++ return ret; ++} + +- if (ext4_has_feature_encrypt(sb) && es->s_encryption_level) { +- ext4_msg(sb, KERN_ERR, "Unsupported encryption level %d", +- es->s_encryption_level); +- goto failed_mount; ++static int ext4_load_and_init_journal(struct super_block *sb, ++ struct ext4_super_block *es, ++ struct ext4_fs_context *ctx) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ int err; ++ ++ err = ext4_load_journal(sb, es, ctx->journal_devnum); ++ if (err) ++ return err; ++ ++ if (ext4_has_feature_64bit(sb) && ++ !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, ++ JBD2_FEATURE_INCOMPAT_64BIT)) { ++ ext4_msg(sb, KERN_ERR, "Failed to set 64-bit journal feature"); ++ goto out; + } + +- if (sb->s_blocksize != blocksize) { +- /* +- * bh must be released before kill_bdev(), otherwise +- * it won't be freed and its page also. kill_bdev() +- * is called by sb_set_blocksize(). ++ if (!set_journal_csum_feature_set(sb)) { ++ ext4_msg(sb, KERN_ERR, "Failed to set journal checksum " ++ "feature set"); ++ goto out; ++ } ++ ++ if (test_opt2(sb, JOURNAL_FAST_COMMIT) && ++ !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, ++ JBD2_FEATURE_INCOMPAT_FAST_COMMIT)) { ++ ext4_msg(sb, KERN_ERR, ++ "Failed to set fast commit journal feature"); ++ goto out; ++ } ++ ++ /* We have now updated the journal if required, so we can ++ * validate the data journaling mode. */ ++ switch (test_opt(sb, DATA_FLAGS)) { ++ case 0: ++ /* No mode set, assume a default based on the journal ++ * capabilities: ORDERED_DATA if the journal can ++ * cope, else JOURNAL_DATA + */ +- brelse(bh); +- /* Validate the filesystem blocksize */ +- if (!sb_set_blocksize(sb, blocksize)) { +- ext4_msg(sb, KERN_ERR, "bad block size %d", +- blocksize); +- bh = NULL; +- goto failed_mount; ++ if (jbd2_journal_check_available_features ++ (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { ++ set_opt(sb, ORDERED_DATA); ++ sbi->s_def_mount_opt |= EXT4_MOUNT_ORDERED_DATA; ++ } else { ++ set_opt(sb, JOURNAL_DATA); ++ sbi->s_def_mount_opt |= EXT4_MOUNT_JOURNAL_DATA; ++ } ++ break; ++ ++ case EXT4_MOUNT_ORDERED_DATA: ++ case EXT4_MOUNT_WRITEBACK_DATA: ++ if (!jbd2_journal_check_available_features ++ (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { ++ ext4_msg(sb, KERN_ERR, "Journal does not support " ++ "requested data journaling mode"); ++ goto out; + } ++ break; ++ default: ++ break; ++ } ++ ++ if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && ++ test_opt(sb, JOURNAL_ASYNC_COMMIT)) { ++ ext4_msg(sb, KERN_ERR, "can't mount with " ++ "journal_async_commit in data=ordered mode"); ++ goto out; ++ } + ++ set_task_ioprio(sbi->s_journal->j_task, ctx->journal_ioprio); ++ ++ sbi->s_journal->j_submit_inode_data_buffers = ++ ext4_journal_submit_inode_data_buffers; ++ sbi->s_journal->j_finish_inode_data_buffers = ++ ext4_journal_finish_inode_data_buffers; ++ ++ return 0; ++ ++out: ++ /* flush s_error_work before journal destroy. */ ++ flush_work(&sbi->s_error_work); ++ jbd2_journal_destroy(sbi->s_journal); ++ sbi->s_journal = NULL; ++ return err; ++} ++ ++static int ext4_journal_data_mode_check(struct super_block *sb) ++{ ++ if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) { ++ printk_once(KERN_WARNING "EXT4-fs: Warning: mounting with " ++ "data=journal disables delayed allocation, " ++ "dioread_nolock, O_DIRECT and fast_commit support!\n"); ++ /* can't mount with both data=journal and dioread_nolock. */ ++ clear_opt(sb, DIOREAD_NOLOCK); ++ clear_opt2(sb, JOURNAL_FAST_COMMIT); ++ if (test_opt2(sb, EXPLICIT_DELALLOC)) { ++ ext4_msg(sb, KERN_ERR, "can't mount with " ++ "both data=journal and delalloc"); ++ return -EINVAL; ++ } ++ if (test_opt(sb, DAX_ALWAYS)) { ++ ext4_msg(sb, KERN_ERR, "can't mount with " ++ "both data=journal and dax"); ++ return -EINVAL; ++ } ++ if (ext4_has_feature_encrypt(sb)) { ++ ext4_msg(sb, KERN_WARNING, ++ "encrypted files will use data=ordered " ++ "instead of data journaling mode"); ++ } ++ if (test_opt(sb, DELALLOC)) ++ clear_opt(sb, DELALLOC); ++ } else { ++ sb->s_iflags |= SB_I_CGROUPWB; ++ } ++ ++ return 0; ++} ++ ++static int ext4_load_super(struct super_block *sb, ext4_fsblk_t *lsb, ++ int silent) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ struct ext4_super_block *es; ++ ext4_fsblk_t logical_sb_block; ++ unsigned long offset = 0; ++ struct buffer_head *bh; ++ int ret = -EINVAL; ++ int blocksize; ++ ++ blocksize = sb_min_blocksize(sb, EXT4_MIN_BLOCK_SIZE); ++ if (!blocksize) { ++ ext4_msg(sb, KERN_ERR, "unable to set blocksize"); ++ return -EINVAL; ++ } ++ ++ /* ++ * The ext4 superblock will not be buffer aligned for other than 1kB ++ * block sizes. We need to calculate the offset from buffer start. ++ */ ++ if (blocksize != EXT4_MIN_BLOCK_SIZE) { + logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; + offset = do_div(logical_sb_block, blocksize); +- bh = ext4_sb_bread_unmovable(sb, logical_sb_block); +- if (IS_ERR(bh)) { +- ext4_msg(sb, KERN_ERR, +- "Can't read superblock on 2nd try"); +- ret = PTR_ERR(bh); +- bh = NULL; ++ } else { ++ logical_sb_block = sbi->s_sb_block; ++ } ++ ++ bh = ext4_sb_bread_unmovable(sb, logical_sb_block); ++ if (IS_ERR(bh)) { ++ ext4_msg(sb, KERN_ERR, "unable to read superblock"); ++ return PTR_ERR(bh); ++ } ++ /* ++ * Note: s_es must be initialized as soon as possible because ++ * some ext4 macro-instructions depend on its value ++ */ ++ es = (struct ext4_super_block *) (bh->b_data + offset); ++ sbi->s_es = es; ++ sb->s_magic = le16_to_cpu(es->s_magic); ++ if (sb->s_magic != EXT4_SUPER_MAGIC) { ++ if (!silent) ++ ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); ++ goto out; ++ } ++ ++ if (le32_to_cpu(es->s_log_block_size) > ++ (EXT4_MAX_BLOCK_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { ++ ext4_msg(sb, KERN_ERR, ++ "Invalid log block size: %u", ++ le32_to_cpu(es->s_log_block_size)); ++ goto out; ++ } ++ if (le32_to_cpu(es->s_log_cluster_size) > ++ (EXT4_MAX_CLUSTER_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { ++ ext4_msg(sb, KERN_ERR, ++ "Invalid log cluster size: %u", ++ le32_to_cpu(es->s_log_cluster_size)); ++ goto out; ++ } ++ ++ blocksize = EXT4_MIN_BLOCK_SIZE << le32_to_cpu(es->s_log_block_size); ++ ++ /* ++ * If the default block size is not the same as the real block size, ++ * we need to reload it. ++ */ ++ if (sb->s_blocksize == blocksize) { ++ *lsb = logical_sb_block; ++ sbi->s_sbh = bh; ++ return 0; ++ } ++ ++ /* ++ * bh must be released before kill_bdev(), otherwise ++ * it won't be freed and its page also. kill_bdev() ++ * is called by sb_set_blocksize(). ++ */ ++ brelse(bh); ++ /* Validate the filesystem blocksize */ ++ if (!sb_set_blocksize(sb, blocksize)) { ++ ext4_msg(sb, KERN_ERR, "bad block size %d", ++ blocksize); ++ bh = NULL; ++ goto out; ++ } ++ ++ logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; ++ offset = do_div(logical_sb_block, blocksize); ++ bh = ext4_sb_bread_unmovable(sb, logical_sb_block); ++ if (IS_ERR(bh)) { ++ ext4_msg(sb, KERN_ERR, "Can't read superblock on 2nd try"); ++ ret = PTR_ERR(bh); ++ bh = NULL; ++ goto out; ++ } ++ es = (struct ext4_super_block *)(bh->b_data + offset); ++ sbi->s_es = es; ++ if (es->s_magic != cpu_to_le16(EXT4_SUPER_MAGIC)) { ++ ext4_msg(sb, KERN_ERR, "Magic mismatch, very weird!"); ++ goto out; ++ } ++ *lsb = logical_sb_block; ++ sbi->s_sbh = bh; ++ return 0; ++out: ++ brelse(bh); ++ return ret; ++} ++ ++static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ++{ ++ struct ext4_super_block *es = NULL; ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ struct flex_groups **flex_groups; ++ ext4_fsblk_t block; ++ ext4_fsblk_t logical_sb_block; ++ struct inode *root; ++ int ret = -ENOMEM; ++ unsigned int i; ++ int needs_recovery, has_huge_files; ++ int err = 0; ++ ext4_group_t first_not_zeroed; ++ struct ext4_fs_context *ctx = fc->fs_private; ++ int silent = fc->sb_flags & SB_SILENT; ++ ++ /* Set defaults for the variables that will be set during parsing */ ++ if (!(ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO)) ++ ctx->journal_ioprio = DEFAULT_JOURNAL_IOPRIO; ++ ++ sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; ++ sbi->s_sectors_written_start = ++ part_stat_read(sb->s_bdev, sectors[STAT_WRITE]); ++ ++ /* -EINVAL is default */ ++ ret = -EINVAL; ++ err = ext4_load_super(sb, &logical_sb_block, silent); ++ if (err) ++ goto out_fail; ++ ++ es = sbi->s_es; ++ sbi->s_kbytes_written = le64_to_cpu(es->s_kbytes_written); ++ ++ err = ext4_init_metadata_csum(sb, es); ++ if (err) ++ goto failed_mount; ++ ++ ext4_set_def_opts(sb, es); ++ ++ sbi->s_resuid = make_kuid(&init_user_ns, le16_to_cpu(es->s_def_resuid)); ++ sbi->s_resgid = make_kgid(&init_user_ns, le16_to_cpu(es->s_def_resgid)); ++ sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; ++ sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; ++ sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; ++ ++ /* ++ * set default s_li_wait_mult for lazyinit, for the case there is ++ * no mount option specified. ++ */ ++ sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; ++ ++ if (ext4_inode_info_init(sb, es)) ++ goto failed_mount; ++ ++ err = parse_apply_sb_mount_options(sb, ctx); ++ if (err < 0) ++ goto failed_mount; ++ ++ sbi->s_def_mount_opt = sbi->s_mount_opt; ++ ++ err = ext4_check_opt_consistency(fc, sb); ++ if (err < 0) ++ goto failed_mount; ++ ++ ext4_apply_options(fc, sb); ++ ++ if (ext4_encoding_init(sb, es)) ++ goto failed_mount; ++ ++ if (ext4_journal_data_mode_check(sb)) ++ goto failed_mount; ++ ++ sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | ++ (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); ++ ++ /* i_version is always enabled now */ ++ sb->s_flags |= SB_I_VERSION; ++ ++ if (ext4_check_feature_compatibility(sb, es, silent)) ++ goto failed_mount; ++ ++ if (le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) > (sb->s_blocksize / 4)) { ++ ext4_msg(sb, KERN_ERR, ++ "Number of reserved GDT blocks insanely large: %d", ++ le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks)); ++ goto failed_mount; ++ } ++ ++ if (sbi->s_daxdev) { ++ if (sb->s_blocksize == PAGE_SIZE) ++ set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); ++ else ++ ext4_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); ++ } ++ ++ if (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) { ++ if (ext4_has_feature_inline_data(sb)) { ++ ext4_msg(sb, KERN_ERR, "Cannot use DAX on a filesystem" ++ " that may contain inline data"); + goto failed_mount; + } +- es = (struct ext4_super_block *)(bh->b_data + offset); +- sbi->s_es = es; +- if (es->s_magic != cpu_to_le16(EXT4_SUPER_MAGIC)) { ++ if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) { + ext4_msg(sb, KERN_ERR, +- "Magic mismatch, very weird!"); ++ "DAX unsupported by block device."); + goto failed_mount; + } + } + ++ if (ext4_has_feature_encrypt(sb) && es->s_encryption_level) { ++ ext4_msg(sb, KERN_ERR, "Unsupported encryption level %d", ++ es->s_encryption_level); ++ goto failed_mount; ++ } ++ + has_huge_files = ext4_has_feature_huge_file(sb); + sbi->s_bitmap_maxbytes = ext4_max_bitmap_size(sb->s_blocksize_bits, + has_huge_files); +@@ -4797,19 +5167,21 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group); + sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group); + +- sbi->s_inodes_per_block = blocksize / EXT4_INODE_SIZE(sb); +- if (sbi->s_inodes_per_block == 0) +- goto cantfind_ext4; ++ sbi->s_inodes_per_block = sb->s_blocksize / EXT4_INODE_SIZE(sb); ++ if (sbi->s_inodes_per_block == 0 || sbi->s_blocks_per_group == 0) { ++ if (!silent) ++ ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); ++ goto failed_mount; ++ } + if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || +- sbi->s_inodes_per_group > blocksize * 8) { ++ sbi->s_inodes_per_group > sb->s_blocksize * 8) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", + sbi->s_inodes_per_group); + goto failed_mount; + } + sbi->s_itb_per_group = sbi->s_inodes_per_group / + sbi->s_inodes_per_block; +- sbi->s_desc_per_block = blocksize / EXT4_DESC_SIZE(sb); +- sbi->s_sbh = bh; ++ sbi->s_desc_per_block = sb->s_blocksize / EXT4_DESC_SIZE(sb); + sbi->s_mount_state = le16_to_cpu(es->s_state) & ~EXT4_FC_REPLAY; + sbi->s_addr_per_block_bits = ilog2(EXT4_ADDR_PER_BLOCK(sb)); + sbi->s_desc_per_block_bits = ilog2(EXT4_DESC_PER_BLOCK(sb)); +@@ -4835,54 +5207,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + } + } + +- /* Handle clustersize */ +- clustersize = BLOCK_SIZE << le32_to_cpu(es->s_log_cluster_size); +- if (ext4_has_feature_bigalloc(sb)) { +- if (clustersize < blocksize) { +- ext4_msg(sb, KERN_ERR, +- "cluster size (%d) smaller than " +- "block size (%d)", clustersize, blocksize); +- goto failed_mount; +- } +- sbi->s_cluster_bits = le32_to_cpu(es->s_log_cluster_size) - +- le32_to_cpu(es->s_log_block_size); +- sbi->s_clusters_per_group = +- le32_to_cpu(es->s_clusters_per_group); +- if (sbi->s_clusters_per_group > blocksize * 8) { +- ext4_msg(sb, KERN_ERR, +- "#clusters per group too big: %lu", +- sbi->s_clusters_per_group); +- goto failed_mount; +- } +- if (sbi->s_blocks_per_group != +- (sbi->s_clusters_per_group * (clustersize / blocksize))) { +- ext4_msg(sb, KERN_ERR, "blocks per group (%lu) and " +- "clusters per group (%lu) inconsistent", +- sbi->s_blocks_per_group, +- sbi->s_clusters_per_group); +- goto failed_mount; +- } +- } else { +- if (clustersize != blocksize) { +- ext4_msg(sb, KERN_ERR, +- "fragment/cluster size (%d) != " +- "block size (%d)", clustersize, blocksize); +- goto failed_mount; +- } +- if (sbi->s_blocks_per_group > blocksize * 8) { +- ext4_msg(sb, KERN_ERR, +- "#blocks per group too big: %lu", +- sbi->s_blocks_per_group); +- goto failed_mount; +- } +- sbi->s_clusters_per_group = sbi->s_blocks_per_group; +- sbi->s_cluster_bits = 0; +- } +- sbi->s_cluster_ratio = clustersize / blocksize; +- +- /* Do we have standard group size of clustersize * 8 blocks ? */ +- if (sbi->s_blocks_per_group == clustersize << 3) +- set_opt2(sb, STD_GROUP_SIZE); ++ if (ext4_handle_clustersize(sb)) ++ goto failed_mount; + + /* + * Test whether we have more sectors than will fit in sector_t, +@@ -4896,111 +5222,12 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + goto failed_mount; + } + +- if (EXT4_BLOCKS_PER_GROUP(sb) == 0) +- goto cantfind_ext4; +- +- /* check blocks count against device size */ +- blocks_count = sb_bdev_nr_blocks(sb); +- if (blocks_count && ext4_blocks_count(es) > blocks_count) { +- ext4_msg(sb, KERN_WARNING, "bad geometry: block count %llu " +- "exceeds size of device (%llu blocks)", +- ext4_blocks_count(es), blocks_count); +- goto failed_mount; +- } +- +- /* +- * It makes no sense for the first data block to be beyond the end +- * of the filesystem. +- */ +- if (le32_to_cpu(es->s_first_data_block) >= ext4_blocks_count(es)) { +- ext4_msg(sb, KERN_WARNING, "bad geometry: first data " +- "block %u is beyond end of filesystem (%llu)", +- le32_to_cpu(es->s_first_data_block), +- ext4_blocks_count(es)); +- goto failed_mount; +- } +- if ((es->s_first_data_block == 0) && (es->s_log_block_size == 0) && +- (sbi->s_cluster_ratio == 1)) { +- ext4_msg(sb, KERN_WARNING, "bad geometry: first data " +- "block is 0 with a 1k block and cluster size"); ++ if (ext4_geometry_check(sb, es)) + goto failed_mount; +- } + +- blocks_count = (ext4_blocks_count(es) - +- le32_to_cpu(es->s_first_data_block) + +- EXT4_BLOCKS_PER_GROUP(sb) - 1); +- do_div(blocks_count, EXT4_BLOCKS_PER_GROUP(sb)); +- if (blocks_count > ((uint64_t)1<<32) - EXT4_DESC_PER_BLOCK(sb)) { +- ext4_msg(sb, KERN_WARNING, "groups count too large: %llu " +- "(block count %llu, first data block %u, " +- "blocks per group %lu)", blocks_count, +- ext4_blocks_count(es), +- le32_to_cpu(es->s_first_data_block), +- EXT4_BLOCKS_PER_GROUP(sb)); +- goto failed_mount; +- } +- sbi->s_groups_count = blocks_count; +- sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count, +- (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); +- if (((u64)sbi->s_groups_count * sbi->s_inodes_per_group) != +- le32_to_cpu(es->s_inodes_count)) { +- ext4_msg(sb, KERN_ERR, "inodes count not valid: %u vs %llu", +- le32_to_cpu(es->s_inodes_count), +- ((u64)sbi->s_groups_count * sbi->s_inodes_per_group)); +- ret = -EINVAL; +- goto failed_mount; +- } +- db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / +- EXT4_DESC_PER_BLOCK(sb); +- if (ext4_has_feature_meta_bg(sb)) { +- if (le32_to_cpu(es->s_first_meta_bg) > db_count) { +- ext4_msg(sb, KERN_WARNING, +- "first meta block group too large: %u " +- "(group descriptor block count %u)", +- le32_to_cpu(es->s_first_meta_bg), db_count); +- goto failed_mount; +- } +- } +- rcu_assign_pointer(sbi->s_group_desc, +- kvmalloc_array(db_count, +- sizeof(struct buffer_head *), +- GFP_KERNEL)); +- if (sbi->s_group_desc == NULL) { +- ext4_msg(sb, KERN_ERR, "not enough memory"); +- ret = -ENOMEM; ++ err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); ++ if (err) + goto failed_mount; +- } +- +- bgl_lock_init(sbi->s_blockgroup_lock); +- +- /* Pre-read the descriptors into the buffer cache */ +- for (i = 0; i < db_count; i++) { +- block = descriptor_loc(sb, logical_sb_block, i); +- ext4_sb_breadahead_unmovable(sb, block); +- } +- +- for (i = 0; i < db_count; i++) { +- struct buffer_head *bh; +- +- block = descriptor_loc(sb, logical_sb_block, i); +- bh = ext4_sb_bread_unmovable(sb, block); +- if (IS_ERR(bh)) { +- ext4_msg(sb, KERN_ERR, +- "can't read group descriptor %d", i); +- db_count = i; +- ret = PTR_ERR(bh); +- goto failed_mount2; +- } +- rcu_read_lock(); +- rcu_dereference(sbi->s_group_desc)[i] = bh; +- rcu_read_unlock(); +- } +- sbi->s_gdb_count = db_count; +- if (!ext4_check_descriptors(sb, logical_sb_block, &first_not_zeroed)) { +- ext4_msg(sb, KERN_ERR, "group descriptors corrupted!"); +- ret = -EFSCORRUPTED; +- goto failed_mount2; +- } + + timer_setup(&sbi->s_err_report, print_daily_error_info, 0); + spin_lock_init(&sbi->s_error_lock); +@@ -5038,24 +5265,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + INIT_LIST_HEAD(&sbi->s_orphan); /* unlinked but open files */ + mutex_init(&sbi->s_orphan_lock); + +- /* Initialize fast commit stuff */ +- atomic_set(&sbi->s_fc_subtid, 0); +- INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_MAIN]); +- INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_STAGING]); +- INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_MAIN]); +- INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_STAGING]); +- sbi->s_fc_bytes = 0; +- ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); +- sbi->s_fc_ineligible_tid = 0; +- spin_lock_init(&sbi->s_fc_lock); +- memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); +- sbi->s_fc_replay_state.fc_regions = NULL; +- sbi->s_fc_replay_state.fc_regions_size = 0; +- sbi->s_fc_replay_state.fc_regions_used = 0; +- sbi->s_fc_replay_state.fc_regions_valid = 0; +- sbi->s_fc_replay_state.fc_modified_inodes = NULL; +- sbi->s_fc_replay_state.fc_modified_inodes_size = 0; +- sbi->s_fc_replay_state.fc_modified_inodes_used = 0; ++ ext4_fast_commit_init(sb); + + sb->s_root = NULL; + +@@ -5072,37 +5282,37 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + * root first: it may be modified in the journal! + */ + if (!test_opt(sb, NOLOAD) && ext4_has_feature_journal(sb)) { +- err = ext4_load_journal(sb, es, ctx->journal_devnum); ++ err = ext4_load_and_init_journal(sb, es, ctx); + if (err) + goto failed_mount3a; + } else if (test_opt(sb, NOLOAD) && !sb_rdonly(sb) && + ext4_has_feature_journal_needs_recovery(sb)) { + ext4_msg(sb, KERN_ERR, "required journal recovery " + "suppressed and not mounted read-only"); +- goto failed_mount_wq; ++ goto failed_mount3a; + } else { + /* Nojournal mode, all journal mount options are illegal */ + if (test_opt2(sb, EXPLICIT_JOURNAL_CHECKSUM)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_checksum, fs mounted w/o journal"); +- goto failed_mount_wq; ++ goto failed_mount3a; + } + if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_async_commit, fs mounted w/o journal"); +- goto failed_mount_wq; ++ goto failed_mount3a; + } + if (sbi->s_commit_interval != JBD2_DEFAULT_MAX_COMMIT_AGE*HZ) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "commit=%lu, fs mounted w/o journal", + sbi->s_commit_interval / HZ); +- goto failed_mount_wq; ++ goto failed_mount3a; + } + if (EXT4_MOUNT_DATA_FLAGS & + (sbi->s_mount_opt ^ sbi->s_def_mount_opt)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "data=, fs mounted w/o journal"); +- goto failed_mount_wq; ++ goto failed_mount3a; + } + sbi->s_def_mount_opt &= ~EXT4_MOUNT_JOURNAL_CHECKSUM; + clear_opt(sb, JOURNAL_CHECKSUM); +@@ -5110,76 +5320,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + clear_opt2(sb, JOURNAL_FAST_COMMIT); + sbi->s_journal = NULL; + needs_recovery = 0; +- goto no_journal; +- } +- +- if (ext4_has_feature_64bit(sb) && +- !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, +- JBD2_FEATURE_INCOMPAT_64BIT)) { +- ext4_msg(sb, KERN_ERR, "Failed to set 64-bit journal feature"); +- goto failed_mount_wq; +- } +- +- if (!set_journal_csum_feature_set(sb)) { +- ext4_msg(sb, KERN_ERR, "Failed to set journal checksum " +- "feature set"); +- goto failed_mount_wq; +- } +- +- if (test_opt2(sb, JOURNAL_FAST_COMMIT) && +- !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, +- JBD2_FEATURE_INCOMPAT_FAST_COMMIT)) { +- ext4_msg(sb, KERN_ERR, +- "Failed to set fast commit journal feature"); +- goto failed_mount_wq; +- } +- +- /* We have now updated the journal if required, so we can +- * validate the data journaling mode. */ +- switch (test_opt(sb, DATA_FLAGS)) { +- case 0: +- /* No mode set, assume a default based on the journal +- * capabilities: ORDERED_DATA if the journal can +- * cope, else JOURNAL_DATA +- */ +- if (jbd2_journal_check_available_features +- (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { +- set_opt(sb, ORDERED_DATA); +- sbi->s_def_mount_opt |= EXT4_MOUNT_ORDERED_DATA; +- } else { +- set_opt(sb, JOURNAL_DATA); +- sbi->s_def_mount_opt |= EXT4_MOUNT_JOURNAL_DATA; +- } +- break; +- +- case EXT4_MOUNT_ORDERED_DATA: +- case EXT4_MOUNT_WRITEBACK_DATA: +- if (!jbd2_journal_check_available_features +- (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { +- ext4_msg(sb, KERN_ERR, "Journal does not support " +- "requested data journaling mode"); +- goto failed_mount_wq; +- } +- break; +- default: +- break; +- } +- +- if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && +- test_opt(sb, JOURNAL_ASYNC_COMMIT)) { +- ext4_msg(sb, KERN_ERR, "can't mount with " +- "journal_async_commit in data=ordered mode"); +- goto failed_mount_wq; + } + +- set_task_ioprio(sbi->s_journal->j_task, ctx->journal_ioprio); +- +- sbi->s_journal->j_submit_inode_data_buffers = +- ext4_journal_submit_inode_data_buffers; +- sbi->s_journal->j_finish_inode_data_buffers = +- ext4_journal_finish_inode_data_buffers; +- +-no_journal: + if (!test_opt(sb, NO_MBCACHE)) { + sbi->s_ea_block_cache = ext4_xattr_create_cache(); + if (!sbi->s_ea_block_cache) { +@@ -5198,7 +5340,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + } + } + +- if (ext4_has_feature_verity(sb) && blocksize != PAGE_SIZE) { ++ if (ext4_has_feature_verity(sb) && sb->s_blocksize != PAGE_SIZE) { + ext4_msg(sb, KERN_ERR, "Unsupported blocksize for fs-verity"); + goto failed_mount_wq; + } +@@ -5408,11 +5550,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + + return 0; + +-cantfind_ext4: +- if (!silent) +- ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); +- goto failed_mount; +- + failed_mount9: + ext4_release_orphan_info(sb); + failed_mount8: +@@ -5466,13 +5603,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + flush_work(&sbi->s_error_work); + del_timer_sync(&sbi->s_err_report); + ext4_stop_mmpd(sbi); +-failed_mount2: +- rcu_read_lock(); +- group_desc = rcu_dereference(sbi->s_group_desc); +- for (i = 0; i < db_count; i++) +- brelse(group_desc[i]); +- kvfree(group_desc); +- rcu_read_unlock(); ++ ext4_group_desc_free(sbi); + failed_mount: + if (sbi->s_chksum_driver) + crypto_free_shash(sbi->s_chksum_driver); +@@ -5487,7 +5618,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) + #endif + fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy); + /* ext4_blkdev_remove() calls kill_bdev(), release bh before it. */ +- brelse(bh); ++ brelse(sbi->s_sbh); + ext4_blkdev_remove(sbi); + out_fail: + sb->s_fs_info = NULL; +@@ -6653,7 +6784,7 @@ static int ext4_write_info(struct super_block *sb, int type) + handle_t *handle; + + /* Data block + inode block */ +- handle = ext4_journal_start(d_inode(sb->s_root), EXT4_HT_QUOTA, 2); ++ handle = ext4_journal_start_sb(sb, EXT4_HT_QUOTA, 2); + if (IS_ERR(handle)) + return PTR_ERR(handle); + ret = dquot_commit_info(sb, type); +diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c +index d233c24ea342..f0d42cf44c71 100644 +--- a/fs/ext4/sysfs.c ++++ b/fs/ext4/sysfs.c +@@ -214,7 +214,6 @@ EXT4_RW_ATTR_SBI_UI(mb_min_to_scan, s_mb_min_to_scan); + EXT4_RW_ATTR_SBI_UI(mb_order2_req, s_mb_order2_reqs); + EXT4_RW_ATTR_SBI_UI(mb_stream_req, s_mb_stream_request); + EXT4_RW_ATTR_SBI_UI(mb_group_prealloc, s_mb_group_prealloc); +-EXT4_RW_ATTR_SBI_UI(mb_max_inode_prealloc, s_mb_max_inode_prealloc); + EXT4_RW_ATTR_SBI_UI(mb_max_linear_groups, s_mb_max_linear_groups); + EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb); + EXT4_ATTR(trigger_fs_error, 0200, trigger_test_error); +@@ -264,7 +263,6 @@ static struct attribute *ext4_attrs[] = { + ATTR_LIST(mb_order2_req), + ATTR_LIST(mb_stream_req), + ATTR_LIST(mb_group_prealloc), +- ATTR_LIST(mb_max_inode_prealloc), + ATTR_LIST(mb_max_linear_groups), + ATTR_LIST(max_writeback_mb_bump), + ATTR_LIST(extent_max_zeroout_kb), +diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c +index b051d19b5c8a..20cadfb740dc 100644 +--- a/fs/ext4/verity.c ++++ b/fs/ext4/verity.c +@@ -298,16 +298,14 @@ static int ext4_get_verity_descriptor_location(struct inode *inode, + last_extent = path[path->p_depth].p_ext; + if (!last_extent) { + EXT4_ERROR_INODE(inode, "verity file has no extents"); +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + return -EFSCORRUPTED; + } + + end_lblk = le32_to_cpu(last_extent->ee_block) + + ext4_ext_get_actual_len(last_extent); + desc_size_pos = (u64)end_lblk << inode->i_blkbits; +- ext4_ext_drop_refs(path); +- kfree(path); ++ ext4_free_ext_path(path); + + if (desc_size_pos < sizeof(desc_size_disk)) + goto bad; +diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c +index 533216e80fa2..36d6ba7190b6 100644 +--- a/fs/ext4/xattr.c ++++ b/fs/ext4/xattr.c +@@ -2412,6 +2412,7 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index, + if (!error) { + ext4_xattr_update_super_block(handle, inode->i_sb); + inode->i_ctime = current_time(inode); ++ inode_inc_iversion(inode); + if (!value) + no_expand = 0; + error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); +diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c +index 08a1993ab7fd..443f83382b9b 100644 +--- a/fs/fs-writeback.c ++++ b/fs/fs-writeback.c +@@ -1718,9 +1718,14 @@ static int writeback_single_inode(struct inode *inode, + */ + if (!(inode->i_state & I_DIRTY_ALL)) + inode_cgwb_move_to_attached(inode, wb); +- else if (!(inode->i_state & I_SYNC_QUEUED) && +- (inode->i_state & I_DIRTY)) +- redirty_tail_locked(inode, wb); ++ else if (!(inode->i_state & I_SYNC_QUEUED)) { ++ if ((inode->i_state & I_DIRTY)) ++ redirty_tail_locked(inode, wb); ++ else if (inode->i_state & I_DIRTY_TIME) { ++ inode->dirtied_when = jiffies; ++ inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); ++ } ++ } + + spin_unlock(&wb->list_lock); + inode_sync_complete(inode); +@@ -2369,6 +2374,20 @@ void __mark_inode_dirty(struct inode *inode, int flags) + trace_writeback_mark_inode_dirty(inode, flags); + + if (flags & I_DIRTY_INODE) { ++ /* ++ * Inode timestamp update will piggback on this dirtying. ++ * We tell ->dirty_inode callback that timestamps need to ++ * be updated by setting I_DIRTY_TIME in flags. ++ */ ++ if (inode->i_state & I_DIRTY_TIME) { ++ spin_lock(&inode->i_lock); ++ if (inode->i_state & I_DIRTY_TIME) { ++ inode->i_state &= ~I_DIRTY_TIME; ++ flags |= I_DIRTY_TIME; ++ } ++ spin_unlock(&inode->i_lock); ++ } ++ + /* + * Notify the filesystem about the inode being dirtied, so that + * (if needed) it can update on-disk fields and journal the +@@ -2378,7 +2397,8 @@ void __mark_inode_dirty(struct inode *inode, int flags) + */ + trace_writeback_dirty_inode_start(inode, flags); + if (sb->s_op->dirty_inode) +- sb->s_op->dirty_inode(inode, flags & I_DIRTY_INODE); ++ sb->s_op->dirty_inode(inode, ++ flags & (I_DIRTY_INODE | I_DIRTY_TIME)); + trace_writeback_dirty_inode(inode, flags); + + /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ +@@ -2399,21 +2419,15 @@ void __mark_inode_dirty(struct inode *inode, int flags) + */ + smp_mb(); + +- if (((inode->i_state & flags) == flags) || +- (dirtytime && (inode->i_state & I_DIRTY_INODE))) ++ if ((inode->i_state & flags) == flags) + return; + + spin_lock(&inode->i_lock); +- if (dirtytime && (inode->i_state & I_DIRTY_INODE)) +- goto out_unlock_inode; + if ((inode->i_state & flags) != flags) { + const int was_dirty = inode->i_state & I_DIRTY; + + inode_attach_wb(inode, NULL); + +- /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ +- if (flags & I_DIRTY_INODE) +- inode->i_state &= ~I_DIRTY_TIME; + inode->i_state |= flags; + + /* +@@ -2486,7 +2500,6 @@ void __mark_inode_dirty(struct inode *inode, int flags) + out_unlock: + if (wb) + spin_unlock(&wb->list_lock); +-out_unlock_inode: + spin_unlock(&inode->i_lock); + } + EXPORT_SYMBOL(__mark_inode_dirty); +diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c +index ca5c62901541..91ee0b308e13 100644 +--- a/fs/iomap/buffered-io.c ++++ b/fs/iomap/buffered-io.c +@@ -1360,6 +1360,7 @@ iomap_writepage_map(struct iomap_writepage_ctx *wpc, + error = wpc->ops->map_blocks(wpc, inode, pos); + if (error) + break; ++ trace_iomap_writepage_map(inode, &wpc->iomap); + if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE)) + continue; + if (wpc->iomap.type == IOMAP_HOLE) +@@ -1421,7 +1422,7 @@ iomap_writepage_map(struct iomap_writepage_ctx *wpc, + if (!count) + folio_end_writeback(folio); + done: +- mapping_set_error(folio->mapping, error); ++ mapping_set_error(inode->i_mapping, error); + return error; + } + +diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h +index d48868fc40d7..f6ea9540d082 100644 +--- a/fs/iomap/trace.h ++++ b/fs/iomap/trace.h +@@ -148,6 +148,7 @@ DEFINE_EVENT(iomap_class, name, \ + TP_ARGS(inode, iomap)) + DEFINE_IOMAP_EVENT(iomap_iter_dstmap); + DEFINE_IOMAP_EVENT(iomap_iter_srcmap); ++DEFINE_IOMAP_EVENT(iomap_writepage_map); + + TRACE_EVENT(iomap_iter, + TP_PROTO(struct iomap_iter *iter, const void *ops, +diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c +index b2b2bc9b88d9..885a7a6cc53e 100644 +--- a/fs/jbd2/commit.c ++++ b/fs/jbd2/commit.c +@@ -122,8 +122,8 @@ static int journal_submit_commit_record(journal_t *journal, + { + struct commit_header *tmp; + struct buffer_head *bh; +- int ret; + struct timespec64 now; ++ blk_opf_t write_flags = REQ_OP_WRITE | REQ_SYNC; + + *cbh = NULL; + +@@ -155,13 +155,11 @@ static int journal_submit_commit_record(journal_t *journal, + + if (journal->j_flags & JBD2_BARRIER && + !jbd2_has_feature_async_commit(journal)) +- ret = submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH | +- REQ_FUA, bh); +- else +- ret = submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); ++ write_flags |= REQ_PREFLUSH | REQ_FUA; + ++ submit_bh(write_flags, bh); + *cbh = bh; +- return ret; ++ return 0; + } + + /* +@@ -570,7 +568,7 @@ void jbd2_journal_commit_transaction(journal_t *journal) + journal->j_running_transaction = NULL; + start_time = ktime_get(); + commit_transaction->t_log_start = journal->j_head; +- wake_up(&journal->j_wait_transaction_locked); ++ wake_up_all(&journal->j_wait_transaction_locked); + write_unlock(&journal->j_state_lock); + + jbd2_debug(3, "JBD2: commit phase 2a\n"); +diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c +index 6350d3857c89..bc8270e0d7d0 100644 +--- a/fs/jbd2/journal.c ++++ b/fs/jbd2/journal.c +@@ -923,10 +923,16 @@ int jbd2_fc_wait_bufs(journal_t *journal, int num_blks) + for (i = j_fc_off - 1; i >= j_fc_off - num_blks; i--) { + bh = journal->j_fc_wbuf[i]; + wait_on_buffer(bh); ++ /* ++ * Update j_fc_off so jbd2_fc_release_bufs can release remain ++ * buffer head. ++ */ ++ if (unlikely(!buffer_uptodate(bh))) { ++ journal->j_fc_off = i + 1; ++ return -EIO; ++ } + put_bh(bh); + journal->j_fc_wbuf[i] = NULL; +- if (unlikely(!buffer_uptodate(bh))) +- return -EIO; + } + + return 0; +@@ -1606,7 +1612,7 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) + { + struct buffer_head *bh = journal->j_sb_buffer; + journal_superblock_t *sb = journal->j_superblock; +- int ret; ++ int ret = 0; + + /* Buffer got discarded which means block device got invalidated */ + if (!buffer_mapped(bh)) { +@@ -1636,7 +1642,7 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) + sb->s_checksum = jbd2_superblock_csum(journal, sb); + get_bh(bh); + bh->b_end_io = end_buffer_write_sync; +- ret = submit_bh(REQ_OP_WRITE | write_flags, bh); ++ submit_bh(REQ_OP_WRITE | write_flags, bh); + wait_on_buffer(bh); + if (buffer_write_io_error(bh)) { + clear_buffer_write_io_error(bh); +@@ -1644,9 +1650,8 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) + ret = -EIO; + } + if (ret) { +- printk(KERN_ERR "JBD2: Error %d detected when updating " +- "journal superblock for %s.\n", ret, +- journal->j_devname); ++ printk(KERN_ERR "JBD2: I/O error when updating journal superblock for %s.\n", ++ journal->j_devname); + if (!is_journal_aborted(journal)) + jbd2_journal_abort(journal, ret); + } +diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c +index f548479615c6..3688d16fe83b 100644 +--- a/fs/jbd2/recovery.c ++++ b/fs/jbd2/recovery.c +@@ -256,6 +256,7 @@ static int fc_do_one_pass(journal_t *journal, + err = journal->j_fc_replay_callback(journal, bh, pass, + next_fc_block - journal->j_fc_first, + expected_commit_id); ++ brelse(bh); + next_fc_block++; + if (err < 0 || err == JBD2_FC_REPLAY_STOP) + break; +diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c +index e1be93ccd81c..6a404ac1c178 100644 +--- a/fs/jbd2/transaction.c ++++ b/fs/jbd2/transaction.c +@@ -168,7 +168,7 @@ static void wait_transaction_locked(journal_t *journal) + int need_to_start; + tid_t tid = journal->j_running_transaction->t_tid; + +- prepare_to_wait(&journal->j_wait_transaction_locked, &wait, ++ prepare_to_wait_exclusive(&journal->j_wait_transaction_locked, &wait, + TASK_UNINTERRUPTIBLE); + need_to_start = !tid_geq(journal->j_commit_request, tid); + read_unlock(&journal->j_state_lock); +@@ -194,7 +194,7 @@ static void wait_transaction_switching(journal_t *journal) + read_unlock(&journal->j_state_lock); + return; + } +- prepare_to_wait(&journal->j_wait_transaction_locked, &wait, ++ prepare_to_wait_exclusive(&journal->j_wait_transaction_locked, &wait, + TASK_UNINTERRUPTIBLE); + read_unlock(&journal->j_state_lock); + /* +@@ -920,7 +920,7 @@ void jbd2_journal_unlock_updates (journal_t *journal) + write_lock(&journal->j_state_lock); + --journal->j_barrier_count; + write_unlock(&journal->j_state_lock); +- wake_up(&journal->j_wait_transaction_locked); ++ wake_up_all(&journal->j_wait_transaction_locked); + } + + static void warn_dirty_buffer(struct buffer_head *bh) +diff --git a/fs/mbcache.c b/fs/mbcache.c +index 47ccfcbe0a22..e272ad738faf 100644 +--- a/fs/mbcache.c ++++ b/fs/mbcache.c +@@ -90,8 +90,14 @@ int mb_cache_entry_create(struct mb_cache *cache, gfp_t mask, u32 key, + return -ENOMEM; + + INIT_LIST_HEAD(&entry->e_list); +- /* Initial hash reference */ +- atomic_set(&entry->e_refcnt, 1); ++ /* ++ * We create entry with two references. One reference is kept by the ++ * hash table, the other reference is used to protect us from ++ * mb_cache_entry_delete_or_get() until the entry is fully setup. This ++ * avoids nesting of cache->c_list_lock into hash table bit locks which ++ * is problematic for RT. ++ */ ++ atomic_set(&entry->e_refcnt, 2); + entry->e_key = key; + entry->e_value = value; + entry->e_reusable = reusable; +@@ -106,15 +112,12 @@ int mb_cache_entry_create(struct mb_cache *cache, gfp_t mask, u32 key, + } + } + hlist_bl_add_head(&entry->e_hash_list, head); +- /* +- * Add entry to LRU list before it can be found by +- * mb_cache_entry_delete() to avoid races +- */ ++ hlist_bl_unlock(head); + spin_lock(&cache->c_list_lock); + list_add_tail(&entry->e_list, &cache->c_list); + cache->c_entry_count++; + spin_unlock(&cache->c_list_lock); +- hlist_bl_unlock(head); ++ mb_cache_entry_put(cache, entry); + + return 0; + } +diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c +index 58b660dbbee9..c481b14e4fd9 100644 +--- a/fs/ntfs/file.c ++++ b/fs/ntfs/file.c +@@ -527,12 +527,12 @@ static inline int __ntfs_grab_cache_pages(struct address_space *mapping, + goto out; + } + +-static inline int ntfs_submit_bh_for_read(struct buffer_head *bh) ++static inline void ntfs_submit_bh_for_read(struct buffer_head *bh) + { + lock_buffer(bh); + get_bh(bh); + bh->b_end_io = end_buffer_read_sync; +- return submit_bh(REQ_OP_READ, bh); ++ submit_bh(REQ_OP_READ, bh); + } + + /** +diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c +index e56723dc9cd5..49d0d4ea63fc 100644 +--- a/fs/xfs/libxfs/xfs_bmap.c ++++ b/fs/xfs/libxfs/xfs_bmap.c +@@ -294,7 +294,7 @@ xfs_check_block( + else + thispa = XFS_BMBT_PTR_ADDR(mp, block, j, dmxr); + if (*thispa == *pp) { +- xfs_warn(mp, "%s: thispa(%d) == pp(%d) %Ld", ++ xfs_warn(mp, "%s: thispa(%d) == pp(%d) %lld", + __func__, j, i, + (unsigned long long)be64_to_cpu(*thispa)); + xfs_err(mp, "%s: ptrs are equal in node\n", +diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c +index e7201dc68f43..e576560b46e9 100644 +--- a/fs/xfs/libxfs/xfs_da_btree.c ++++ b/fs/xfs/libxfs/xfs_da_btree.c +@@ -2192,8 +2192,8 @@ xfs_da_grow_inode_int( + */ + mapp = kmem_alloc(sizeof(*mapp) * count, 0); + for (b = *bno, mapi = 0; b < *bno + count; ) { +- nmap = min(XFS_BMAP_MAX_NMAP, count); + c = (int)(*bno + count - b); ++ nmap = min(XFS_BMAP_MAX_NMAP, c); + error = xfs_bmapi_write(tp, dp, b, c, + xfs_bmapi_aflag(w)|XFS_BMAPI_METADATA, + args->total, &mapp[mapi], &nmap); +diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c +index 76eedc2756b3..92bac3373f1f 100644 +--- a/fs/xfs/libxfs/xfs_dir2.c ++++ b/fs/xfs/libxfs/xfs_dir2.c +@@ -261,7 +261,7 @@ xfs_dir_createname( + { + struct xfs_da_args *args; + int rval; +- int v; /* type-checking value */ ++ bool v; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + +@@ -357,7 +357,7 @@ xfs_dir_lookup( + { + struct xfs_da_args *args; + int rval; +- int v; /* type-checking value */ ++ bool v; + int lock_mode; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); +@@ -435,7 +435,7 @@ xfs_dir_removename( + { + struct xfs_da_args *args; + int rval; +- int v; /* type-checking value */ ++ bool v; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + XFS_STATS_INC(dp->i_mount, xs_dir_remove); +@@ -493,7 +493,7 @@ xfs_dir_replace( + { + struct xfs_da_args *args; + int rval; +- int v; /* type-checking value */ ++ bool v; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + +@@ -610,19 +610,23 @@ xfs_dir2_grow_inode( + int + xfs_dir2_isblock( + struct xfs_da_args *args, +- int *vp) /* out: 1 is block, 0 is not block */ ++ bool *isblock) + { +- xfs_fileoff_t last; /* last file offset */ +- int rval; ++ struct xfs_mount *mp = args->dp->i_mount; ++ xfs_fileoff_t eof; ++ int error; + +- if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK))) +- return rval; +- rval = XFS_FSB_TO_B(args->dp->i_mount, last) == args->geo->blksize; +- if (XFS_IS_CORRUPT(args->dp->i_mount, +- rval != 0 && +- args->dp->i_disk_size != args->geo->blksize)) ++ error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK); ++ if (error) ++ return error; ++ ++ *isblock = false; ++ if (XFS_FSB_TO_B(mp, eof) != args->geo->blksize) ++ return 0; ++ ++ *isblock = true; ++ if (XFS_IS_CORRUPT(mp, args->dp->i_disk_size != args->geo->blksize)) + return -EFSCORRUPTED; +- *vp = rval; + return 0; + } + +@@ -632,14 +636,20 @@ xfs_dir2_isblock( + int + xfs_dir2_isleaf( + struct xfs_da_args *args, +- int *vp) /* out: 1 is block, 0 is not block */ ++ bool *isleaf) + { +- xfs_fileoff_t last; /* last file offset */ +- int rval; ++ xfs_fileoff_t eof; ++ int error; + +- if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK))) +- return rval; +- *vp = last == args->geo->leafblk + args->geo->fsbcount; ++ error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK); ++ if (error) ++ return error; ++ ++ *isleaf = false; ++ if (eof != args->geo->leafblk + args->geo->fsbcount) ++ return 0; ++ ++ *isleaf = true; + return 0; + } + +diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h +index b6df3c34b26a..dd39f17dd9a9 100644 +--- a/fs/xfs/libxfs/xfs_dir2.h ++++ b/fs/xfs/libxfs/xfs_dir2.h +@@ -61,8 +61,8 @@ extern int xfs_dir2_sf_to_block(struct xfs_da_args *args); + /* + * Interface routines used by userspace utilities + */ +-extern int xfs_dir2_isblock(struct xfs_da_args *args, int *r); +-extern int xfs_dir2_isleaf(struct xfs_da_args *args, int *r); ++extern int xfs_dir2_isblock(struct xfs_da_args *args, bool *isblock); ++extern int xfs_dir2_isleaf(struct xfs_da_args *args, bool *isleaf); + extern int xfs_dir2_shrink_inode(struct xfs_da_args *args, xfs_dir2_db_t db, + struct xfs_buf *bp); + +diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c +index 003812fd7d35..8cd37e6e9d38 100644 +--- a/fs/xfs/libxfs/xfs_dir2_sf.c ++++ b/fs/xfs/libxfs/xfs_dir2_sf.c +@@ -865,7 +865,6 @@ xfs_dir2_sf_lookup( + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + int i; /* entry index */ +- int error; + xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */ + xfs_dir2_sf_hdr_t *sfp; /* shortform structure */ + enum xfs_dacmp cmp; /* comparison result */ +@@ -929,8 +928,7 @@ xfs_dir2_sf_lookup( + if (!ci_sfep) + return -ENOENT; + /* otherwise process the CI match as required by the caller */ +- error = xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen); +- return error; ++ return xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen); + } + + /* +diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c +index 9327a4f39206..6b21760184d9 100644 +--- a/fs/xfs/libxfs/xfs_inode_fork.c ++++ b/fs/xfs/libxfs/xfs_inode_fork.c +@@ -78,7 +78,7 @@ xfs_iformat_local( + */ + if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) { + xfs_warn(ip->i_mount, +- "corrupt inode %Lu (bad size %d for local fork, size = %zd).", ++ "corrupt inode %llu (bad size %d for local fork, size = %zd).", + (unsigned long long) ip->i_ino, size, + XFS_DFORK_SIZE(dip, ip->i_mount, whichfork)); + xfs_inode_verifier_error(ip, -EFSCORRUPTED, +@@ -192,7 +192,7 @@ xfs_iformat_btree( + XFS_DFORK_SIZE(dip, mp, whichfork) || + ifp->if_nextents > ip->i_nblocks) || + level == 0 || level > XFS_BM_MAXLEVELS(mp, whichfork)) { +- xfs_warn(mp, "corrupt inode %Lu (btree).", ++ xfs_warn(mp, "corrupt inode %llu (btree).", + (unsigned long long) ip->i_ino); + xfs_inode_verifier_error(ip, -EFSCORRUPTED, + "xfs_iformat_btree", dfp, size, +diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c +index 5abb5fdb71d9..b9c5764e7437 100644 +--- a/fs/xfs/scrub/dir.c ++++ b/fs/xfs/scrub/dir.c +@@ -676,7 +676,7 @@ xchk_directory_blocks( + xfs_dablk_t dabno; + xfs_dir2_db_t last_data_db = 0; + bool found; +- int is_block = 0; ++ bool is_block = false; + int error; + + /* Ignore local format directories. */ +diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c +index 5077a7ad5646..cf5ce607dc05 100644 +--- a/fs/xfs/xfs_attr_item.c ++++ b/fs/xfs/xfs_attr_item.c +@@ -86,8 +86,6 @@ xfs_attri_log_nameval_alloc( + */ + nv = xlog_kvmalloc(sizeof(struct xfs_attri_log_nameval) + + name_len + value_len); +- if (!nv) +- return nv; + + nv->name.i_addr = nv + 1; + nv->name.i_len = name_len; +@@ -441,8 +439,6 @@ xfs_attr_create_intent( + attr->xattri_nameval = xfs_attri_log_nameval_alloc(args->name, + args->namelen, args->value, args->valuelen); + } +- if (!attr->xattri_nameval) +- return ERR_PTR(-ENOMEM); + + attrip = xfs_attri_init(mp, attr->xattri_nameval); + xfs_trans_add_item(tp, &attrip->attri_item); +@@ -762,8 +758,6 @@ xlog_recover_attri_commit_pass2( + nv = xfs_attri_log_nameval_alloc(attr_name, + attri_formatp->alfi_name_len, attr_value, + attri_formatp->alfi_value_len); +- if (!nv) +- return -ENOMEM; + + attrip = xfs_attri_init(mp, nv); + error = xfs_attri_copy_format(&item->ri_buf[0], &attrip->attri_format); +diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c +index e295fc8062d8..9f3ceb461515 100644 +--- a/fs/xfs/xfs_dir2_readdir.c ++++ b/fs/xfs/xfs_dir2_readdir.c +@@ -512,7 +512,7 @@ xfs_readdir( + { + struct xfs_da_args args = { NULL }; + unsigned int lock_mode; +- int isblock; ++ bool isblock; + int error; + + trace_xfs_readdir(dp); +diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c +index 28493c8e9bb2..c000b74dd203 100644 +--- a/fs/xfs/xfs_inode.c ++++ b/fs/xfs/xfs_inode.c +@@ -835,9 +835,8 @@ xfs_init_new_inode( + * ID or one of the supplementary group IDs, the S_ISGID bit is cleared + * (and only if the irix_sgid_inherit compatibility variable is set). + */ +- if (irix_sgid_inherit && +- (inode->i_mode & S_ISGID) && +- !in_group_p(i_gid_into_mnt(mnt_userns, inode))) ++ if (irix_sgid_inherit && (inode->i_mode & S_ISGID) && ++ !vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode))) + inode->i_mode &= ~S_ISGID; + + ip->i_disk_size = 0; +@@ -3119,7 +3118,7 @@ xfs_iflush( + if (XFS_TEST_ERROR(dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC), + mp, XFS_ERRTAG_IFLUSH_1)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, +- "%s: Bad inode %Lu magic number 0x%x, ptr "PTR_FMT, ++ "%s: Bad inode %llu magic number 0x%x, ptr "PTR_FMT, + __func__, ip->i_ino, be16_to_cpu(dip->di_magic), dip); + goto flush_out; + } +@@ -3129,7 +3128,7 @@ xfs_iflush( + ip->i_df.if_format != XFS_DINODE_FMT_BTREE, + mp, XFS_ERRTAG_IFLUSH_3)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, +- "%s: Bad regular inode %Lu, ptr "PTR_FMT, ++ "%s: Bad regular inode %llu, ptr "PTR_FMT, + __func__, ip->i_ino, ip); + goto flush_out; + } +@@ -3140,7 +3139,7 @@ xfs_iflush( + ip->i_df.if_format != XFS_DINODE_FMT_LOCAL, + mp, XFS_ERRTAG_IFLUSH_4)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, +- "%s: Bad directory inode %Lu, ptr "PTR_FMT, ++ "%s: Bad directory inode %llu, ptr "PTR_FMT, + __func__, ip->i_ino, ip); + goto flush_out; + } +@@ -3158,7 +3157,7 @@ xfs_iflush( + if (XFS_TEST_ERROR(ip->i_forkoff > mp->m_sb.sb_inodesize, + mp, XFS_ERRTAG_IFLUSH_6)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, +- "%s: bad inode %Lu, forkoff 0x%x, ptr "PTR_FMT, ++ "%s: bad inode %llu, forkoff 0x%x, ptr "PTR_FMT, + __func__, ip->i_ino, ip->i_forkoff, ip); + goto flush_out; + } +diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c +index 6e19ece916bf..ca2941ab6cbc 100644 +--- a/fs/xfs/xfs_inode_item.c ++++ b/fs/xfs/xfs_inode_item.c +@@ -550,7 +550,7 @@ xfs_inode_item_push( + + if (!bp || (ip->i_flags & XFS_ISTALE)) { + /* +- * Inode item/buffer is being being aborted due to cluster ++ * Inode item/buffer is being aborted due to cluster + * buffer deletion. Trigger a log force to have that operation + * completed and items removed from the AIL before the next push + * attempt. +diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c +index d28ffaebd067..0e5dba2343ea 100644 +--- a/fs/xfs/xfs_inode_item_recover.c ++++ b/fs/xfs/xfs_inode_item_recover.c +@@ -321,7 +321,7 @@ xlog_recover_inode_commit_pass2( + */ + if (XFS_IS_CORRUPT(mp, !xfs_verify_magic16(bp, dip->di_magic))) { + xfs_alert(mp, +- "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %Ld", ++ "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %lld", + __func__, dip, bp, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; +@@ -329,7 +329,7 @@ xlog_recover_inode_commit_pass2( + ldip = item->ri_buf[1].i_addr; + if (XFS_IS_CORRUPT(mp, ldip->di_magic != XFS_DINODE_MAGIC)) { + xfs_alert(mp, +- "%s: Bad inode log record, rec ptr "PTR_FMT", ino %Ld", ++ "%s: Bad inode log record, rec ptr "PTR_FMT", ino %lld", + __func__, item, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; +diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c +index 45518b8c613c..5d670c85dcc2 100644 +--- a/fs/xfs/xfs_iops.c ++++ b/fs/xfs/xfs_iops.c +@@ -558,6 +558,8 @@ xfs_vn_getattr( + struct inode *inode = d_inode(path->dentry); + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; ++ vfsuid_t vfsuid = i_uid_into_vfsuid(mnt_userns, inode); ++ vfsgid_t vfsgid = i_gid_into_vfsgid(mnt_userns, inode); + + trace_xfs_getattr(ip); + +@@ -568,8 +570,8 @@ xfs_vn_getattr( + stat->dev = inode->i_sb->s_dev; + stat->mode = inode->i_mode; + stat->nlink = inode->i_nlink; +- stat->uid = i_uid_into_mnt(mnt_userns, inode); +- stat->gid = i_gid_into_mnt(mnt_userns, inode); ++ stat->uid = vfsuid_into_kuid(vfsuid); ++ stat->gid = vfsgid_into_kgid(vfsgid); + stat->ino = ip->i_ino; + stat->atime = inode->i_atime; + stat->mtime = inode->i_mtime; +diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h +index cb5fc68c9ea0..e570dcb5df8d 100644 +--- a/fs/xfs/xfs_iops.h ++++ b/fs/xfs/xfs_iops.h +@@ -13,7 +13,6 @@ extern const struct file_operations xfs_dir_file_operations; + + extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size); + +-extern void xfs_setattr_time(struct xfs_inode *ip, struct iattr *iattr); + int xfs_vn_setattr_size(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *vap); + +diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c +index 36312b00b164..a1c2bcf65d37 100644 +--- a/fs/xfs/xfs_itable.c ++++ b/fs/xfs/xfs_itable.c +@@ -66,6 +66,8 @@ xfs_bulkstat_one_int( + struct xfs_bulkstat *buf = bc->buf; + xfs_extnum_t nextents; + int error = -EINVAL; ++ vfsuid_t vfsuid; ++ vfsgid_t vfsgid; + + if (xfs_internal_inum(mp, ino)) + goto out_advance; +@@ -81,14 +83,16 @@ xfs_bulkstat_one_int( + ASSERT(ip != NULL); + ASSERT(ip->i_imap.im_blkno != 0); + inode = VFS_I(ip); ++ vfsuid = i_uid_into_vfsuid(mnt_userns, inode); ++ vfsgid = i_gid_into_vfsgid(mnt_userns, inode); + + /* xfs_iget returns the following without needing + * further change. + */ + buf->bs_projectid = ip->i_projid; + buf->bs_ino = ino; +- buf->bs_uid = from_kuid(sb_userns, i_uid_into_mnt(mnt_userns, inode)); +- buf->bs_gid = from_kgid(sb_userns, i_gid_into_mnt(mnt_userns, inode)); ++ buf->bs_uid = from_kuid(sb_userns, vfsuid_into_kuid(vfsuid)); ++ buf->bs_gid = from_kgid(sb_userns, vfsgid_into_kgid(vfsgid)); + buf->bs_size = ip->i_disk_size; + + buf->bs_nlink = inode->i_nlink; +diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c +index 386b0307aed8..f6e7e4fd72ae 100644 +--- a/fs/xfs/xfs_log.c ++++ b/fs/xfs/xfs_log.c +@@ -226,12 +226,12 @@ xlog_ticket_reservation( + if (head == &log->l_write_head) { + ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); + return tic->t_unit_res; +- } else { +- if (tic->t_flags & XLOG_TIC_PERM_RESERV) +- return tic->t_unit_res * tic->t_cnt; +- else +- return tic->t_unit_res; + } ++ ++ if (tic->t_flags & XLOG_TIC_PERM_RESERV) ++ return tic->t_unit_res * tic->t_cnt; ++ ++ return tic->t_unit_res; + } + + STATIC bool +diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c +index f10c88cee116..e8bb3c2e847e 100644 +--- a/fs/xfs/xfs_mount.c ++++ b/fs/xfs/xfs_mount.c +@@ -300,26 +300,28 @@ xfs_validate_new_dalign( + "alignment check failed: sunit/swidth vs. blocksize(%d)", + mp->m_sb.sb_blocksize); + return -EINVAL; +- } else { +- /* +- * Convert the stripe unit and width to FSBs. +- */ +- mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign); +- if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) { +- xfs_warn(mp, +- "alignment check failed: sunit/swidth vs. agsize(%d)", +- mp->m_sb.sb_agblocks); +- return -EINVAL; +- } else if (mp->m_dalign) { +- mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth); +- } else { +- xfs_warn(mp, +- "alignment check failed: sunit(%d) less than bsize(%d)", +- mp->m_dalign, mp->m_sb.sb_blocksize); +- return -EINVAL; +- } + } + ++ /* ++ * Convert the stripe unit and width to FSBs. ++ */ ++ mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign); ++ if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) { ++ xfs_warn(mp, ++ "alignment check failed: sunit/swidth vs. agsize(%d)", ++ mp->m_sb.sb_agblocks); ++ return -EINVAL; ++ } ++ ++ if (!mp->m_dalign) { ++ xfs_warn(mp, ++ "alignment check failed: sunit(%d) less than bsize(%d)", ++ mp->m_dalign, mp->m_sb.sb_blocksize); ++ return -EINVAL; ++ } ++ ++ mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth); ++ + if (!xfs_has_dalign(mp)) { + xfs_warn(mp, + "cannot change alignment: superblock does not support data alignment"); +diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c +index 5b1f9a24ed59..c4078d0ec108 100644 +--- a/fs/xfs/xfs_notify_failure.c ++++ b/fs/xfs/xfs_notify_failure.c +@@ -23,17 +23,18 @@ + #include + #include + +-struct failure_info { ++struct xfs_failure_info { + xfs_agblock_t startblock; + xfs_extlen_t blockcount; + int mf_flags; ++ bool want_shutdown; + }; + + static pgoff_t + xfs_failure_pgoff( + struct xfs_mount *mp, + const struct xfs_rmap_irec *rec, +- const struct failure_info *notify) ++ const struct xfs_failure_info *notify) + { + loff_t pos = XFS_FSB_TO_B(mp, rec->rm_offset); + +@@ -47,7 +48,7 @@ static unsigned long + xfs_failure_pgcnt( + struct xfs_mount *mp, + const struct xfs_rmap_irec *rec, +- const struct failure_info *notify) ++ const struct xfs_failure_info *notify) + { + xfs_agblock_t end_rec; + xfs_agblock_t end_notify; +@@ -71,13 +72,13 @@ xfs_dax_failure_fn( + { + struct xfs_mount *mp = cur->bc_mp; + struct xfs_inode *ip; +- struct failure_info *notify = data; ++ struct xfs_failure_info *notify = data; + int error = 0; + + if (XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) || + (rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))) { +- xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK); +- return -EFSCORRUPTED; ++ notify->want_shutdown = true; ++ return 0; + } + + /* Get files that incore, filter out others that are not in use. */ +@@ -86,8 +87,10 @@ xfs_dax_failure_fn( + /* Continue the rmap query if the inode isn't incore */ + if (error == -ENODATA) + return 0; +- if (error) +- return error; ++ if (error) { ++ notify->want_shutdown = true; ++ return 0; ++ } + + error = mf_dax_kill_procs(VFS_I(ip)->i_mapping, + xfs_failure_pgoff(mp, rec, notify), +@@ -104,6 +107,7 @@ xfs_dax_notify_ddev_failure( + xfs_daddr_t bblen, + int mf_flags) + { ++ struct xfs_failure_info notify = { .mf_flags = mf_flags }; + struct xfs_trans *tp = NULL; + struct xfs_btree_cur *cur = NULL; + struct xfs_buf *agf_bp = NULL; +@@ -120,7 +124,6 @@ xfs_dax_notify_ddev_failure( + for (; agno <= end_agno; agno++) { + struct xfs_rmap_irec ri_low = { }; + struct xfs_rmap_irec ri_high; +- struct failure_info notify; + struct xfs_agf *agf; + xfs_agblock_t agend; + struct xfs_perag *pag; +@@ -161,6 +164,11 @@ xfs_dax_notify_ddev_failure( + } + + xfs_trans_cancel(tp); ++ if (error || notify.want_shutdown) { ++ xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK); ++ if (!error) ++ error = -EFSCORRUPTED; ++ } + return error; + } + +diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c +index 251f20ddd368..93bdd25680bc 100644 +--- a/fs/xfs/xfs_reflink.c ++++ b/fs/xfs/xfs_reflink.c +@@ -200,7 +200,9 @@ xfs_reflink_trim_around_shared( + if (fbno == NULLAGBLOCK) { + /* No shared blocks at all. */ + return 0; +- } else if (fbno == agbno) { ++ } ++ ++ if (fbno == agbno) { + /* + * The start of this extent is shared. Truncate the + * mapping at the end of the shared region so that a +@@ -210,16 +212,16 @@ xfs_reflink_trim_around_shared( + irec->br_blockcount = flen; + *shared = true; + return 0; +- } else { +- /* +- * There's a shared extent midway through this extent. +- * Truncate the mapping at the start of the shared +- * extent so that a subsequent iteration starts at the +- * start of the shared region. +- */ +- irec->br_blockcount = fbno - agbno; +- return 0; + } ++ ++ /* ++ * There's a shared extent midway through this extent. ++ * Truncate the mapping at the start of the shared ++ * extent so that a subsequent iteration starts at the ++ * start of the shared region. ++ */ ++ irec->br_blockcount = fbno - agbno; ++ return 0; + } + + int +diff --git a/fs/xfs/xfs_stats.c b/fs/xfs/xfs_stats.c +index 20e0534a772c..90a77cd3ebad 100644 +--- a/fs/xfs/xfs_stats.c ++++ b/fs/xfs/xfs_stats.c +@@ -74,7 +74,7 @@ int xfs_stats_format(struct xfsstats __percpu *stats, char *buf) + defer_relog += per_cpu_ptr(stats, i)->s.defer_relog; + } + +- len += scnprintf(buf + len, PATH_MAX-len, "xpc %Lu %Lu %Lu\n", ++ len += scnprintf(buf + len, PATH_MAX-len, "xpc %llu %llu %llu\n", + xs_xstrat_bytes, xs_write_bytes, xs_read_bytes); + len += scnprintf(buf + len, PATH_MAX-len, "defer_relog %llu\n", + defer_relog); +@@ -125,7 +125,7 @@ static int xqmstat_proc_show(struct seq_file *m, void *v) + { + int j; + +- seq_printf(m, "qm"); ++ seq_puts(m, "qm"); + for (j = XFSSTAT_START_XQMSTAT; j < XFSSTAT_END_XQMSTAT; j++) + seq_printf(m, " %u", counter_val(xfsstats.xs_stats, j)); + seq_putc(m, '\n'); +diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c +index 9ac59814bbb6..f029c6702dda 100644 +--- a/fs/xfs/xfs_super.c ++++ b/fs/xfs/xfs_super.c +@@ -653,7 +653,7 @@ xfs_fs_destroy_inode( + static void + xfs_fs_dirty_inode( + struct inode *inode, +- int flag) ++ int flags) + { + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; +@@ -661,7 +661,13 @@ xfs_fs_dirty_inode( + + if (!(inode->i_sb->s_flags & SB_LAZYTIME)) + return; +- if (flag != I_DIRTY_SYNC || !(inode->i_state & I_DIRTY_TIME)) ++ ++ /* ++ * Only do the timestamp update if the inode is dirty (I_DIRTY_SYNC) ++ * and has dirty timestamp (I_DIRTY_TIME). I_DIRTY_TIME can be passed ++ * in flags possibly together with I_DIRTY_SYNC. ++ */ ++ if ((flags & ~I_DIRTY_TIME) != I_DIRTY_SYNC || !(flags & I_DIRTY_TIME)) + return; + + if (xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp)) +diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h +index f9057af6e0c8..cb7c81ba7fa3 100644 +--- a/fs/xfs/xfs_trace.h ++++ b/fs/xfs/xfs_trace.h +@@ -1170,7 +1170,7 @@ DECLARE_EVENT_CLASS(xfs_dqtrx_class, + __entry->ino_res_used = qtrx->qt_ino_res_used; + __entry->icount_delta = qtrx->qt_icount_delta; + ), +- TP_printk("dev %d:%d dquot id 0x%x type %s flags %s" ++ TP_printk("dev %d:%d dquot id 0x%x type %s flags %s " + "blk_res %llu bcount_delta %lld delbcnt_delta %lld " + "rtblk_res %llu rtblk_res_used %llu rtbcount_delta %lld delrtb_delta %lld " + "ino_res %llu ino_res_used %llu icount_delta %lld", +@@ -1602,7 +1602,7 @@ TRACE_EVENT(xfs_bunmap, + __entry->caller_ip = caller_ip; + __entry->flags = flags; + ), +- TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx" ++ TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx " + "flags %s caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, +diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h +index df518c429667..06089390d81d 100644 +--- a/include/linux/buffer_head.h ++++ b/include/linux/buffer_head.h +@@ -240,7 +240,7 @@ void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]); + int sync_dirty_buffer(struct buffer_head *bh); + int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); + void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); +-int submit_bh(blk_opf_t, struct buffer_head *); ++void submit_bh(blk_opf_t, struct buffer_head *); + void write_boundary_block(struct block_device *bdev, + sector_t bblock, unsigned blocksize); + int bh_uptodate_or_lock(struct buffer_head *bh); +diff --git a/include/linux/fs.h b/include/linux/fs.h +index 9eced4cc286e..56a4b4b02477 100644 +--- a/include/linux/fs.h ++++ b/include/linux/fs.h +@@ -2371,13 +2371,14 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, + * don't have to write inode on fdatasync() when only + * e.g. the timestamps have changed. + * I_DIRTY_PAGES Inode has dirty pages. Inode itself may be clean. +- * I_DIRTY_TIME The inode itself only has dirty timestamps, and the ++ * I_DIRTY_TIME The inode itself has dirty timestamps, and the + * lazytime mount option is enabled. We keep track of this + * separately from I_DIRTY_SYNC in order to implement + * lazytime. This gets cleared if I_DIRTY_INODE +- * (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. I.e. +- * either I_DIRTY_TIME *or* I_DIRTY_INODE can be set in +- * i_state, but not both. I_DIRTY_PAGES may still be set. ++ * (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. But ++ * I_DIRTY_TIME can still be set if I_DIRTY_SYNC is already ++ * in place because writeback might already be in progress ++ * and we don't want to lose the time update + * I_NEW Serves as both a mutex and completion notification. + * New inodes set I_NEW. If two processes both create + * the same inode, one of them will release its inode and +diff --git a/mm/page-writeback.c b/mm/page-writeback.c +index 8f20c037a87a..ef3b960ab5f2 100644 +--- a/mm/page-writeback.c ++++ b/mm/page-writeback.c +@@ -1941,6 +1941,7 @@ int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, + wb_put(wb); + return ret; + } ++EXPORT_SYMBOL_GPL(balance_dirty_pages_ratelimited_flags); + + /** + * balance_dirty_pages_ratelimited - balance dirty memory state. +-- +2.38.0 + +From f73b5e7c35d47cc6730c8c6e3415e5e1b9190266 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 10 Oct 2022 20:24:53 +0200 +Subject: [PATCH 16/17] ksm + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/mm/ksm.rst | 36 +++ + fs/proc/base.c | 15 ++ + include/linux/mm_types.h | 5 + + mm/ksm.c | 252 ++++++++++++++++++- + mm/madvise.c | 4 + + tools/testing/selftests/vm/.gitignore | 1 + + tools/testing/selftests/vm/Makefile | 1 + + tools/testing/selftests/vm/test-ksm-auto.c | 273 +++++++++++++++++++++ + 8 files changed, 581 insertions(+), 6 deletions(-) + create mode 100644 tools/testing/selftests/vm/test-ksm-auto.c + +diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst +index b244f0202a03..fb6ba2002a4b 100644 +--- a/Documentation/admin-guide/mm/ksm.rst ++++ b/Documentation/admin-guide/mm/ksm.rst +@@ -184,6 +184,42 @@ The maximum possible ``pages_sharing/pages_shared`` ratio is limited by the + ``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must + be increased accordingly. + ++Monitoring KSM profit ++===================== ++ ++KSM can save memory by merging identical pages, but also can consume ++additional memory, because it needs to generate a number of rmap_items to ++save each scanned page's brief rmap information. Some of these pages may ++be merged, but some may not be abled to be merged after being checked ++several times, which are unprofitable memory consumed. ++ ++1) How to determine whether KSM save memory or consume memory in system-wide ++ range? Here is a simple approximate calculation for reference:: ++ ++ general_profit =~ pages_sharing * sizeof(page) - (all_rmap_items) * ++ sizeof(rmap_item); ++ ++ where all_rmap_items can be easily obtained by summing ``pages_sharing``, ++ ``pages_shared``, ``pages_unshared`` and ``pages_volatile``. ++ ++2) The KSM profit inner a single process can be similarly obtained by the ++ following approximate calculation:: ++ ++ process_profit =~ ksm_merging_pages * sizeof(page) - ++ ksm_rmap_items * sizeof(rmap_item). ++ ++ where ksm_merging_pages is shown under the directory ``/proc//``, ++ and ksm_rmap_items is shown in ``/proc//ksm_stat``. ++ ++From the perspective of application, a high ratio of ``ksm_rmap_items`` to ++``ksm_merging_pages`` means a bad madvise-applied policy, so developers or ++administrators have to rethink how to change madvise policy. Giving an example ++for reference, a page's size is usually 4K, and the rmap_item's size is ++separately 32B on 32-bit CPU architecture and 64B on 64-bit CPU architecture. ++so if the ``ksm_rmap_items/ksm_merging_pages`` ratio exceeds 64 on 64-bit CPU ++or exceeds 128 on 32-bit CPU, then the app's madvise policy should be dropped, ++because the ksm profit is approximately zero or negative. ++ + Monitoring KSM events + ===================== + +diff --git a/fs/proc/base.c b/fs/proc/base.c +index 12885a75913f..ca3e836377e8 100644 +--- a/fs/proc/base.c ++++ b/fs/proc/base.c +@@ -3199,6 +3199,19 @@ static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace * + + return 0; + } ++static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, ++ struct pid *pid, struct task_struct *task) ++{ ++ struct mm_struct *mm; ++ ++ mm = get_task_mm(task); ++ if (mm) { ++ seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); ++ mmput(mm); ++ } ++ ++ return 0; ++} + #endif /* CONFIG_KSM */ + + #ifdef CONFIG_STACKLEAK_METRICS +@@ -3334,6 +3347,7 @@ static const struct pid_entry tgid_base_stuff[] = { + #endif + #ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), ++ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), + #endif + }; + +@@ -3671,6 +3685,7 @@ static const struct pid_entry tid_base_stuff[] = { + #endif + #ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), ++ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), + #endif + }; + +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index 5e32211cb5a9..41935c55a438 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -654,6 +654,11 @@ struct mm_struct { + * merging. + */ + unsigned long ksm_merging_pages; ++ /* ++ * Represent how many pages are checked for ksm merging ++ * including merged and not merged. ++ */ ++ unsigned long ksm_rmap_items; + #endif + #ifdef CONFIG_LRU_GEN + struct { +diff --git a/mm/ksm.c b/mm/ksm.c +index 123b74d0b94f..95b6a2e37cfc 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -131,6 +131,10 @@ struct mm_slot { + * @address: the next address inside that to be scanned + * @rmap_list: link to the next rmap to be scanned in the rmap_list + * @seqnr: count of completed full scans (needed when removing unstable node) ++ * @new_ksmpages: count of the new merged KSM pages in the current scanning ++ * of mm_lists (cleared after every turn of ksm_do_scan() ends) ++ * @prev_ksmpages: the record of the new merged KSM pages in the last turn of ++ * scanning by ksm_do_scan(). + * + * There is only the one ksm_scan instance of this cursor structure. + */ +@@ -139,6 +143,8 @@ struct ksm_scan { + unsigned long address; + struct rmap_item **rmap_list; + unsigned long seqnr; ++ unsigned long new_ksmpages; ++ unsigned long prev_ksmpages; + }; + + /** +@@ -277,6 +283,33 @@ static unsigned int zero_checksum __read_mostly; + /* Whether to merge empty (zeroed) pages with actual zero pages */ + static bool ksm_use_zero_pages __read_mostly; + ++/* ++ * Work in auto-mode. ++ * The multiplicative factor of pages_to_scan. ++ * Real pages to scan equals to the product of scanning_factor ++ * and pages_to_scan ++ */ ++#define INIT_SCANNING_FACTOR 1 ++static unsigned int scanning_factor = INIT_SCANNING_FACTOR; ++ ++/* The upper limit of scanning_factor */ ++#define DEFAULT_MAX_SCANNING_FACTOR 16 ++static unsigned int max_scanning_factor = DEFAULT_MAX_SCANNING_FACTOR; ++ ++/* ++ * Work in auto mode. ++ * Value: 0~100. Default 20 means "20%". When free memory is lower ++ * than this total memory * ksm_auto_threshold/100, auto_triggered ++ * will be set true. ++ */ ++unsigned int ksm_auto_threshold = 20; ++ ++/* Work in auto-mode. Whether trigger ksmd to compare and merge pages */ ++static bool auto_triggered; ++ ++/* Count of times that ksmd is triggered due to low free memory */ ++static unsigned long triggered_times; ++ + #ifdef CONFIG_NUMA + /* Zeroed when merging across nodes is not allowed */ + static unsigned int ksm_merge_across_nodes = 1; +@@ -290,6 +323,7 @@ static int ksm_nr_node_ids = 1; + #define KSM_RUN_MERGE 1 + #define KSM_RUN_UNMERGE 2 + #define KSM_RUN_OFFLINE 4 ++#define KSM_RUN_AUTO 8 + static unsigned long ksm_run = KSM_RUN_STOP; + static void wait_while_offlining(void); + +@@ -387,6 +421,7 @@ static inline struct rmap_item *alloc_rmap_item(void) + static inline void free_rmap_item(struct rmap_item *rmap_item) + { + ksm_rmap_items--; ++ rmap_item->mm->ksm_rmap_items--; + rmap_item->mm = NULL; /* debug safety */ + kmem_cache_free(rmap_item_cache, rmap_item); + } +@@ -2022,6 +2057,8 @@ static void stable_tree_append(struct rmap_item *rmap_item, + rmap_item->address |= STABLE_FLAG; + hlist_add_head(&rmap_item->hlist, &stable_node->hlist); + ++ ksm_scan.new_ksmpages++; ++ + if (rmap_item->hlist.next) + ksm_pages_sharing++; + else +@@ -2221,6 +2258,7 @@ static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, + if (rmap_item) { + /* It has already been zeroed */ + rmap_item->mm = mm_slot->mm; ++ rmap_item->mm->ksm_rmap_items++; + rmap_item->address = addr; + rmap_item->rmap_list = *rmap_list; + *rmap_list = rmap_item; +@@ -2403,16 +2441,107 @@ static void ksm_do_scan(unsigned int scan_npages) + rmap_item = scan_get_next_rmap_item(&page); + if (!rmap_item) + return; +- cmp_and_merge_page(page, rmap_item); ++ if (ksm_run & KSM_RUN_AUTO && !auto_triggered) { ++ /* ++ * This should happens only when ksm_run is KSM_RUN_AUTO ++ * and free memory threshold still not reached. ++ * The reason to calculate it's checksum is to reduce the ++ * waiting time the rmap_item is added to unstable tree. ++ */ ++ rmap_item->oldchecksum = calc_checksum(page); ++ } else ++ cmp_and_merge_page(page, rmap_item); ++ + put_page(page); + } + } + ++#define RIGHT_SHIFT_FOUR_BIT 4 ++/* Work in auto mode, should reset auto_triggered ? */ ++static bool should_stop_ksmd_to_merge(void) ++{ ++ unsigned long total_ram_pages, free_pages; ++ unsigned int threshold; ++ ++ total_ram_pages = totalram_pages(); ++ free_pages = global_zone_page_state(NR_FREE_PAGES); ++ threshold = READ_ONCE(ksm_auto_threshold); ++ ++ return free_pages > (total_ram_pages * threshold / 100) + ++ (total_ram_pages >> RIGHT_SHIFT_FOUR_BIT); ++} ++ ++/* Work in auto mode, should ksmd start to merge ? */ ++static bool should_trigger_ksmd_to_merge(void) ++{ ++ unsigned long total_ram_pages, free_pages; ++ unsigned int threshold; ++ ++ total_ram_pages = totalram_pages(); ++ free_pages = global_zone_page_state(NR_FREE_PAGES); ++ threshold = READ_ONCE(ksm_auto_threshold); ++ ++ return free_pages < (total_ram_pages * threshold / 100); ++} ++ ++static inline void trigger_ksmd_to_merge(void) ++{ ++ if (!auto_triggered) { ++ triggered_times++; ++ auto_triggered = true; ++ } ++} ++ ++static inline void stop_ksmd_to_merge(void) ++{ ++ if (auto_triggered) ++ auto_triggered = false; ++} ++ + static int ksmd_should_run(void) + { +- return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list); ++ if (!list_empty(&ksm_mm_head.mm_list)) ++ return ksm_run & KSM_RUN_AUTO || ksm_run & KSM_RUN_MERGE; ++ return 0; ++} ++ ++/* ++ * Work in auto mode, the scan-enhanced algorithm. ++ * current_factor: the current scanning_factor. ++ * return: the scanning_factor caculated by scan-enhanced algorithm. ++ */ ++static unsigned int scan_enhanced_algorithm(unsigned int current_factor) ++{ ++ unsigned int next_factor; ++ unsigned int max, min; ++ ++ /* ++ * The calculation is divied into three cases as follows: ++ * ++ * Case 1: when new_ksmpages > prev_ksmpages * 1/2, get the ++ * next factor by double the current factor. ++ * Case 2: when 0 < new_ksmpages < prev_ksmpages * 1/2, keep ++ * the factor unchanged. ++ * Case 3: when new_ksmpages equals 0, then get the next ++ * factor by halfing the current factor. ++ */ ++ max = READ_ONCE(max_scanning_factor); ++ min = INIT_SCANNING_FACTOR; ++ if (ksm_scan.new_ksmpages * 2 > ksm_scan.prev_ksmpages) { ++ next_factor = current_factor << 1; /* Doubling */ ++ if (next_factor > max) ++ next_factor = max; ++ } else if (ksm_scan.new_ksmpages == 0) { ++ next_factor = current_factor >> 1; /* Halfing */ ++ next_factor = next_factor < min ? min : next_factor; ++ } else ++ next_factor = current_factor; ++ ++ return next_factor; + } + ++#define SLOW_SCAN_PAGES 5 /* Used when ksmd is not triggered to merge*/ ++ + static int ksm_scan_thread(void *nothing) + { + unsigned int sleep_ms; +@@ -2423,8 +2552,27 @@ static int ksm_scan_thread(void *nothing) + while (!kthread_should_stop()) { + mutex_lock(&ksm_thread_mutex); + wait_while_offlining(); +- if (ksmd_should_run()) +- ksm_do_scan(ksm_thread_pages_to_scan); ++ if (ksmd_should_run()) { ++ if (ksm_run & KSM_RUN_AUTO) { ++ if (!auto_triggered) ++ ksm_do_scan(SLOW_SCAN_PAGES); ++ else ++ ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor); ++ ++ scanning_factor = scan_enhanced_algorithm(scanning_factor); ++ /* ++ * Reset ksm_scan.new_ksmpages after ++ * updating scanning_factor by scan_enhanced_algorithm. ++ */ ++ ksm_scan.new_ksmpages = 0; ++ ++ if (should_trigger_ksmd_to_merge()) ++ trigger_ksmd_to_merge(); ++ else if (should_stop_ksmd_to_merge()) ++ stop_ksmd_to_merge(); ++ } else ++ ksm_do_scan(ksm_thread_pages_to_scan); ++ } + mutex_unlock(&ksm_thread_mutex); + + try_to_freeze(); +@@ -2900,6 +3048,34 @@ static ssize_t pages_to_scan_store(struct kobject *kobj, + } + KSM_ATTR(pages_to_scan); + ++static ssize_t max_scanning_factor_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%u\n", max_scanning_factor); ++} ++ ++static ssize_t max_scanning_factor_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int value, max; ++ int err; ++ ++ err = kstrtouint(buf, 10, &value); ++ if (err) ++ return -EINVAL; ++ ++ max = totalram_pages() / ksm_thread_pages_to_scan; ++ ++ if (value < 1 && value > max) ++ return -EINVAL; ++ ++ max_scanning_factor = value; ++ ++ return count; ++} ++KSM_ATTR(max_scanning_factor); ++ + static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) + { +@@ -2915,7 +3091,7 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, + err = kstrtouint(buf, 10, &flags); + if (err) + return -EINVAL; +- if (flags > KSM_RUN_UNMERGE) ++ if (flags > KSM_RUN_UNMERGE && flags != KSM_RUN_AUTO) + return -EINVAL; + + /* +@@ -2941,13 +3117,73 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, + } + mutex_unlock(&ksm_thread_mutex); + +- if (flags & KSM_RUN_MERGE) ++ if (flags & KSM_RUN_MERGE || flags & KSM_RUN_AUTO) + wake_up_interruptible(&ksm_thread_wait); + + return count; + } + KSM_ATTR(run); + ++static ssize_t ksmd_status_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ int len = 0; ++ unsigned int mergeable_mms = 0; ++ struct list_head *pos; ++ ++ list_for_each(pos, &ksm_mm_head.mm_list) ++ mergeable_mms++; ++ ++ if (ksm_run & KSM_RUN_AUTO) { ++ len += sysfs_emit_at(buf, len, "mode: auto\n"); ++ len += sysfs_emit_at(buf, len, "auto_triggered: %d\n", ++ auto_triggered); ++ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", ++ mergeable_mms); ++ len += sysfs_emit_at(buf, len, "scanning_factor: %u\n", ++ scanning_factor); ++ len += sysfs_emit_at(buf, len, "triggered_times: %lu\n", ++ triggered_times); ++ } else if (ksm_run & KSM_RUN_MERGE) { ++ len += sysfs_emit_at(buf, len, "mode: on\n"); ++ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", ++ mergeable_mms); ++ } else if (ksm_run & KSM_RUN_UNMERGE) ++ len += sysfs_emit_at(buf, len, "mode: unmerge\n"); ++ else ++ len += sysfs_emit_at(buf, len, "mode: off\n"); ++ ++ ++ return len; ++} ++KSM_ATTR_RO(ksmd_status); ++ ++static ssize_t auto_threshold_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%u\n", ksm_auto_threshold); ++} ++ ++static ssize_t auto_threshold_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int value; ++ int err; ++ ++ err = kstrtouint(buf, 10, &value); ++ if (err) ++ return -EINVAL; ++ ++ if (value > 100) ++ return -EINVAL; ++ ++ ksm_auto_threshold = value; ++ ++ return count; ++} ++KSM_ATTR(auto_threshold); ++ + #ifdef CONFIG_NUMA + static ssize_t merge_across_nodes_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +@@ -3157,7 +3393,10 @@ KSM_ATTR_RO(full_scans); + static struct attribute *ksm_attrs[] = { + &sleep_millisecs_attr.attr, + &pages_to_scan_attr.attr, ++ &max_scanning_factor_attr.attr, + &run_attr.attr, ++ &ksmd_status_attr.attr, ++ &auto_threshold_attr.attr, + &pages_shared_attr.attr, + &pages_sharing_attr.attr, + &pages_unshared_attr.attr, +@@ -3189,6 +3428,7 @@ static int __init ksm_init(void) + zero_checksum = calc_checksum(ZERO_PAGE(0)); + /* Default to false for backwards compatibility */ + ksm_use_zero_pages = false; ++ auto_triggered = false; + + err = ksm_slab_init(); + if (err) +diff --git a/mm/madvise.c b/mm/madvise.c +index 0c872ac11920..826f940dca6a 100644 +--- a/mm/madvise.c ++++ b/mm/madvise.c +@@ -1176,6 +1176,10 @@ process_madvise_behavior_valid(int behavior) + case MADV_COLD: + case MADV_PAGEOUT: + case MADV_WILLNEED: ++#ifdef CONFIG_KSM ++ case MADV_MERGEABLE: ++ case MADV_UNMERGEABLE: ++#endif + return true; + default: + return false; +diff --git a/tools/testing/selftests/vm/.gitignore b/tools/testing/selftests/vm/.gitignore +index 31e5eea2a9b9..1cd8816c055d 100644 +--- a/tools/testing/selftests/vm/.gitignore ++++ b/tools/testing/selftests/vm/.gitignore +@@ -34,3 +34,4 @@ local_config.* + soft-dirty + split_huge_page_test + ksm_tests ++test-ksm-auto +diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile +index d9fa6a9ea584..002d9482c37f 100644 +--- a/tools/testing/selftests/vm/Makefile ++++ b/tools/testing/selftests/vm/Makefile +@@ -54,6 +54,7 @@ TEST_GEN_FILES += userfaultfd + TEST_GEN_PROGS += soft-dirty + TEST_GEN_PROGS += split_huge_page_test + TEST_GEN_FILES += ksm_tests ++TEST_GEN_FILES += test-ksm-auto + + ifeq ($(MACHINE),x86_64) + CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32) +diff --git a/tools/testing/selftests/vm/test-ksm-auto.c b/tools/testing/selftests/vm/test-ksm-auto.c +new file mode 100644 +index 000000000000..0d71593e008e +--- /dev/null ++++ b/tools/testing/selftests/vm/test-ksm-auto.c +@@ -0,0 +1,273 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define KSM_CLEAR_MODE "2\n" ++#define KSM_NORMAL_MODE "1\n" ++#define KSM_AUTO_MODE "8\n" ++ ++#define PAGESIZE (4*1024) ++/* Don't change the value, it will afffect the result */ ++#define TOTAL_MADVISE_SIZE (300*1024*1024) ++ ++char *ksm_run_file = "/sys/kernel/mm/ksm/run"; ++char *ksm_auto_threshold_file = "/sys/kernel/mm/ksm/auto_threshold"; ++char *ksm_pages_volatile_file = "/sys/kernel/mm/ksm/pages_volatile"; ++char *ksm_pages_sharing_file = "/sys/kernel/mm/ksm/pages_sharing"; ++ ++#define SHAPE_FULL 1 ++#define SHAPE_SPARSE 2 ++/* They are related to the shape of memory */ ++int final_pages[3] = {0, 76500, 42}; ++ ++static char *mmap_and_madvise(long long size, int advise) ++{ ++ char *ptr; ++ int err; ++ ++ err = 0; ++ ++ ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); ++ if (!ptr) ++ return NULL; ++ ++ err = madvise(ptr, size, advise); ++ if (err) { ++ perror("Madvise failed\n"); ++ free(ptr); ++ return NULL; ++ } ++ ++ return ptr; ++} ++ ++void make_samepage_ares(char *ptr, int size, int shape_type) ++{ ++ int i, j; ++ char rnd_num; ++ ++ switch (shape_type) { ++ case SHAPE_FULL: ++ for (i = 0; i < (size / PAGESIZE); i++) ++ memset(ptr + (i * PAGESIZE), 0x1, PAGESIZE); ++ break; ++ case SHAPE_SPARSE: ++ /* Make pages different */ ++ j = 0; ++ for (i = 1; i < (size / PAGESIZE); i++) { ++ ptr[i * PAGESIZE + (j%PAGESIZE)] = j%127 + 1; ++ j++; ++ } ++ for (i = 0; i < (size / PAGESIZE); i += 1800) ++ memset(ptr + (i * PAGESIZE), -1, PAGESIZE); ++ } ++ ++ return; ++} ++ ++int read_file(char *file, char *buffer, int buf_len) ++{ ++ FILE *fp; ++ size_t result; ++ long lSize; ++ ++ fp = fopen(file, "r"); ++ if (!fp) ++ return -1; ++ ++ fseek(fp, 0, SEEK_END); ++ lSize = ftell(fp); ++ rewind(fp); ++ ++ memset(buffer, 0, buf_len); ++ result = fread(buffer, 1, buf_len, fp); ++ if (result == 0) ++ return -1; ++ ++ fclose(fp); ++ ++ return 0; ++} ++ ++int write_file(char *file, const char *buffer, int len) ++{ ++ FILE *fp; ++ size_t result; ++ ++ fp = fopen(file, "w+"); ++ if (!fp) ++ return -1; ++ ++ result = fwrite(buffer, len, 1, fp); ++ if (result == 0) ++ return -1; ++ ++ fclose(fp); ++ ++ return 0; ++} ++ ++static inline void get_orig_info(int *run, int *auto_threshold) ++{ ++ char buffer[50]; ++ ++ /* Read the original state of ksm/run */ ++ if (read_file(ksm_run_file, buffer, sizeof(buffer))) { ++ printf("read file %s failed\n", ksm_run_file); ++ exit(1); ++ } ++ *run = atoi(buffer); ++ ++ if (read_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { ++ printf("read file: %s failed\n", ksm_auto_threshold_file); ++ exit(1); ++ } ++ *auto_threshold = atoi(buffer); ++} ++ ++static inline void restore_orig_state(int run, int auto_threshold) ++{ ++ char buffer[50]; ++ ++ /* restore the original state */ ++ memset(buffer, 0, sizeof(buffer)); ++ snprintf(buffer, sizeof(buffer) - 1, "%d\n", run); ++ if (write_file(ksm_run_file, buffer, sizeof(buffer))) { ++ printf("write file %s failed\n", ksm_run_file); ++ exit(1); ++ } ++ ++ memset(buffer, 0, sizeof(buffer)); ++ snprintf(buffer, sizeof(buffer) - 1, "%d\n", auto_threshold); ++ if (write_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { ++ printf("write file %s failed\n", ksm_run_file); ++ exit(1); ++ } ++} ++ ++void set_ksmd_run_mode(char *mode) ++{ ++ if (write_file(ksm_run_file, mode, 2)) { ++ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); ++ exit(1); ++ } ++} ++ ++static inline void wait_ksmpages_converged(int final_pages) ++{ ++ int pages_sharing; ++ char buffer[50]; ++ ++ for (;;) { ++ if (read_file(ksm_pages_sharing_file, buffer, sizeof(buffer))) { ++ printf("read file %s failed\n", ksm_pages_sharing_file); ++ exit(1); ++ } ++ ++ pages_sharing = atoi(buffer); ++ if (pages_sharing >= final_pages) ++ break; ++ } ++} ++ ++void print_shape(int shape_type) ++{ ++ switch (shape_type) { ++ case SHAPE_FULL: ++ printf("Now the shape of memory area is full-samepages:\n"); ++ printf("[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n\n"); ++ break; ++ case SHAPE_SPARSE: ++ printf("Now the shape of memory area is sparse-samepages:\n"); ++ printf("[xx] [xx] [xx] \n\n"); ++ break; ++ } ++} ++ ++void print_ksmd_cpu_comsuption(void) ++{ ++ system("(ps x| grep \"ksmd\" | grep -v grep | awk \'{print $1}\' |" ++ " xargs -i cat /proc/{}/stat) | awk \'{print \"ksm current " ++ "cpu total slice: \" $14+$15+$16+$17}\'"); ++} ++ ++void test_ksmd_performance(char *madvise_area, int shape_type) ++{ ++ struct timeval tv_start, tv_end; ++ ++ make_samepage_ares(madvise_area, TOTAL_MADVISE_SIZE, shape_type); ++ print_shape(shape_type); ++ ++ /********* Start to time ksmd's normal-run mode **********/ ++ printf("Start to test normal-run ksmd...\n"); ++ ++ print_ksmd_cpu_comsuption(); ++ ++ set_ksmd_run_mode(KSM_CLEAR_MODE); ++ set_ksmd_run_mode(KSM_NORMAL_MODE); ++ ++ gettimeofday(&tv_start, NULL); ++ ++ wait_ksmpages_converged(final_pages[shape_type]); ++ ++ gettimeofday(&tv_end, NULL); ++ printf("ksm normal-run's merging time: %lf seconds\n", ++ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - ++ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); ++ ++ /******* Start to time ksmd's auto-run mode **********/ ++ print_ksmd_cpu_comsuption(); ++ ++ printf("Start to test auto-run ksmd...\n"); ++ set_ksmd_run_mode(KSM_CLEAR_MODE); ++ set_ksmd_run_mode(KSM_AUTO_MODE); ++ if (write_file(ksm_auto_threshold_file, "99\n", 2)) ++ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); ++ ++ gettimeofday(&tv_start, NULL); ++ ++ wait_ksmpages_converged(shape_type); ++ ++ gettimeofday(&tv_end, NULL); ++ printf("ksm auto-run's merging time: %lf seconds\n", ++ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - ++ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); ++ ++ print_ksmd_cpu_comsuption(); ++} ++ ++int main(int argc, char **argv) ++{ ++ char *madvise_area; ++ int orig_run, orig_auto_threshold; ++ ++ /* Get the original state of ksm */ ++ get_orig_info(&orig_run, &orig_auto_threshold); ++ printf("Now we mmap 300MB anouymous memory for testing.\n" ++ "There are two type of TEST which have different shape of\n" ++ "samepage areas.\n" ++ "Note: the test requires no other MERGEABLE-madvised vm areas\n" ++ "in system than the areas our testing process allocs.\n"); ++ madvise_area = mmap_and_madvise(TOTAL_MADVISE_SIZE, MADV_MERGEABLE); ++ if (!madvise_area) { ++ printf("madvise failed\n"); ++ exit(1); ++ } ++ ++ printf("\n****************** TEST 1 ******************\n"); ++ test_ksmd_performance(madvise_area, SHAPE_FULL); ++ printf("\n****************** TEST 2 ******************\n"); ++ test_ksmd_performance(madvise_area, SHAPE_SPARSE); ++ ++ /* Restore the original state */ ++ restore_orig_state(orig_run, orig_auto_threshold); ++ ++ return 0; ++} +-- +2.38.0 + +From 7028e64abc9ada4f0bc9cf636a2abc7810c189c4 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Tue, 11 Oct 2022 23:12:15 +0200 +Subject: [PATCH 17/17] THP Shrinker + +https://lore.kernel.org/lkml/cover.1664323311.git.alexlzhu@fb.com/ + +Transparent Hugepages use a larger page size of 2MB in comparison to +normal sized pages that are 4kb. A larger page size allows for fewer TLB +cache misses and thus more efficient use of the CPU. Using a larger page +size also results in more memory waste, which can hurt performance in some +use cases. THPs are currently enabled in the Linux Kernel by applications +in limited virtual address ranges via the madvise system call. The THP +shrinker tries to find a balance between increased use of THPs, and +increased use of memory. It shrinks the size of memory by removing the +underutilized THPs that are identified by the thp_utilization scanner. + +In our experiments we have noticed that the least utilized THPs are almost +entirely unutilized. + +Sample Output: + +Utilized[0-50]: 1331 680884 +Utilized[51-101]: 9 3983 +Utilized[102-152]: 3 1187 +Utilized[153-203]: 0 0 +Utilized[204-255]: 2 539 +Utilized[256-306]: 5 1135 +Utilized[307-357]: 1 192 +Utilized[358-408]: 0 0 +Utilized[409-459]: 1 57 +Utilized[460-512]: 400 13 +Last Scan Time: 223.98s +Last Scan Duration: 70.65s + +Above is a sample obtained from one of our test machines when THP is always +enabled. Of the 1331 THPs in this thp_utilization sample that have from +0-50 utilized subpages, we see that there are 680884 free pages. This +comes out to 680884 / (512 * 1331) = 99.91% zero pages in the least +utilized bucket. This represents 680884 * 4KB = 2.7GB memory waste. + +Also note that the vast majority of pages are either in the least utilized +[0-50] or most utilized [460-512] buckets. The least utilized THPs are +responsible for almost all of the memory waste when THP is always +enabled. Thus by clearing out THPs in the lowest utilization bucket +we extract most of the improvement in CPU efficiency. We have seen +similar results on our production hosts. + +This patchset introduces the THP shrinker we have developed to identify +and split the least utilized THPs. It includes the thp_utilization +changes that groups anonymous THPs into buckets, the split_huge_page() +changes that identify and zap zero 4KB pages within THPs and the shrinker +changes. It should be noted that the split_huge_page() changes are based +off previous work done by Yu Zhao. + +In the future, we intend to allow additional tuning to the shrinker +based on workload depending on CPU/IO/Memory pressure and the +amount of anonymous memory. The long term goal is to eventually always +enable THP for all applications and deprecate madvise entirely. + +In production we thus far have observed 2-3% reduction in overall cpu +usage on stateless web servers when THP is always enabled. + +Alexander Zhu (3): + mm: add thp_utilization metrics to debugfs + mm: changes to split_huge_page() to free zero filled tail pages + mm: THP low utilization shrinker + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/mm/transhuge.rst | 9 + + include/linux/huge_mm.h | 10 + + include/linux/list_lru.h | 24 ++ + include/linux/mm_types.h | 5 + + include/linux/rmap.h | 2 +- + include/linux/vm_event_item.h | 3 + + mm/huge_memory.c | 342 +++++++++++++++++- + mm/list_lru.c | 49 +++ + mm/migrate.c | 72 +++- + mm/migrate_device.c | 4 +- + mm/page_alloc.c | 6 + + mm/vmstat.c | 3 + + .../selftests/vm/split_huge_page_test.c | 113 +++++- + tools/testing/selftests/vm/vm_util.c | 23 ++ + tools/testing/selftests/vm/vm_util.h | 1 + + 15 files changed, 648 insertions(+), 18 deletions(-) + +diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst +index c9c37f16eef8..d883ff9fddc7 100644 +--- a/Documentation/admin-guide/mm/transhuge.rst ++++ b/Documentation/admin-guide/mm/transhuge.rst +@@ -297,6 +297,15 @@ To identify what applications are mapping file transparent huge pages, it + is necessary to read ``/proc/PID/smaps`` and count the FileHugeMapped fields + for each mapping. + ++The utilization of transparent hugepages can be viewed by reading ++``/sys/kernel/debug/thp_utilization``. The utilization of a THP is defined ++as the ratio of non zero filled 4kb pages to the total number of pages in a ++THP. The buckets are labelled by the range of total utilized 4kb pages with ++one line per utilization bucket. Each line contains the total number of ++THPs in that bucket and the total number of zero filled 4kb pages summed ++over all THPs in that bucket. The last two lines show the timestamp and ++duration respectively of the most recent scan over all of physical memory. ++ + Note that reading the smaps file is expensive and reading it + frequently will incur overhead. + +diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h +index 768e5261fdae..d6e55bf67625 100644 +--- a/include/linux/huge_mm.h ++++ b/include/linux/huge_mm.h +@@ -179,6 +179,9 @@ bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags); + ++int thp_number_utilized_pages(struct page *page); ++int thp_utilization_bucket(int num_utilized_pages); ++ + void prep_transhuge_page(struct page *page); + void free_transhuge_page(struct page *page); + +@@ -190,6 +193,8 @@ static inline int split_huge_page(struct page *page) + } + void deferred_split_huge_page(struct page *page); + ++void add_underutilized_thp(struct page *page); ++ + void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long address, bool freeze, struct folio *folio); + +@@ -300,6 +305,11 @@ static inline struct list_head *page_deferred_list(struct page *page) + return &page[2].deferred_list; + } + ++static inline struct list_head *page_underutilized_thp_list(struct page *page) ++{ ++ return &page[3].underutilized_thp_list; ++} ++ + #else /* CONFIG_TRANSPARENT_HUGEPAGE */ + #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) + #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) +diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h +index b35968ee9fb5..c2cf146ea880 100644 +--- a/include/linux/list_lru.h ++++ b/include/linux/list_lru.h +@@ -89,6 +89,18 @@ void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *paren + */ + bool list_lru_add(struct list_lru *lru, struct list_head *item); + ++/** ++ * list_lru_add_page: add an element to the lru list's tail ++ * @list_lru: the lru pointer ++ * @page: the page containing the item ++ * @item: the item to be deleted. ++ * ++ * This function works the same as list_lru_add in terms of list ++ * manipulation. Used for non slab objects contained in the page. ++ * ++ * Return value: true if the list was updated, false otherwise ++ */ ++bool list_lru_add_page(struct list_lru *lru, struct page *page, struct list_head *item); + /** + * list_lru_del: delete an element to the lru list + * @list_lru: the lru pointer +@@ -102,6 +114,18 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item); + */ + bool list_lru_del(struct list_lru *lru, struct list_head *item); + ++/** ++ * list_lru_del_page: delete an element to the lru list ++ * @list_lru: the lru pointer ++ * @page: the page containing the item ++ * @item: the item to be deleted. ++ * ++ * This function works the same as list_lru_del in terms of list ++ * manipulation. Used for non slab objects contained in the page. ++ * ++ * Return value: true if the list was updated, false otherwise ++ */ ++bool list_lru_del_page(struct list_lru *lru, struct page *page, struct list_head *item); + /** + * list_lru_count_one: return the number of objects currently held by @lru + * @lru: the lru pointer. +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index 41935c55a438..99f6a749bab7 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -152,6 +152,11 @@ struct page { + /* For both global and memcg */ + struct list_head deferred_list; + }; ++ struct { /* Third tail page of compound page */ ++ unsigned long _compound_pad_3; /* compound_head */ ++ unsigned long _compound_pad_4; ++ struct list_head underutilized_thp_list; ++ }; + struct { /* Page table pages */ + unsigned long _pt_pad_1; /* compound_head */ + pgtable_t pmd_huge_pte; /* protected by page->ptl */ +diff --git a/include/linux/rmap.h b/include/linux/rmap.h +index b89b4b86951f..f7d5d5639dea 100644 +--- a/include/linux/rmap.h ++++ b/include/linux/rmap.h +@@ -372,7 +372,7 @@ int folio_mkclean(struct folio *); + int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, + struct vm_area_struct *vma); + +-void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked); ++void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked, bool unmap_clean); + + int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma); + +diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h +index 3518dba1e02f..3618b10ddec9 100644 +--- a/include/linux/vm_event_item.h ++++ b/include/linux/vm_event_item.h +@@ -111,6 +111,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD + THP_SPLIT_PUD, + #endif ++ THP_SPLIT_FREE, ++ THP_SPLIT_UNMAP, ++ THP_SPLIT_REMAP_READONLY_ZERO_PAGE, + THP_ZERO_PAGE_ALLOC, + THP_ZERO_PAGE_ALLOC_FAILED, + THP_SWPOUT, +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 2a98e4125f24..8ed70e1fea3e 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -46,6 +46,16 @@ + #define CREATE_TRACE_POINTS + #include + ++/* ++ * The number of utilization buckets THPs will be grouped in ++ * under /sys/kernel/debug/thp_utilization. ++ */ ++#define THP_UTIL_BUCKET_NR 10 ++/* ++ * The number of PFNs (and hence hugepages) to scan through on each periodic ++ * run of the scanner that generates /sys/kernel/debug/thp_utilization. ++ */ ++#define THP_UTIL_SCAN_SIZE 256 + /* + * By default, transparent hugepage support is disabled in order to avoid + * risking an increased memory footprint for applications that are not +@@ -71,6 +81,27 @@ static atomic_t huge_zero_refcount; + struct page *huge_zero_page __read_mostly; + unsigned long huge_zero_pfn __read_mostly = ~0UL; + ++struct list_lru huge_low_util_page_lru; ++ ++static void thp_utilization_workfn(struct work_struct *work); ++static DECLARE_DELAYED_WORK(thp_utilization_work, thp_utilization_workfn); ++ ++struct thp_scan_info_bucket { ++ int nr_thps; ++ int nr_zero_pages; ++}; ++ ++struct thp_scan_info { ++ struct thp_scan_info_bucket buckets[THP_UTIL_BUCKET_NR]; ++ struct zone *scan_zone; ++ struct timespec64 last_scan_duration; ++ struct timespec64 last_scan_time; ++ unsigned long pfn; ++}; ++ ++static struct thp_scan_info thp_scan_debugfs; ++static struct thp_scan_info thp_scan; ++ + bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long vm_flags, + bool smaps, bool in_pf) +@@ -236,6 +267,51 @@ static struct shrinker huge_zero_page_shrinker = { + .seeks = DEFAULT_SEEKS, + }; + ++static enum lru_status low_util_free_page(struct list_head *item, ++ struct list_lru_one *lru, ++ spinlock_t *lock, ++ void *cb_arg) ++{ ++ int bucket, num_utilized_pages; ++ struct page *head = compound_head(list_entry(item, ++ struct page, ++ underutilized_thp_list)); ++ ++ if (get_page_unless_zero(head)) { ++ lock_page(head); ++ list_lru_isolate(lru, item); ++ num_utilized_pages = thp_number_utilized_pages(head); ++ bucket = thp_utilization_bucket(num_utilized_pages); ++ if (bucket < THP_UTIL_BUCKET_NR - 1) ++ split_huge_page(head); ++ unlock_page(head); ++ put_page(head); ++ } ++ ++ return LRU_REMOVED_RETRY; ++} ++ ++static unsigned long shrink_huge_low_util_page_count(struct shrinker *shrink, ++ struct shrink_control *sc) ++{ ++ return HPAGE_PMD_NR * list_lru_shrink_count(&huge_low_util_page_lru, sc); ++} ++ ++static unsigned long shrink_huge_low_util_page_scan(struct shrinker *shrink, ++ struct shrink_control *sc) ++{ ++ return HPAGE_PMD_NR * list_lru_shrink_walk(&huge_low_util_page_lru, ++ sc, low_util_free_page, NULL); ++} ++ ++static struct shrinker huge_low_util_page_shrinker = { ++ .count_objects = shrink_huge_low_util_page_count, ++ .scan_objects = shrink_huge_low_util_page_scan, ++ .seeks = DEFAULT_SEEKS, ++ .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE | ++ SHRINKER_NONSLAB, ++}; ++ + #ifdef CONFIG_SYSFS + static ssize_t enabled_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +@@ -487,13 +563,19 @@ static int __init hugepage_init(void) + if (err) + goto err_slab; + ++ schedule_delayed_work(&thp_utilization_work, HZ); ++ err = register_shrinker(&huge_low_util_page_shrinker, "thp-low-util"); ++ if (err) ++ goto err_low_util_shrinker; + err = register_shrinker(&huge_zero_page_shrinker, "thp-zero"); + if (err) + goto err_hzp_shrinker; + err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split"); + if (err) + goto err_split_shrinker; +- ++ err = list_lru_init_memcg(&huge_low_util_page_lru, &huge_low_util_page_shrinker); ++ if (err) ++ goto err_low_util_list_lru; + /* + * By default disable transparent hugepages on smaller systems, + * where the extra memory used could hurt more than TLB overhead +@@ -509,11 +591,16 @@ static int __init hugepage_init(void) + goto err_khugepaged; + + return 0; ++ + err_khugepaged: ++ list_lru_destroy(&huge_low_util_page_lru); ++err_low_util_list_lru: + unregister_shrinker(&deferred_split_shrinker); + err_split_shrinker: + unregister_shrinker(&huge_zero_page_shrinker); + err_hzp_shrinker: ++ unregister_shrinker(&huge_low_util_page_shrinker); ++err_low_util_shrinker: + khugepaged_destroy(); + err_slab: + hugepage_exit_sysfs(hugepage_kobj); +@@ -588,6 +675,7 @@ void prep_transhuge_page(struct page *page) + */ + + INIT_LIST_HEAD(page_deferred_list(page)); ++ INIT_LIST_HEAD(page_underutilized_thp_list(page)); + set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); + } + +@@ -601,6 +689,11 @@ static inline bool is_transparent_hugepage(struct page *page) + page[1].compound_dtor == TRANSHUGE_PAGE_DTOR; + } + ++static inline bool is_anon_transparent_hugepage(struct page *page) ++{ ++ return PageAnon(page) && is_transparent_hugepage(page); ++} ++ + static unsigned long __thp_get_unmapped_area(struct file *filp, + unsigned long addr, unsigned long len, + loff_t off, unsigned long flags, unsigned long size) +@@ -651,6 +744,49 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, + } + EXPORT_SYMBOL_GPL(thp_get_unmapped_area); + ++int thp_number_utilized_pages(struct page *page) ++{ ++ struct folio *folio; ++ unsigned long page_offset, value; ++ int thp_nr_utilized_pages = HPAGE_PMD_NR; ++ int step_size = sizeof(unsigned long); ++ bool is_all_zeroes; ++ void *kaddr; ++ int i; ++ ++ if (!page || !is_anon_transparent_hugepage(page)) ++ return -1; ++ ++ folio = page_folio(page); ++ for (i = 0; i < folio_nr_pages(folio); i++) { ++ kaddr = kmap_local_folio(folio, i); ++ is_all_zeroes = true; ++ for (page_offset = 0; page_offset < PAGE_SIZE; page_offset += step_size) { ++ value = *(unsigned long *)(kaddr + page_offset); ++ if (value != 0) { ++ is_all_zeroes = false; ++ break; ++ } ++ } ++ if (is_all_zeroes) ++ thp_nr_utilized_pages--; ++ ++ kunmap_local(kaddr); ++ } ++ return thp_nr_utilized_pages; ++} ++ ++int thp_utilization_bucket(int num_utilized_pages) ++{ ++ int bucket; ++ ++ if (num_utilized_pages < 0 || num_utilized_pages > HPAGE_PMD_NR) ++ return -1; ++ /* Group THPs into utilization buckets */ ++ bucket = num_utilized_pages * THP_UTIL_BUCKET_NR / HPAGE_PMD_NR; ++ return min(bucket, THP_UTIL_BUCKET_NR - 1); ++} ++ + static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, + struct page *page, gfp_t gfp) + { +@@ -2365,7 +2501,7 @@ static void unmap_page(struct page *page) + try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); + } + +-static void remap_page(struct folio *folio, unsigned long nr) ++static void remap_page(struct folio *folio, unsigned long nr, bool unmap_clean) + { + int i = 0; + +@@ -2373,7 +2509,7 @@ static void remap_page(struct folio *folio, unsigned long nr) + if (!folio_test_anon(folio)) + return; + for (;;) { +- remove_migration_ptes(folio, folio, true); ++ remove_migration_ptes(folio, folio, true, unmap_clean); + i += folio_nr_pages(folio); + if (i >= nr) + break; +@@ -2443,8 +2579,7 @@ static void __split_huge_page_tail(struct page *head, int tail, + LRU_GEN_MASK | LRU_REFS_MASK)); + + /* ->mapping in first tail page is compound_mapcount */ +- VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, +- page_tail); ++ VM_BUG_ON_PAGE(tail > 3 && page_tail->mapping != TAIL_MAPPING, page_tail); + page_tail->mapping = head->mapping; + page_tail->index = head->index + tail; + if (!PageSwapCache(page_tail)) +@@ -2489,6 +2624,8 @@ static void __split_huge_page(struct page *page, struct list_head *list, + struct address_space *swap_cache = NULL; + unsigned long offset = 0; + unsigned int nr = thp_nr_pages(head); ++ LIST_HEAD(pages_to_free); ++ int nr_pages_to_free = 0; + int i; + + /* complete memcg works before add pages to LRU */ +@@ -2551,7 +2688,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, + } + local_irq_enable(); + +- remap_page(folio, nr); ++ remap_page(folio, nr, PageAnon(head)); + + if (PageSwapCache(head)) { + swp_entry_t entry = { .val = page_private(head) }; +@@ -2565,6 +2702,33 @@ static void __split_huge_page(struct page *page, struct list_head *list, + continue; + unlock_page(subpage); + ++ /* ++ * If a tail page has only two references left, one inherited ++ * from the isolation of its head and the other from ++ * lru_add_page_tail() which we are about to drop, it means this ++ * tail page was concurrently zapped. Then we can safely free it ++ * and save page reclaim or migration the trouble of trying it. ++ */ ++ if (list && page_ref_freeze(subpage, 2)) { ++ VM_BUG_ON_PAGE(PageLRU(subpage), subpage); ++ VM_BUG_ON_PAGE(PageCompound(subpage), subpage); ++ VM_BUG_ON_PAGE(page_mapped(subpage), subpage); ++ ++ ClearPageActive(subpage); ++ ClearPageUnevictable(subpage); ++ list_move(&subpage->lru, &pages_to_free); ++ nr_pages_to_free++; ++ continue; ++ } ++ /* ++ * If a tail page has only one reference left, it will be freed ++ * by the call to free_page_and_swap_cache below. Since zero ++ * subpages are no longer remapped, there will only be one ++ * reference left in cases outside of reclaim or migration. ++ */ ++ if (page_ref_count(subpage) == 1) ++ nr_pages_to_free++; ++ + /* + * Subpages may be freed if there wasn't any mapping + * like if add_to_swap() is running on a lru page that +@@ -2574,6 +2738,13 @@ static void __split_huge_page(struct page *page, struct list_head *list, + */ + free_page_and_swap_cache(subpage); + } ++ ++ if (!nr_pages_to_free) ++ return; ++ ++ mem_cgroup_uncharge_list(&pages_to_free); ++ free_unref_page_list(&pages_to_free); ++ count_vm_events(THP_SPLIT_FREE, nr_pages_to_free); + } + + /* Racy check whether the huge page can be split */ +@@ -2616,6 +2787,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + struct folio *folio = page_folio(page); + struct page *head = &folio->page; + struct deferred_split *ds_queue = get_deferred_split_queue(head); ++ struct list_head *underutilized_thp_list = page_underutilized_thp_list(head); + XA_STATE(xas, &head->mapping->i_pages, head->index); + struct anon_vma *anon_vma = NULL; + struct address_space *mapping = NULL; +@@ -2714,6 +2886,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + list_del(page_deferred_list(head)); + } + spin_unlock(&ds_queue->split_queue_lock); ++ if (!list_empty(underutilized_thp_list)) ++ list_lru_del_page(&huge_low_util_page_lru, head, underutilized_thp_list); + if (mapping) { + int nr = thp_nr_pages(head); + +@@ -2736,7 +2910,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + if (mapping) + xas_unlock(&xas); + local_irq_enable(); +- remap_page(folio, folio_nr_pages(folio)); ++ remap_page(folio, folio_nr_pages(folio), false); + ret = -EBUSY; + } + +@@ -2756,6 +2930,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + void free_transhuge_page(struct page *page) + { + struct deferred_split *ds_queue = get_deferred_split_queue(page); ++ struct list_head *underutilized_thp_list = page_underutilized_thp_list(page); + unsigned long flags; + + spin_lock_irqsave(&ds_queue->split_queue_lock, flags); +@@ -2764,6 +2939,12 @@ void free_transhuge_page(struct page *page) + list_del(page_deferred_list(page)); + } + spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); ++ if (!list_empty(underutilized_thp_list)) ++ list_lru_del_page(&huge_low_util_page_lru, page, underutilized_thp_list); ++ ++ if (PageLRU(page)) ++ __clear_page_lru_flags(page); ++ + free_compound_page(page); + } + +@@ -2804,6 +2985,26 @@ void deferred_split_huge_page(struct page *page) + spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); + } + ++void add_underutilized_thp(struct page *page) ++{ ++ VM_BUG_ON_PAGE(!PageTransHuge(page), page); ++ ++ if (PageSwapCache(page)) ++ return; ++ ++ /* ++ * Need to take a reference on the page to prevent the page from getting free'd from ++ * under us while we are adding the THP to the shrinker. ++ */ ++ if (!get_page_unless_zero(page)) ++ return; ++ ++ if (!is_huge_zero_page(page) && is_anon_transparent_hugepage(page)) ++ list_lru_add_page(&huge_low_util_page_lru, page, page_underutilized_thp_list(page)); ++ ++ put_page(page); ++} ++ + static unsigned long deferred_split_count(struct shrinker *shrink, + struct shrink_control *sc) + { +@@ -3158,6 +3359,42 @@ static int __init split_huge_pages_debugfs(void) + return 0; + } + late_initcall(split_huge_pages_debugfs); ++ ++static int thp_utilization_show(struct seq_file *seqf, void *pos) ++{ ++ int i; ++ int start; ++ int end; ++ ++ for (i = 0; i < THP_UTIL_BUCKET_NR; i++) { ++ start = i * HPAGE_PMD_NR / THP_UTIL_BUCKET_NR; ++ end = (i + 1 == THP_UTIL_BUCKET_NR) ++ ? HPAGE_PMD_NR ++ : ((i + 1) * HPAGE_PMD_NR / THP_UTIL_BUCKET_NR - 1); ++ /* The last bucket will need to contain 100 */ ++ seq_printf(seqf, "Utilized[%d-%d]: %d %d\n", start, end, ++ thp_scan_debugfs.buckets[i].nr_thps, ++ thp_scan_debugfs.buckets[i].nr_zero_pages); ++ } ++ seq_printf(seqf, "Last Scan Time: %lu.%02lus\n", ++ (unsigned long)thp_scan_debugfs.last_scan_time.tv_sec, ++ (thp_scan_debugfs.last_scan_time.tv_nsec / (NSEC_PER_SEC / 100))); ++ ++ seq_printf(seqf, "Last Scan Duration: %lu.%02lus\n", ++ (unsigned long)thp_scan_debugfs.last_scan_duration.tv_sec, ++ (thp_scan_debugfs.last_scan_duration.tv_nsec / (NSEC_PER_SEC / 100))); ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(thp_utilization); ++ ++static int __init thp_utilization_debugfs(void) ++{ ++ debugfs_create_file("thp_utilization", 0200, NULL, NULL, ++ &thp_utilization_fops); ++ return 0; ++} ++late_initcall(thp_utilization_debugfs); + #endif + + #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +@@ -3243,3 +3480,94 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) + trace_remove_migration_pmd(address, pmd_val(pmde)); + } + #endif ++ ++static void thp_scan_next_zone(void) ++{ ++ struct timespec64 current_time; ++ int i; ++ bool update_debugfs; ++ /* ++ * THP utilization worker thread has reached the end ++ * of the memory zone. Proceed to the next zone. ++ */ ++ thp_scan.scan_zone = next_zone(thp_scan.scan_zone); ++ update_debugfs = !thp_scan.scan_zone; ++ thp_scan.scan_zone = update_debugfs ? (first_online_pgdat())->node_zones ++ : thp_scan.scan_zone; ++ thp_scan.pfn = (thp_scan.scan_zone->zone_start_pfn + HPAGE_PMD_NR - 1) ++ & ~(HPAGE_PMD_SIZE - 1); ++ if (!update_debugfs) ++ return; ++ /* ++ * If the worker has scanned through all of physical ++ * memory. Then update information displayed in /sys/kernel/debug/thp_utilization ++ */ ++ ktime_get_ts64(¤t_time); ++ thp_scan_debugfs.last_scan_duration = timespec64_sub(current_time, ++ thp_scan_debugfs.last_scan_time); ++ thp_scan_debugfs.last_scan_time = current_time; ++ ++ for (i = 0; i < THP_UTIL_BUCKET_NR; i++) { ++ thp_scan_debugfs.buckets[i].nr_thps = thp_scan.buckets[i].nr_thps; ++ thp_scan_debugfs.buckets[i].nr_zero_pages = thp_scan.buckets[i].nr_zero_pages; ++ thp_scan.buckets[i].nr_thps = 0; ++ thp_scan.buckets[i].nr_zero_pages = 0; ++ } ++} ++ ++static void thp_util_scan(unsigned long pfn_end) ++{ ++ struct page *page = NULL; ++ int bucket, num_utilized_pages, current_pfn; ++ int i; ++ /* ++ * Scan through each memory zone in chunks of THP_UTIL_SCAN_SIZE ++ * PFNs every second looking for anonymous THPs. ++ */ ++ for (i = 0; i < THP_UTIL_SCAN_SIZE; i++) { ++ current_pfn = thp_scan.pfn; ++ thp_scan.pfn += HPAGE_PMD_NR; ++ if (current_pfn >= pfn_end) ++ return; ++ ++ if (!pfn_valid(current_pfn)) ++ continue; ++ ++ page = pfn_to_page(current_pfn); ++ num_utilized_pages = thp_number_utilized_pages(page); ++ bucket = thp_utilization_bucket(num_utilized_pages); ++ if (bucket < 0) ++ continue; ++ ++ if (bucket < THP_UTIL_BUCKET_NR - 1) ++ add_underutilized_thp(page); ++ ++ thp_scan.buckets[bucket].nr_thps++; ++ thp_scan.buckets[bucket].nr_zero_pages += (HPAGE_PMD_NR - num_utilized_pages); ++ } ++} ++ ++static void thp_utilization_workfn(struct work_struct *work) ++{ ++ unsigned long pfn_end; ++ ++ if (!thp_scan.scan_zone) ++ thp_scan.scan_zone = (first_online_pgdat())->node_zones; ++ /* ++ * Worker function that scans through all of physical memory ++ * for anonymous THPs. ++ */ ++ pfn_end = (thp_scan.scan_zone->zone_start_pfn + ++ thp_scan.scan_zone->spanned_pages + HPAGE_PMD_NR - 1) ++ & ~(HPAGE_PMD_SIZE - 1); ++ /* If we have reached the end of the zone or end of physical memory ++ * move on to the next zone. Otherwise, scan the next PFNs in the ++ * current zone. ++ */ ++ if (!populated_zone(thp_scan.scan_zone) || thp_scan.pfn >= pfn_end) ++ thp_scan_next_zone(); ++ else ++ thp_util_scan(pfn_end); ++ ++ schedule_delayed_work(&thp_utilization_work, HZ); ++} +diff --git a/mm/list_lru.c b/mm/list_lru.c +index a05e5bef3b40..7e8b324cc840 100644 +--- a/mm/list_lru.c ++++ b/mm/list_lru.c +@@ -140,6 +140,32 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item) + } + EXPORT_SYMBOL_GPL(list_lru_add); + ++bool list_lru_add_page(struct list_lru *lru, struct page *page, struct list_head *item) ++{ ++ int nid = page_to_nid(page); ++ struct list_lru_node *nlru = &lru->node[nid]; ++ struct list_lru_one *l; ++ struct mem_cgroup *memcg; ++ ++ spin_lock(&nlru->lock); ++ if (list_empty(item)) { ++ memcg = page_memcg(page); ++ memcg_list_lru_alloc(memcg, lru, GFP_KERNEL); ++ l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); ++ list_add_tail(item, &l->list); ++ /* Set shrinker bit if the first element was added */ ++ if (!l->nr_items++) ++ set_shrinker_bit(memcg, nid, ++ lru_shrinker_id(lru)); ++ nlru->nr_items++; ++ spin_unlock(&nlru->lock); ++ return true; ++ } ++ spin_unlock(&nlru->lock); ++ return false; ++} ++EXPORT_SYMBOL_GPL(list_lru_add_page); ++ + bool list_lru_del(struct list_lru *lru, struct list_head *item) + { + int nid = page_to_nid(virt_to_page(item)); +@@ -160,6 +186,29 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item) + } + EXPORT_SYMBOL_GPL(list_lru_del); + ++bool list_lru_del_page(struct list_lru *lru, struct page *page, struct list_head *item) ++{ ++ int nid = page_to_nid(page); ++ struct list_lru_node *nlru = &lru->node[nid]; ++ struct list_lru_one *l; ++ struct mem_cgroup *memcg; ++ ++ spin_lock(&nlru->lock); ++ if (!list_empty(item)) { ++ memcg = page_memcg(page); ++ memcg_list_lru_alloc(memcg, lru, GFP_KERNEL); ++ l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); ++ list_del_init(item); ++ l->nr_items--; ++ nlru->nr_items--; ++ spin_unlock(&nlru->lock); ++ return true; ++ } ++ spin_unlock(&nlru->lock); ++ return false; ++} ++EXPORT_SYMBOL_GPL(list_lru_del_page); ++ + void list_lru_isolate(struct list_lru_one *list, struct list_head *item) + { + list_del_init(item); +diff --git a/mm/migrate.c b/mm/migrate.c +index 55e7718cfe45..57908d680276 100644 +--- a/mm/migrate.c ++++ b/mm/migrate.c +@@ -168,13 +168,62 @@ void putback_movable_pages(struct list_head *l) + } + } + ++static bool try_to_unmap_clean(struct page_vma_mapped_walk *pvmw, struct page *page) ++{ ++ void *addr; ++ bool dirty; ++ pte_t newpte; ++ ++ VM_BUG_ON_PAGE(PageCompound(page), page); ++ VM_BUG_ON_PAGE(!PageAnon(page), page); ++ VM_BUG_ON_PAGE(!PageLocked(page), page); ++ VM_BUG_ON_PAGE(pte_present(*pvmw->pte), page); ++ ++ if (PageMlocked(page) || (pvmw->vma->vm_flags & VM_LOCKED)) ++ return false; ++ ++ /* ++ * The pmd entry mapping the old thp was flushed and the pte mapping ++ * this subpage has been non present. Therefore, this subpage is ++ * inaccessible. We don't need to remap it if it contains only zeros. ++ */ ++ addr = kmap_local_page(page); ++ dirty = memchr_inv(addr, 0, PAGE_SIZE); ++ kunmap_local(addr); ++ ++ if (dirty) ++ return false; ++ ++ pte_clear_not_present_full(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, false); ++ ++ if (userfaultfd_armed(pvmw->vma)) { ++ newpte = pte_mkspecial(pfn_pte(page_to_pfn(ZERO_PAGE(pvmw->address)), ++ pvmw->vma->vm_page_prot)); ++ ptep_clear_flush(pvmw->vma, pvmw->address, pvmw->pte); ++ set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); ++ dec_mm_counter(pvmw->vma->vm_mm, MM_ANONPAGES); ++ count_vm_event(THP_SPLIT_REMAP_READONLY_ZERO_PAGE); ++ return true; ++ } ++ ++ dec_mm_counter(pvmw->vma->vm_mm, mm_counter(page)); ++ count_vm_event(THP_SPLIT_UNMAP); ++ return true; ++} ++ ++struct rmap_walk_arg { ++ struct folio *folio; ++ bool unmap_clean; ++}; ++ + /* + * Restore a potential migration pte to a working pte entry + */ + static bool remove_migration_pte(struct folio *folio, +- struct vm_area_struct *vma, unsigned long addr, void *old) ++ struct vm_area_struct *vma, unsigned long addr, void *arg) + { +- DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION); ++ struct rmap_walk_arg *rmap_walk_arg = arg; ++ DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + + while (page_vma_mapped_walk(&pvmw)) { + rmap_t rmap_flags = RMAP_NONE; +@@ -197,6 +246,8 @@ static bool remove_migration_pte(struct folio *folio, + continue; + } + #endif ++ if (rmap_walk_arg->unmap_clean && try_to_unmap_clean(&pvmw, new)) ++ continue; + + folio_get(folio); + pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); +@@ -268,13 +319,20 @@ static bool remove_migration_pte(struct folio *folio, + * Get rid of all migration entries and replace them by + * references to the indicated page. + */ +-void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked) ++void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked, bool unmap_clean) + { ++ struct rmap_walk_arg rmap_walk_arg = { ++ .folio = src, ++ .unmap_clean = unmap_clean, ++ }; ++ + struct rmap_walk_control rwc = { + .rmap_one = remove_migration_pte, +- .arg = src, ++ .arg = &rmap_walk_arg, + }; + ++ VM_BUG_ON_FOLIO(unmap_clean && src != dst, src); ++ + if (locked) + rmap_walk_locked(dst, &rwc); + else +@@ -850,7 +908,7 @@ static int writeout(struct address_space *mapping, struct folio *folio) + * At this point we know that the migration attempt cannot + * be successful. + */ +- remove_migration_ptes(folio, folio, false); ++ remove_migration_ptes(folio, folio, false, false); + + rc = mapping->a_ops->writepage(&folio->page, &wbc); + +@@ -1109,7 +1167,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, + + if (page_was_mapped) + remove_migration_ptes(folio, +- rc == MIGRATEPAGE_SUCCESS ? dst : folio, false); ++ rc == MIGRATEPAGE_SUCCESS ? dst : folio, false, false); + + out_unlock_both: + unlock_page(newpage); +@@ -1319,7 +1377,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, + + if (page_was_mapped) + remove_migration_ptes(src, +- rc == MIGRATEPAGE_SUCCESS ? dst : src, false); ++ rc == MIGRATEPAGE_SUCCESS ? dst : src, false, false); + + unlock_put_anon: + unlock_page(new_hpage); +diff --git a/mm/migrate_device.c b/mm/migrate_device.c +index dbf6c7a7a7c9..518aacc914c9 100644 +--- a/mm/migrate_device.c ++++ b/mm/migrate_device.c +@@ -413,7 +413,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) + continue; + + folio = page_folio(page); +- remove_migration_ptes(folio, folio, false); ++ remove_migration_ptes(folio, folio, false, false); + + migrate->src[i] = 0; + folio_unlock(folio); +@@ -789,7 +789,7 @@ void migrate_vma_finalize(struct migrate_vma *migrate) + + src = page_folio(page); + dst = page_folio(newpage); +- remove_migration_ptes(src, dst, false); ++ remove_migration_ptes(src, dst, false, false); + folio_unlock(src); + + if (is_zone_device_page(page)) +diff --git a/mm/page_alloc.c b/mm/page_alloc.c +index cc6179d3a7dc..ed2eddb2fbf4 100644 +--- a/mm/page_alloc.c ++++ b/mm/page_alloc.c +@@ -1327,6 +1327,12 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) + * deferred_list.next -- ignore value. + */ + break; ++ case 3: ++ /* ++ * the third tail page: ->mapping is ++ * underutilized_thp_list.next -- ignore value. ++ */ ++ break; + default: + if (page->mapping != TAIL_MAPPING) { + bad_page(page, "corrupted mapping in tail page"); +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 33091a67627e..f6c5d0e97499 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -1369,6 +1369,9 @@ const char * const vmstat_text[] = { + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD + "thp_split_pud", + #endif ++ "thp_split_free", ++ "thp_split_unmap", ++ "thp_split_remap_readonly_zero_page", + "thp_zero_page_alloc", + "thp_zero_page_alloc_failed", + "thp_swpout", +diff --git a/tools/testing/selftests/vm/split_huge_page_test.c b/tools/testing/selftests/vm/split_huge_page_test.c +index 6aa2b8253aed..2c669aadbfd0 100644 +--- a/tools/testing/selftests/vm/split_huge_page_test.c ++++ b/tools/testing/selftests/vm/split_huge_page_test.c +@@ -16,6 +16,9 @@ + #include + #include + #include ++#include /* Definition of SYS_* constants */ ++#include ++#include + #include "vm_util.h" + + uint64_t pagesize; +@@ -88,6 +91,113 @@ static void write_debugfs(const char *fmt, ...) + } + } + ++static char *allocate_zero_filled_hugepage(size_t len) ++{ ++ char *result; ++ size_t i; ++ ++ result = memalign(pmd_pagesize, len); ++ if (!result) { ++ printf("Fail to allocate memory\n"); ++ exit(EXIT_FAILURE); ++ } ++ madvise(result, len, MADV_HUGEPAGE); ++ ++ for (i = 0; i < len; i++) ++ result[i] = (char)0; ++ ++ return result; ++} ++ ++static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, size_t len) ++{ ++ uint64_t thp_size, rss_anon_before, rss_anon_after; ++ size_t i; ++ ++ thp_size = check_huge(one_page); ++ if (!thp_size) { ++ printf("No THP is allocated\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ rss_anon_before = rss_anon(); ++ if (!rss_anon_before) { ++ printf("No RssAnon is allocated before split\n"); ++ exit(EXIT_FAILURE); ++ } ++ /* split all THPs */ ++ write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, ++ (uint64_t)one_page + len); ++ ++ for (i = 0; i < len; i++) ++ if (one_page[i] != (char)0) { ++ printf("%ld byte corrupted\n", i); ++ exit(EXIT_FAILURE); ++ } ++ ++ thp_size = check_huge(one_page); ++ if (thp_size) { ++ printf("Still %ld kB AnonHugePages not split\n", thp_size); ++ exit(EXIT_FAILURE); ++ } ++ ++ rss_anon_after = rss_anon(); ++ if (rss_anon_after >= rss_anon_before) { ++ printf("Incorrect RssAnon value. Before: %ld After: %ld\n", ++ rss_anon_before, rss_anon_after); ++ exit(EXIT_FAILURE); ++ } ++} ++ ++void split_pmd_zero_pages(void) ++{ ++ char *one_page; ++ size_t len = 4 * pmd_pagesize; ++ ++ one_page = allocate_zero_filled_hugepage(len); ++ verify_rss_anon_split_huge_page_all_zeroes(one_page, len); ++ printf("Split zero filled huge pages successful\n"); ++ free(one_page); ++} ++ ++void split_pmd_zero_pages_uffd(void) ++{ ++ char *one_page; ++ size_t len = 4 * pmd_pagesize; ++ long uffd; /* userfaultfd file descriptor */ ++ struct uffdio_api uffdio_api; ++ struct uffdio_register uffdio_register; ++ ++ /* Create and enable userfaultfd object. */ ++ ++ uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); ++ if (uffd == -1) { ++ perror("userfaultfd"); ++ exit(1); ++ } ++ ++ uffdio_api.api = UFFD_API; ++ uffdio_api.features = 0; ++ if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) { ++ perror("ioctl-UFFDIO_API"); ++ exit(1); ++ } ++ ++ one_page = allocate_zero_filled_hugepage(len); ++ ++ uffdio_register.range.start = (unsigned long)one_page; ++ uffdio_register.range.len = len; ++ uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; ++ if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) { ++ perror("ioctl-UFFDIO_REGISTER"); ++ exit(1); ++ } ++ ++ verify_rss_anon_split_huge_page_all_zeroes(one_page, len); ++ printf("Split zero filled huge pages with uffd successful\n"); ++ free(one_page); ++} ++ + void split_pmd_thp(void) + { + char *one_page; +@@ -123,7 +233,6 @@ void split_pmd_thp(void) + exit(EXIT_FAILURE); + } + +- + thp_size = check_huge(one_page); + if (thp_size) { + printf("Still %ld kB AnonHugePages not split\n", thp_size); +@@ -305,6 +414,8 @@ int main(int argc, char **argv) + pageshift = ffs(pagesize) - 1; + pmd_pagesize = read_pmd_pagesize(); + ++ split_pmd_zero_pages(); ++ split_pmd_zero_pages_uffd(); + split_pmd_thp(); + split_pte_mapped_thp(); + split_file_backed_thp(); +diff --git a/tools/testing/selftests/vm/vm_util.c b/tools/testing/selftests/vm/vm_util.c +index b58ab11a7a30..c6a785a67fc9 100644 +--- a/tools/testing/selftests/vm/vm_util.c ++++ b/tools/testing/selftests/vm/vm_util.c +@@ -6,6 +6,7 @@ + + #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" + #define SMAP_FILE_PATH "/proc/self/smaps" ++#define STATUS_FILE_PATH "/proc/self/status" + #define MAX_LINE_LENGTH 500 + + uint64_t pagemap_get_entry(int fd, char *start) +@@ -72,6 +73,28 @@ uint64_t read_pmd_pagesize(void) + return strtoul(buf, NULL, 10); + } + ++uint64_t rss_anon(void) ++{ ++ uint64_t rss_anon = 0; ++ int ret; ++ FILE *fp; ++ char buffer[MAX_LINE_LENGTH]; ++ ++ fp = fopen(STATUS_FILE_PATH, "r"); ++ if (!fp) ++ ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); ++ ++ if (!check_for_pattern(fp, "RssAnon:", buffer)) ++ goto err_out; ++ ++ if (sscanf(buffer, "RssAnon:%10ld kB", &rss_anon) != 1) ++ ksft_exit_fail_msg("Reading status error\n"); ++ ++err_out: ++ fclose(fp); ++ return rss_anon; ++} ++ + uint64_t check_huge(void *addr) + { + uint64_t thp = 0; +diff --git a/tools/testing/selftests/vm/vm_util.h b/tools/testing/selftests/vm/vm_util.h +index 2e512bd57ae1..00b92ccef20d 100644 +--- a/tools/testing/selftests/vm/vm_util.h ++++ b/tools/testing/selftests/vm/vm_util.h +@@ -6,4 +6,5 @@ uint64_t pagemap_get_entry(int fd, char *start); + bool pagemap_is_softdirty(int fd, char *start); + void clear_softdirty(void); + uint64_t read_pmd_pagesize(void); ++uint64_t rss_anon(void); + uint64_t check_huge(void *addr); +-- +2.38.0 + diff --git a/6.0/0002-bbr2.patch b/6.0/0002-bbr2.patch index 0e613dcd..eedf950d 100644 --- a/6.0/0002-bbr2.patch +++ b/6.0/0002-bbr2.patch @@ -1,7 +1,7 @@ From c4125b4903e46e792158e16439f728813dfe3f44 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 5 Sep 2022 08:34:43 +0200 -Subject: [PATCH 02/15] bbr2 +Subject: [PATCH 02/17] bbr2 Signed-off-by: Peter Jung --- diff --git a/6.0/0003-futex-winesync.patch b/6.0/0003-futex-winesync.patch index e90db06a..945f0bab 100644 --- a/6.0/0003-futex-winesync.patch +++ b/6.0/0003-futex-winesync.patch @@ -1,7 +1,7 @@ From 0d426832aeb2f836b0d96b608bd4a884c0d19220 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Wed, 5 Oct 2022 19:08:58 +0200 -Subject: [PATCH 03/15] futex-winesync +Subject: [PATCH 03/17] futex-winesync Signed-off-by: Peter Jung --- diff --git a/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch b/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch index 7f2471aa..98502389 100644 --- a/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch +++ b/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch @@ -1,7 +1,7 @@ From b5f6f1f7ab1cd65a7e8a40ac3eae83835402e8e8 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:33:47 +0200 -Subject: [PATCH 04/15] Introducing-OpenVPN-Data-Channel-Offload +Subject: [PATCH 04/17] Introducing-OpenVPN-Data-Channel-Offload Signed-off-by: Peter Jung --- diff --git a/6.0/0005-mm-multi-gen-LRU.patch b/6.0/0005-mm-multi-gen-LRU.patch index 913a5923..373d444a 100644 --- a/6.0/0005-mm-multi-gen-LRU.patch +++ b/6.0/0005-mm-multi-gen-LRU.patch @@ -1,7 +1,7 @@ From 0f8d23628b7a867dabd253cf6d5e21fa8a3a9c08 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Sat, 8 Oct 2022 15:02:07 +0200 -Subject: [PATCH 05/15] mm: multi-gen LRU +Subject: [PATCH 05/17] mm: multi-gen LRU Signed-off-by: Peter Jung --- diff --git a/6.0/0006-Introducing-the-Maple-Tree.patch b/6.0/0006-Introducing-the-Maple-Tree.patch index 2ab1f797..dfeebf8d 100644 --- a/6.0/0006-Introducing-the-Maple-Tree.patch +++ b/6.0/0006-Introducing-the-Maple-Tree.patch @@ -1,7 +1,7 @@ From 386f84c5b5b2f5f427c8776d5735cf56f31e0618 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Sat, 8 Oct 2022 15:03:49 +0200 -Subject: [PATCH 06/15] Introducing the Maple Tree +Subject: [PATCH 06/17] Introducing the Maple Tree The maple tree is an RCU-safe range based B-tree designed to use modern processor cache efficiently. There are a number of places in the kernel diff --git a/6.0/0007-mm-demotion-Memory-tiers-and-demotion.patch b/6.0/0007-mm-demotion-Memory-tiers-and-demotion.patch index 65ed7e4b..d2a34a93 100644 --- a/6.0/0007-mm-demotion-Memory-tiers-and-demotion.patch +++ b/6.0/0007-mm-demotion-Memory-tiers-and-demotion.patch @@ -1,7 +1,7 @@ From bc98fba95c9e9b188bcae7ab9cdb292751810fe8 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Wed, 28 Sep 2022 00:26:01 +0200 -Subject: [PATCH 07/15] mm/demotion: Memory tiers and demotion +Subject: [PATCH 07/17] mm/demotion: Memory tiers and demotion The current kernel has the basic memory tiering support: Inactive pages on a higher tier NUMA node can be migrated (demoted) to a lower tier NUMA diff --git a/6.0/0008-rtw88.patch b/6.0/0008-rtw88.patch index f1f9a187..43800718 100644 --- a/6.0/0008-rtw88.patch +++ b/6.0/0008-rtw88.patch @@ -1,7 +1,7 @@ From 628ea87b03b347640307037c86e57664cd6bb47a Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 19 Aug 2022 17:06:47 +0200 -Subject: [PATCH 08/15] rtw88 +Subject: [PATCH 08/17] rtw88 Signed-off-by: Peter Jung --- diff --git a/6.0/0009-lrng.patch b/6.0/0009-lrng.patch index e26373fd..b63d54aa 100644 --- a/6.0/0009-lrng.patch +++ b/6.0/0009-lrng.patch @@ -1,7 +1,7 @@ From f91d174a7cf3721b9c0409552903a5258220d61e Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Tue, 6 Sep 2022 20:04:11 +0200 -Subject: [PATCH 09/15] lrng +Subject: [PATCH 09/17] lrng Signed-off-by: Piotr Gorski --- diff --git a/6.0/0010-folios.patch b/6.0/0010-folios.patch index 4e5ed06d..bd3c885b 100644 --- a/6.0/0010-folios.patch +++ b/6.0/0010-folios.patch @@ -1,7 +1,7 @@ From 4497c11cc44fa6f2f5777eb70d9828add7a1d17e Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Sep 2022 14:40:14 +0200 -Subject: [PATCH 10/15] folios +Subject: [PATCH 10/17] folios Signed-off-by: Peter Jung --- diff --git a/6.0/0011-fixes.patch b/6.0/0011-fixes.patch index e74071d3..429f9ca4 100644 --- a/6.0/0011-fixes.patch +++ b/6.0/0011-fixes.patch @@ -1,7 +1,7 @@ From d33721e72a6bd6579920c553cb1cd11acb809ede Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Wed, 5 Oct 2022 19:09:43 +0200 -Subject: [PATCH 11/15] fixes +Subject: [PATCH 11/17] fixes Signed-off-by: Peter Jung --- diff --git a/6.0/0012-kallsyms.patch b/6.0/0012-kallsyms.patch index af635c16..b68376a9 100644 --- a/6.0/0012-kallsyms.patch +++ b/6.0/0012-kallsyms.patch @@ -1,7 +1,7 @@ From 852ccfcccb332e9ce5767a50c98711cfa1855dd1 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 Sep 2022 00:19:51 +0200 -Subject: [PATCH 12/15] kallsyms +Subject: [PATCH 12/17] kallsyms Signed-off-by: Peter Jung --- diff --git a/6.0/0013-rcu.patch b/6.0/0013-rcu.patch index 003346cc..6329c950 100644 --- a/6.0/0013-rcu.patch +++ b/6.0/0013-rcu.patch @@ -1,7 +1,7 @@ From 318ae604f16ece7680af7a75b1ae427ec2a35b88 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Sun, 2 Oct 2022 19:11:33 +0200 -Subject: [PATCH 13/15] rcu +Subject: [PATCH 13/17] rcu Signed-off-by: Peter Jung --- diff --git a/6.0/0014-clr.patch b/6.0/0014-clr.patch index 79f28273..94d9de61 100644 --- a/6.0/0014-clr.patch +++ b/6.0/0014-clr.patch @@ -1,7 +1,7 @@ From db7064658d7a9e2fee4d9e82b226c7ea1f0c753e Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Tue, 4 Oct 2022 14:06:15 +0200 -Subject: [PATCH 14/15] clr +Subject: [PATCH 14/17] clr Signed-off-by: Peter Jung --- diff --git a/6.0/0015-fs-patches.patch b/6.0/0015-fs-patches.patch index 418463e4..5018f845 100644 --- a/6.0/0015-fs-patches.patch +++ b/6.0/0015-fs-patches.patch @@ -1,7 +1,7 @@ From bbb7e25acdde71641e4a5e82732fbd84e5fc7ade Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 10 Oct 2022 13:17:31 +0200 -Subject: [PATCH 15/15] fs-patches +Subject: [PATCH 15/17] fs-patches Signed-off-by: Peter Jung --- diff --git a/6.0/0016-ksm.patch b/6.0/0016-ksm.patch new file mode 100644 index 00000000..33673422 --- /dev/null +++ b/6.0/0016-ksm.patch @@ -0,0 +1,808 @@ +From f73b5e7c35d47cc6730c8c6e3415e5e1b9190266 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 10 Oct 2022 20:24:53 +0200 +Subject: [PATCH 16/17] ksm + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/mm/ksm.rst | 36 +++ + fs/proc/base.c | 15 ++ + include/linux/mm_types.h | 5 + + mm/ksm.c | 252 ++++++++++++++++++- + mm/madvise.c | 4 + + tools/testing/selftests/vm/.gitignore | 1 + + tools/testing/selftests/vm/Makefile | 1 + + tools/testing/selftests/vm/test-ksm-auto.c | 273 +++++++++++++++++++++ + 8 files changed, 581 insertions(+), 6 deletions(-) + create mode 100644 tools/testing/selftests/vm/test-ksm-auto.c + +diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst +index b244f0202a03..fb6ba2002a4b 100644 +--- a/Documentation/admin-guide/mm/ksm.rst ++++ b/Documentation/admin-guide/mm/ksm.rst +@@ -184,6 +184,42 @@ The maximum possible ``pages_sharing/pages_shared`` ratio is limited by the + ``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must + be increased accordingly. + ++Monitoring KSM profit ++===================== ++ ++KSM can save memory by merging identical pages, but also can consume ++additional memory, because it needs to generate a number of rmap_items to ++save each scanned page's brief rmap information. Some of these pages may ++be merged, but some may not be abled to be merged after being checked ++several times, which are unprofitable memory consumed. ++ ++1) How to determine whether KSM save memory or consume memory in system-wide ++ range? Here is a simple approximate calculation for reference:: ++ ++ general_profit =~ pages_sharing * sizeof(page) - (all_rmap_items) * ++ sizeof(rmap_item); ++ ++ where all_rmap_items can be easily obtained by summing ``pages_sharing``, ++ ``pages_shared``, ``pages_unshared`` and ``pages_volatile``. ++ ++2) The KSM profit inner a single process can be similarly obtained by the ++ following approximate calculation:: ++ ++ process_profit =~ ksm_merging_pages * sizeof(page) - ++ ksm_rmap_items * sizeof(rmap_item). ++ ++ where ksm_merging_pages is shown under the directory ``/proc//``, ++ and ksm_rmap_items is shown in ``/proc//ksm_stat``. ++ ++From the perspective of application, a high ratio of ``ksm_rmap_items`` to ++``ksm_merging_pages`` means a bad madvise-applied policy, so developers or ++administrators have to rethink how to change madvise policy. Giving an example ++for reference, a page's size is usually 4K, and the rmap_item's size is ++separately 32B on 32-bit CPU architecture and 64B on 64-bit CPU architecture. ++so if the ``ksm_rmap_items/ksm_merging_pages`` ratio exceeds 64 on 64-bit CPU ++or exceeds 128 on 32-bit CPU, then the app's madvise policy should be dropped, ++because the ksm profit is approximately zero or negative. ++ + Monitoring KSM events + ===================== + +diff --git a/fs/proc/base.c b/fs/proc/base.c +index 12885a75913f..ca3e836377e8 100644 +--- a/fs/proc/base.c ++++ b/fs/proc/base.c +@@ -3199,6 +3199,19 @@ static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace * + + return 0; + } ++static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, ++ struct pid *pid, struct task_struct *task) ++{ ++ struct mm_struct *mm; ++ ++ mm = get_task_mm(task); ++ if (mm) { ++ seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); ++ mmput(mm); ++ } ++ ++ return 0; ++} + #endif /* CONFIG_KSM */ + + #ifdef CONFIG_STACKLEAK_METRICS +@@ -3334,6 +3347,7 @@ static const struct pid_entry tgid_base_stuff[] = { + #endif + #ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), ++ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), + #endif + }; + +@@ -3671,6 +3685,7 @@ static const struct pid_entry tid_base_stuff[] = { + #endif + #ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), ++ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), + #endif + }; + +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index 5e32211cb5a9..41935c55a438 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -654,6 +654,11 @@ struct mm_struct { + * merging. + */ + unsigned long ksm_merging_pages; ++ /* ++ * Represent how many pages are checked for ksm merging ++ * including merged and not merged. ++ */ ++ unsigned long ksm_rmap_items; + #endif + #ifdef CONFIG_LRU_GEN + struct { +diff --git a/mm/ksm.c b/mm/ksm.c +index 123b74d0b94f..95b6a2e37cfc 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -131,6 +131,10 @@ struct mm_slot { + * @address: the next address inside that to be scanned + * @rmap_list: link to the next rmap to be scanned in the rmap_list + * @seqnr: count of completed full scans (needed when removing unstable node) ++ * @new_ksmpages: count of the new merged KSM pages in the current scanning ++ * of mm_lists (cleared after every turn of ksm_do_scan() ends) ++ * @prev_ksmpages: the record of the new merged KSM pages in the last turn of ++ * scanning by ksm_do_scan(). + * + * There is only the one ksm_scan instance of this cursor structure. + */ +@@ -139,6 +143,8 @@ struct ksm_scan { + unsigned long address; + struct rmap_item **rmap_list; + unsigned long seqnr; ++ unsigned long new_ksmpages; ++ unsigned long prev_ksmpages; + }; + + /** +@@ -277,6 +283,33 @@ static unsigned int zero_checksum __read_mostly; + /* Whether to merge empty (zeroed) pages with actual zero pages */ + static bool ksm_use_zero_pages __read_mostly; + ++/* ++ * Work in auto-mode. ++ * The multiplicative factor of pages_to_scan. ++ * Real pages to scan equals to the product of scanning_factor ++ * and pages_to_scan ++ */ ++#define INIT_SCANNING_FACTOR 1 ++static unsigned int scanning_factor = INIT_SCANNING_FACTOR; ++ ++/* The upper limit of scanning_factor */ ++#define DEFAULT_MAX_SCANNING_FACTOR 16 ++static unsigned int max_scanning_factor = DEFAULT_MAX_SCANNING_FACTOR; ++ ++/* ++ * Work in auto mode. ++ * Value: 0~100. Default 20 means "20%". When free memory is lower ++ * than this total memory * ksm_auto_threshold/100, auto_triggered ++ * will be set true. ++ */ ++unsigned int ksm_auto_threshold = 20; ++ ++/* Work in auto-mode. Whether trigger ksmd to compare and merge pages */ ++static bool auto_triggered; ++ ++/* Count of times that ksmd is triggered due to low free memory */ ++static unsigned long triggered_times; ++ + #ifdef CONFIG_NUMA + /* Zeroed when merging across nodes is not allowed */ + static unsigned int ksm_merge_across_nodes = 1; +@@ -290,6 +323,7 @@ static int ksm_nr_node_ids = 1; + #define KSM_RUN_MERGE 1 + #define KSM_RUN_UNMERGE 2 + #define KSM_RUN_OFFLINE 4 ++#define KSM_RUN_AUTO 8 + static unsigned long ksm_run = KSM_RUN_STOP; + static void wait_while_offlining(void); + +@@ -387,6 +421,7 @@ static inline struct rmap_item *alloc_rmap_item(void) + static inline void free_rmap_item(struct rmap_item *rmap_item) + { + ksm_rmap_items--; ++ rmap_item->mm->ksm_rmap_items--; + rmap_item->mm = NULL; /* debug safety */ + kmem_cache_free(rmap_item_cache, rmap_item); + } +@@ -2022,6 +2057,8 @@ static void stable_tree_append(struct rmap_item *rmap_item, + rmap_item->address |= STABLE_FLAG; + hlist_add_head(&rmap_item->hlist, &stable_node->hlist); + ++ ksm_scan.new_ksmpages++; ++ + if (rmap_item->hlist.next) + ksm_pages_sharing++; + else +@@ -2221,6 +2258,7 @@ static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, + if (rmap_item) { + /* It has already been zeroed */ + rmap_item->mm = mm_slot->mm; ++ rmap_item->mm->ksm_rmap_items++; + rmap_item->address = addr; + rmap_item->rmap_list = *rmap_list; + *rmap_list = rmap_item; +@@ -2403,16 +2441,107 @@ static void ksm_do_scan(unsigned int scan_npages) + rmap_item = scan_get_next_rmap_item(&page); + if (!rmap_item) + return; +- cmp_and_merge_page(page, rmap_item); ++ if (ksm_run & KSM_RUN_AUTO && !auto_triggered) { ++ /* ++ * This should happens only when ksm_run is KSM_RUN_AUTO ++ * and free memory threshold still not reached. ++ * The reason to calculate it's checksum is to reduce the ++ * waiting time the rmap_item is added to unstable tree. ++ */ ++ rmap_item->oldchecksum = calc_checksum(page); ++ } else ++ cmp_and_merge_page(page, rmap_item); ++ + put_page(page); + } + } + ++#define RIGHT_SHIFT_FOUR_BIT 4 ++/* Work in auto mode, should reset auto_triggered ? */ ++static bool should_stop_ksmd_to_merge(void) ++{ ++ unsigned long total_ram_pages, free_pages; ++ unsigned int threshold; ++ ++ total_ram_pages = totalram_pages(); ++ free_pages = global_zone_page_state(NR_FREE_PAGES); ++ threshold = READ_ONCE(ksm_auto_threshold); ++ ++ return free_pages > (total_ram_pages * threshold / 100) + ++ (total_ram_pages >> RIGHT_SHIFT_FOUR_BIT); ++} ++ ++/* Work in auto mode, should ksmd start to merge ? */ ++static bool should_trigger_ksmd_to_merge(void) ++{ ++ unsigned long total_ram_pages, free_pages; ++ unsigned int threshold; ++ ++ total_ram_pages = totalram_pages(); ++ free_pages = global_zone_page_state(NR_FREE_PAGES); ++ threshold = READ_ONCE(ksm_auto_threshold); ++ ++ return free_pages < (total_ram_pages * threshold / 100); ++} ++ ++static inline void trigger_ksmd_to_merge(void) ++{ ++ if (!auto_triggered) { ++ triggered_times++; ++ auto_triggered = true; ++ } ++} ++ ++static inline void stop_ksmd_to_merge(void) ++{ ++ if (auto_triggered) ++ auto_triggered = false; ++} ++ + static int ksmd_should_run(void) + { +- return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list); ++ if (!list_empty(&ksm_mm_head.mm_list)) ++ return ksm_run & KSM_RUN_AUTO || ksm_run & KSM_RUN_MERGE; ++ return 0; ++} ++ ++/* ++ * Work in auto mode, the scan-enhanced algorithm. ++ * current_factor: the current scanning_factor. ++ * return: the scanning_factor caculated by scan-enhanced algorithm. ++ */ ++static unsigned int scan_enhanced_algorithm(unsigned int current_factor) ++{ ++ unsigned int next_factor; ++ unsigned int max, min; ++ ++ /* ++ * The calculation is divied into three cases as follows: ++ * ++ * Case 1: when new_ksmpages > prev_ksmpages * 1/2, get the ++ * next factor by double the current factor. ++ * Case 2: when 0 < new_ksmpages < prev_ksmpages * 1/2, keep ++ * the factor unchanged. ++ * Case 3: when new_ksmpages equals 0, then get the next ++ * factor by halfing the current factor. ++ */ ++ max = READ_ONCE(max_scanning_factor); ++ min = INIT_SCANNING_FACTOR; ++ if (ksm_scan.new_ksmpages * 2 > ksm_scan.prev_ksmpages) { ++ next_factor = current_factor << 1; /* Doubling */ ++ if (next_factor > max) ++ next_factor = max; ++ } else if (ksm_scan.new_ksmpages == 0) { ++ next_factor = current_factor >> 1; /* Halfing */ ++ next_factor = next_factor < min ? min : next_factor; ++ } else ++ next_factor = current_factor; ++ ++ return next_factor; + } + ++#define SLOW_SCAN_PAGES 5 /* Used when ksmd is not triggered to merge*/ ++ + static int ksm_scan_thread(void *nothing) + { + unsigned int sleep_ms; +@@ -2423,8 +2552,27 @@ static int ksm_scan_thread(void *nothing) + while (!kthread_should_stop()) { + mutex_lock(&ksm_thread_mutex); + wait_while_offlining(); +- if (ksmd_should_run()) +- ksm_do_scan(ksm_thread_pages_to_scan); ++ if (ksmd_should_run()) { ++ if (ksm_run & KSM_RUN_AUTO) { ++ if (!auto_triggered) ++ ksm_do_scan(SLOW_SCAN_PAGES); ++ else ++ ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor); ++ ++ scanning_factor = scan_enhanced_algorithm(scanning_factor); ++ /* ++ * Reset ksm_scan.new_ksmpages after ++ * updating scanning_factor by scan_enhanced_algorithm. ++ */ ++ ksm_scan.new_ksmpages = 0; ++ ++ if (should_trigger_ksmd_to_merge()) ++ trigger_ksmd_to_merge(); ++ else if (should_stop_ksmd_to_merge()) ++ stop_ksmd_to_merge(); ++ } else ++ ksm_do_scan(ksm_thread_pages_to_scan); ++ } + mutex_unlock(&ksm_thread_mutex); + + try_to_freeze(); +@@ -2900,6 +3048,34 @@ static ssize_t pages_to_scan_store(struct kobject *kobj, + } + KSM_ATTR(pages_to_scan); + ++static ssize_t max_scanning_factor_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%u\n", max_scanning_factor); ++} ++ ++static ssize_t max_scanning_factor_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int value, max; ++ int err; ++ ++ err = kstrtouint(buf, 10, &value); ++ if (err) ++ return -EINVAL; ++ ++ max = totalram_pages() / ksm_thread_pages_to_scan; ++ ++ if (value < 1 && value > max) ++ return -EINVAL; ++ ++ max_scanning_factor = value; ++ ++ return count; ++} ++KSM_ATTR(max_scanning_factor); ++ + static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) + { +@@ -2915,7 +3091,7 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, + err = kstrtouint(buf, 10, &flags); + if (err) + return -EINVAL; +- if (flags > KSM_RUN_UNMERGE) ++ if (flags > KSM_RUN_UNMERGE && flags != KSM_RUN_AUTO) + return -EINVAL; + + /* +@@ -2941,13 +3117,73 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, + } + mutex_unlock(&ksm_thread_mutex); + +- if (flags & KSM_RUN_MERGE) ++ if (flags & KSM_RUN_MERGE || flags & KSM_RUN_AUTO) + wake_up_interruptible(&ksm_thread_wait); + + return count; + } + KSM_ATTR(run); + ++static ssize_t ksmd_status_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ int len = 0; ++ unsigned int mergeable_mms = 0; ++ struct list_head *pos; ++ ++ list_for_each(pos, &ksm_mm_head.mm_list) ++ mergeable_mms++; ++ ++ if (ksm_run & KSM_RUN_AUTO) { ++ len += sysfs_emit_at(buf, len, "mode: auto\n"); ++ len += sysfs_emit_at(buf, len, "auto_triggered: %d\n", ++ auto_triggered); ++ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", ++ mergeable_mms); ++ len += sysfs_emit_at(buf, len, "scanning_factor: %u\n", ++ scanning_factor); ++ len += sysfs_emit_at(buf, len, "triggered_times: %lu\n", ++ triggered_times); ++ } else if (ksm_run & KSM_RUN_MERGE) { ++ len += sysfs_emit_at(buf, len, "mode: on\n"); ++ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", ++ mergeable_mms); ++ } else if (ksm_run & KSM_RUN_UNMERGE) ++ len += sysfs_emit_at(buf, len, "mode: unmerge\n"); ++ else ++ len += sysfs_emit_at(buf, len, "mode: off\n"); ++ ++ ++ return len; ++} ++KSM_ATTR_RO(ksmd_status); ++ ++static ssize_t auto_threshold_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%u\n", ksm_auto_threshold); ++} ++ ++static ssize_t auto_threshold_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int value; ++ int err; ++ ++ err = kstrtouint(buf, 10, &value); ++ if (err) ++ return -EINVAL; ++ ++ if (value > 100) ++ return -EINVAL; ++ ++ ksm_auto_threshold = value; ++ ++ return count; ++} ++KSM_ATTR(auto_threshold); ++ + #ifdef CONFIG_NUMA + static ssize_t merge_across_nodes_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +@@ -3157,7 +3393,10 @@ KSM_ATTR_RO(full_scans); + static struct attribute *ksm_attrs[] = { + &sleep_millisecs_attr.attr, + &pages_to_scan_attr.attr, ++ &max_scanning_factor_attr.attr, + &run_attr.attr, ++ &ksmd_status_attr.attr, ++ &auto_threshold_attr.attr, + &pages_shared_attr.attr, + &pages_sharing_attr.attr, + &pages_unshared_attr.attr, +@@ -3189,6 +3428,7 @@ static int __init ksm_init(void) + zero_checksum = calc_checksum(ZERO_PAGE(0)); + /* Default to false for backwards compatibility */ + ksm_use_zero_pages = false; ++ auto_triggered = false; + + err = ksm_slab_init(); + if (err) +diff --git a/mm/madvise.c b/mm/madvise.c +index 0c872ac11920..826f940dca6a 100644 +--- a/mm/madvise.c ++++ b/mm/madvise.c +@@ -1176,6 +1176,10 @@ process_madvise_behavior_valid(int behavior) + case MADV_COLD: + case MADV_PAGEOUT: + case MADV_WILLNEED: ++#ifdef CONFIG_KSM ++ case MADV_MERGEABLE: ++ case MADV_UNMERGEABLE: ++#endif + return true; + default: + return false; +diff --git a/tools/testing/selftests/vm/.gitignore b/tools/testing/selftests/vm/.gitignore +index 31e5eea2a9b9..1cd8816c055d 100644 +--- a/tools/testing/selftests/vm/.gitignore ++++ b/tools/testing/selftests/vm/.gitignore +@@ -34,3 +34,4 @@ local_config.* + soft-dirty + split_huge_page_test + ksm_tests ++test-ksm-auto +diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile +index d9fa6a9ea584..002d9482c37f 100644 +--- a/tools/testing/selftests/vm/Makefile ++++ b/tools/testing/selftests/vm/Makefile +@@ -54,6 +54,7 @@ TEST_GEN_FILES += userfaultfd + TEST_GEN_PROGS += soft-dirty + TEST_GEN_PROGS += split_huge_page_test + TEST_GEN_FILES += ksm_tests ++TEST_GEN_FILES += test-ksm-auto + + ifeq ($(MACHINE),x86_64) + CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32) +diff --git a/tools/testing/selftests/vm/test-ksm-auto.c b/tools/testing/selftests/vm/test-ksm-auto.c +new file mode 100644 +index 000000000000..0d71593e008e +--- /dev/null ++++ b/tools/testing/selftests/vm/test-ksm-auto.c +@@ -0,0 +1,273 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define KSM_CLEAR_MODE "2\n" ++#define KSM_NORMAL_MODE "1\n" ++#define KSM_AUTO_MODE "8\n" ++ ++#define PAGESIZE (4*1024) ++/* Don't change the value, it will afffect the result */ ++#define TOTAL_MADVISE_SIZE (300*1024*1024) ++ ++char *ksm_run_file = "/sys/kernel/mm/ksm/run"; ++char *ksm_auto_threshold_file = "/sys/kernel/mm/ksm/auto_threshold"; ++char *ksm_pages_volatile_file = "/sys/kernel/mm/ksm/pages_volatile"; ++char *ksm_pages_sharing_file = "/sys/kernel/mm/ksm/pages_sharing"; ++ ++#define SHAPE_FULL 1 ++#define SHAPE_SPARSE 2 ++/* They are related to the shape of memory */ ++int final_pages[3] = {0, 76500, 42}; ++ ++static char *mmap_and_madvise(long long size, int advise) ++{ ++ char *ptr; ++ int err; ++ ++ err = 0; ++ ++ ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); ++ if (!ptr) ++ return NULL; ++ ++ err = madvise(ptr, size, advise); ++ if (err) { ++ perror("Madvise failed\n"); ++ free(ptr); ++ return NULL; ++ } ++ ++ return ptr; ++} ++ ++void make_samepage_ares(char *ptr, int size, int shape_type) ++{ ++ int i, j; ++ char rnd_num; ++ ++ switch (shape_type) { ++ case SHAPE_FULL: ++ for (i = 0; i < (size / PAGESIZE); i++) ++ memset(ptr + (i * PAGESIZE), 0x1, PAGESIZE); ++ break; ++ case SHAPE_SPARSE: ++ /* Make pages different */ ++ j = 0; ++ for (i = 1; i < (size / PAGESIZE); i++) { ++ ptr[i * PAGESIZE + (j%PAGESIZE)] = j%127 + 1; ++ j++; ++ } ++ for (i = 0; i < (size / PAGESIZE); i += 1800) ++ memset(ptr + (i * PAGESIZE), -1, PAGESIZE); ++ } ++ ++ return; ++} ++ ++int read_file(char *file, char *buffer, int buf_len) ++{ ++ FILE *fp; ++ size_t result; ++ long lSize; ++ ++ fp = fopen(file, "r"); ++ if (!fp) ++ return -1; ++ ++ fseek(fp, 0, SEEK_END); ++ lSize = ftell(fp); ++ rewind(fp); ++ ++ memset(buffer, 0, buf_len); ++ result = fread(buffer, 1, buf_len, fp); ++ if (result == 0) ++ return -1; ++ ++ fclose(fp); ++ ++ return 0; ++} ++ ++int write_file(char *file, const char *buffer, int len) ++{ ++ FILE *fp; ++ size_t result; ++ ++ fp = fopen(file, "w+"); ++ if (!fp) ++ return -1; ++ ++ result = fwrite(buffer, len, 1, fp); ++ if (result == 0) ++ return -1; ++ ++ fclose(fp); ++ ++ return 0; ++} ++ ++static inline void get_orig_info(int *run, int *auto_threshold) ++{ ++ char buffer[50]; ++ ++ /* Read the original state of ksm/run */ ++ if (read_file(ksm_run_file, buffer, sizeof(buffer))) { ++ printf("read file %s failed\n", ksm_run_file); ++ exit(1); ++ } ++ *run = atoi(buffer); ++ ++ if (read_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { ++ printf("read file: %s failed\n", ksm_auto_threshold_file); ++ exit(1); ++ } ++ *auto_threshold = atoi(buffer); ++} ++ ++static inline void restore_orig_state(int run, int auto_threshold) ++{ ++ char buffer[50]; ++ ++ /* restore the original state */ ++ memset(buffer, 0, sizeof(buffer)); ++ snprintf(buffer, sizeof(buffer) - 1, "%d\n", run); ++ if (write_file(ksm_run_file, buffer, sizeof(buffer))) { ++ printf("write file %s failed\n", ksm_run_file); ++ exit(1); ++ } ++ ++ memset(buffer, 0, sizeof(buffer)); ++ snprintf(buffer, sizeof(buffer) - 1, "%d\n", auto_threshold); ++ if (write_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { ++ printf("write file %s failed\n", ksm_run_file); ++ exit(1); ++ } ++} ++ ++void set_ksmd_run_mode(char *mode) ++{ ++ if (write_file(ksm_run_file, mode, 2)) { ++ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); ++ exit(1); ++ } ++} ++ ++static inline void wait_ksmpages_converged(int final_pages) ++{ ++ int pages_sharing; ++ char buffer[50]; ++ ++ for (;;) { ++ if (read_file(ksm_pages_sharing_file, buffer, sizeof(buffer))) { ++ printf("read file %s failed\n", ksm_pages_sharing_file); ++ exit(1); ++ } ++ ++ pages_sharing = atoi(buffer); ++ if (pages_sharing >= final_pages) ++ break; ++ } ++} ++ ++void print_shape(int shape_type) ++{ ++ switch (shape_type) { ++ case SHAPE_FULL: ++ printf("Now the shape of memory area is full-samepages:\n"); ++ printf("[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n\n"); ++ break; ++ case SHAPE_SPARSE: ++ printf("Now the shape of memory area is sparse-samepages:\n"); ++ printf("[xx] [xx] [xx] \n\n"); ++ break; ++ } ++} ++ ++void print_ksmd_cpu_comsuption(void) ++{ ++ system("(ps x| grep \"ksmd\" | grep -v grep | awk \'{print $1}\' |" ++ " xargs -i cat /proc/{}/stat) | awk \'{print \"ksm current " ++ "cpu total slice: \" $14+$15+$16+$17}\'"); ++} ++ ++void test_ksmd_performance(char *madvise_area, int shape_type) ++{ ++ struct timeval tv_start, tv_end; ++ ++ make_samepage_ares(madvise_area, TOTAL_MADVISE_SIZE, shape_type); ++ print_shape(shape_type); ++ ++ /********* Start to time ksmd's normal-run mode **********/ ++ printf("Start to test normal-run ksmd...\n"); ++ ++ print_ksmd_cpu_comsuption(); ++ ++ set_ksmd_run_mode(KSM_CLEAR_MODE); ++ set_ksmd_run_mode(KSM_NORMAL_MODE); ++ ++ gettimeofday(&tv_start, NULL); ++ ++ wait_ksmpages_converged(final_pages[shape_type]); ++ ++ gettimeofday(&tv_end, NULL); ++ printf("ksm normal-run's merging time: %lf seconds\n", ++ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - ++ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); ++ ++ /******* Start to time ksmd's auto-run mode **********/ ++ print_ksmd_cpu_comsuption(); ++ ++ printf("Start to test auto-run ksmd...\n"); ++ set_ksmd_run_mode(KSM_CLEAR_MODE); ++ set_ksmd_run_mode(KSM_AUTO_MODE); ++ if (write_file(ksm_auto_threshold_file, "99\n", 2)) ++ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); ++ ++ gettimeofday(&tv_start, NULL); ++ ++ wait_ksmpages_converged(shape_type); ++ ++ gettimeofday(&tv_end, NULL); ++ printf("ksm auto-run's merging time: %lf seconds\n", ++ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - ++ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); ++ ++ print_ksmd_cpu_comsuption(); ++} ++ ++int main(int argc, char **argv) ++{ ++ char *madvise_area; ++ int orig_run, orig_auto_threshold; ++ ++ /* Get the original state of ksm */ ++ get_orig_info(&orig_run, &orig_auto_threshold); ++ printf("Now we mmap 300MB anouymous memory for testing.\n" ++ "There are two type of TEST which have different shape of\n" ++ "samepage areas.\n" ++ "Note: the test requires no other MERGEABLE-madvised vm areas\n" ++ "in system than the areas our testing process allocs.\n"); ++ madvise_area = mmap_and_madvise(TOTAL_MADVISE_SIZE, MADV_MERGEABLE); ++ if (!madvise_area) { ++ printf("madvise failed\n"); ++ exit(1); ++ } ++ ++ printf("\n****************** TEST 1 ******************\n"); ++ test_ksmd_performance(madvise_area, SHAPE_FULL); ++ printf("\n****************** TEST 2 ******************\n"); ++ test_ksmd_performance(madvise_area, SHAPE_SPARSE); ++ ++ /* Restore the original state */ ++ restore_orig_state(orig_run, orig_auto_threshold); ++ ++ return 0; ++} +-- +2.38.0 + diff --git a/6.0/0017-THP-Shrinker.patch b/6.0/0017-THP-Shrinker.patch new file mode 100644 index 00000000..46102d5c --- /dev/null +++ b/6.0/0017-THP-Shrinker.patch @@ -0,0 +1,1189 @@ +From 7028e64abc9ada4f0bc9cf636a2abc7810c189c4 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Tue, 11 Oct 2022 23:12:15 +0200 +Subject: [PATCH 17/17] THP Shrinker + +https://lore.kernel.org/lkml/cover.1664323311.git.alexlzhu@fb.com/ + +Transparent Hugepages use a larger page size of 2MB in comparison to +normal sized pages that are 4kb. A larger page size allows for fewer TLB +cache misses and thus more efficient use of the CPU. Using a larger page +size also results in more memory waste, which can hurt performance in some +use cases. THPs are currently enabled in the Linux Kernel by applications +in limited virtual address ranges via the madvise system call. The THP +shrinker tries to find a balance between increased use of THPs, and +increased use of memory. It shrinks the size of memory by removing the +underutilized THPs that are identified by the thp_utilization scanner. + +In our experiments we have noticed that the least utilized THPs are almost +entirely unutilized. + +Sample Output: + +Utilized[0-50]: 1331 680884 +Utilized[51-101]: 9 3983 +Utilized[102-152]: 3 1187 +Utilized[153-203]: 0 0 +Utilized[204-255]: 2 539 +Utilized[256-306]: 5 1135 +Utilized[307-357]: 1 192 +Utilized[358-408]: 0 0 +Utilized[409-459]: 1 57 +Utilized[460-512]: 400 13 +Last Scan Time: 223.98s +Last Scan Duration: 70.65s + +Above is a sample obtained from one of our test machines when THP is always +enabled. Of the 1331 THPs in this thp_utilization sample that have from +0-50 utilized subpages, we see that there are 680884 free pages. This +comes out to 680884 / (512 * 1331) = 99.91% zero pages in the least +utilized bucket. This represents 680884 * 4KB = 2.7GB memory waste. + +Also note that the vast majority of pages are either in the least utilized +[0-50] or most utilized [460-512] buckets. The least utilized THPs are +responsible for almost all of the memory waste when THP is always +enabled. Thus by clearing out THPs in the lowest utilization bucket +we extract most of the improvement in CPU efficiency. We have seen +similar results on our production hosts. + +This patchset introduces the THP shrinker we have developed to identify +and split the least utilized THPs. It includes the thp_utilization +changes that groups anonymous THPs into buckets, the split_huge_page() +changes that identify and zap zero 4KB pages within THPs and the shrinker +changes. It should be noted that the split_huge_page() changes are based +off previous work done by Yu Zhao. + +In the future, we intend to allow additional tuning to the shrinker +based on workload depending on CPU/IO/Memory pressure and the +amount of anonymous memory. The long term goal is to eventually always +enable THP for all applications and deprecate madvise entirely. + +In production we thus far have observed 2-3% reduction in overall cpu +usage on stateless web servers when THP is always enabled. + +Alexander Zhu (3): + mm: add thp_utilization metrics to debugfs + mm: changes to split_huge_page() to free zero filled tail pages + mm: THP low utilization shrinker + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/mm/transhuge.rst | 9 + + include/linux/huge_mm.h | 10 + + include/linux/list_lru.h | 24 ++ + include/linux/mm_types.h | 5 + + include/linux/rmap.h | 2 +- + include/linux/vm_event_item.h | 3 + + mm/huge_memory.c | 342 +++++++++++++++++- + mm/list_lru.c | 49 +++ + mm/migrate.c | 72 +++- + mm/migrate_device.c | 4 +- + mm/page_alloc.c | 6 + + mm/vmstat.c | 3 + + .../selftests/vm/split_huge_page_test.c | 113 +++++- + tools/testing/selftests/vm/vm_util.c | 23 ++ + tools/testing/selftests/vm/vm_util.h | 1 + + 15 files changed, 648 insertions(+), 18 deletions(-) + +diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst +index c9c37f16eef8..d883ff9fddc7 100644 +--- a/Documentation/admin-guide/mm/transhuge.rst ++++ b/Documentation/admin-guide/mm/transhuge.rst +@@ -297,6 +297,15 @@ To identify what applications are mapping file transparent huge pages, it + is necessary to read ``/proc/PID/smaps`` and count the FileHugeMapped fields + for each mapping. + ++The utilization of transparent hugepages can be viewed by reading ++``/sys/kernel/debug/thp_utilization``. The utilization of a THP is defined ++as the ratio of non zero filled 4kb pages to the total number of pages in a ++THP. The buckets are labelled by the range of total utilized 4kb pages with ++one line per utilization bucket. Each line contains the total number of ++THPs in that bucket and the total number of zero filled 4kb pages summed ++over all THPs in that bucket. The last two lines show the timestamp and ++duration respectively of the most recent scan over all of physical memory. ++ + Note that reading the smaps file is expensive and reading it + frequently will incur overhead. + +diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h +index 768e5261fdae..d6e55bf67625 100644 +--- a/include/linux/huge_mm.h ++++ b/include/linux/huge_mm.h +@@ -179,6 +179,9 @@ bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags); + ++int thp_number_utilized_pages(struct page *page); ++int thp_utilization_bucket(int num_utilized_pages); ++ + void prep_transhuge_page(struct page *page); + void free_transhuge_page(struct page *page); + +@@ -190,6 +193,8 @@ static inline int split_huge_page(struct page *page) + } + void deferred_split_huge_page(struct page *page); + ++void add_underutilized_thp(struct page *page); ++ + void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long address, bool freeze, struct folio *folio); + +@@ -300,6 +305,11 @@ static inline struct list_head *page_deferred_list(struct page *page) + return &page[2].deferred_list; + } + ++static inline struct list_head *page_underutilized_thp_list(struct page *page) ++{ ++ return &page[3].underutilized_thp_list; ++} ++ + #else /* CONFIG_TRANSPARENT_HUGEPAGE */ + #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) + #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) +diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h +index b35968ee9fb5..c2cf146ea880 100644 +--- a/include/linux/list_lru.h ++++ b/include/linux/list_lru.h +@@ -89,6 +89,18 @@ void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *paren + */ + bool list_lru_add(struct list_lru *lru, struct list_head *item); + ++/** ++ * list_lru_add_page: add an element to the lru list's tail ++ * @list_lru: the lru pointer ++ * @page: the page containing the item ++ * @item: the item to be deleted. ++ * ++ * This function works the same as list_lru_add in terms of list ++ * manipulation. Used for non slab objects contained in the page. ++ * ++ * Return value: true if the list was updated, false otherwise ++ */ ++bool list_lru_add_page(struct list_lru *lru, struct page *page, struct list_head *item); + /** + * list_lru_del: delete an element to the lru list + * @list_lru: the lru pointer +@@ -102,6 +114,18 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item); + */ + bool list_lru_del(struct list_lru *lru, struct list_head *item); + ++/** ++ * list_lru_del_page: delete an element to the lru list ++ * @list_lru: the lru pointer ++ * @page: the page containing the item ++ * @item: the item to be deleted. ++ * ++ * This function works the same as list_lru_del in terms of list ++ * manipulation. Used for non slab objects contained in the page. ++ * ++ * Return value: true if the list was updated, false otherwise ++ */ ++bool list_lru_del_page(struct list_lru *lru, struct page *page, struct list_head *item); + /** + * list_lru_count_one: return the number of objects currently held by @lru + * @lru: the lru pointer. +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index 41935c55a438..99f6a749bab7 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -152,6 +152,11 @@ struct page { + /* For both global and memcg */ + struct list_head deferred_list; + }; ++ struct { /* Third tail page of compound page */ ++ unsigned long _compound_pad_3; /* compound_head */ ++ unsigned long _compound_pad_4; ++ struct list_head underutilized_thp_list; ++ }; + struct { /* Page table pages */ + unsigned long _pt_pad_1; /* compound_head */ + pgtable_t pmd_huge_pte; /* protected by page->ptl */ +diff --git a/include/linux/rmap.h b/include/linux/rmap.h +index b89b4b86951f..f7d5d5639dea 100644 +--- a/include/linux/rmap.h ++++ b/include/linux/rmap.h +@@ -372,7 +372,7 @@ int folio_mkclean(struct folio *); + int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, + struct vm_area_struct *vma); + +-void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked); ++void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked, bool unmap_clean); + + int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma); + +diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h +index 3518dba1e02f..3618b10ddec9 100644 +--- a/include/linux/vm_event_item.h ++++ b/include/linux/vm_event_item.h +@@ -111,6 +111,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD + THP_SPLIT_PUD, + #endif ++ THP_SPLIT_FREE, ++ THP_SPLIT_UNMAP, ++ THP_SPLIT_REMAP_READONLY_ZERO_PAGE, + THP_ZERO_PAGE_ALLOC, + THP_ZERO_PAGE_ALLOC_FAILED, + THP_SWPOUT, +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 2a98e4125f24..8ed70e1fea3e 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -46,6 +46,16 @@ + #define CREATE_TRACE_POINTS + #include + ++/* ++ * The number of utilization buckets THPs will be grouped in ++ * under /sys/kernel/debug/thp_utilization. ++ */ ++#define THP_UTIL_BUCKET_NR 10 ++/* ++ * The number of PFNs (and hence hugepages) to scan through on each periodic ++ * run of the scanner that generates /sys/kernel/debug/thp_utilization. ++ */ ++#define THP_UTIL_SCAN_SIZE 256 + /* + * By default, transparent hugepage support is disabled in order to avoid + * risking an increased memory footprint for applications that are not +@@ -71,6 +81,27 @@ static atomic_t huge_zero_refcount; + struct page *huge_zero_page __read_mostly; + unsigned long huge_zero_pfn __read_mostly = ~0UL; + ++struct list_lru huge_low_util_page_lru; ++ ++static void thp_utilization_workfn(struct work_struct *work); ++static DECLARE_DELAYED_WORK(thp_utilization_work, thp_utilization_workfn); ++ ++struct thp_scan_info_bucket { ++ int nr_thps; ++ int nr_zero_pages; ++}; ++ ++struct thp_scan_info { ++ struct thp_scan_info_bucket buckets[THP_UTIL_BUCKET_NR]; ++ struct zone *scan_zone; ++ struct timespec64 last_scan_duration; ++ struct timespec64 last_scan_time; ++ unsigned long pfn; ++}; ++ ++static struct thp_scan_info thp_scan_debugfs; ++static struct thp_scan_info thp_scan; ++ + bool hugepage_vma_check(struct vm_area_struct *vma, + unsigned long vm_flags, + bool smaps, bool in_pf) +@@ -236,6 +267,51 @@ static struct shrinker huge_zero_page_shrinker = { + .seeks = DEFAULT_SEEKS, + }; + ++static enum lru_status low_util_free_page(struct list_head *item, ++ struct list_lru_one *lru, ++ spinlock_t *lock, ++ void *cb_arg) ++{ ++ int bucket, num_utilized_pages; ++ struct page *head = compound_head(list_entry(item, ++ struct page, ++ underutilized_thp_list)); ++ ++ if (get_page_unless_zero(head)) { ++ lock_page(head); ++ list_lru_isolate(lru, item); ++ num_utilized_pages = thp_number_utilized_pages(head); ++ bucket = thp_utilization_bucket(num_utilized_pages); ++ if (bucket < THP_UTIL_BUCKET_NR - 1) ++ split_huge_page(head); ++ unlock_page(head); ++ put_page(head); ++ } ++ ++ return LRU_REMOVED_RETRY; ++} ++ ++static unsigned long shrink_huge_low_util_page_count(struct shrinker *shrink, ++ struct shrink_control *sc) ++{ ++ return HPAGE_PMD_NR * list_lru_shrink_count(&huge_low_util_page_lru, sc); ++} ++ ++static unsigned long shrink_huge_low_util_page_scan(struct shrinker *shrink, ++ struct shrink_control *sc) ++{ ++ return HPAGE_PMD_NR * list_lru_shrink_walk(&huge_low_util_page_lru, ++ sc, low_util_free_page, NULL); ++} ++ ++static struct shrinker huge_low_util_page_shrinker = { ++ .count_objects = shrink_huge_low_util_page_count, ++ .scan_objects = shrink_huge_low_util_page_scan, ++ .seeks = DEFAULT_SEEKS, ++ .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE | ++ SHRINKER_NONSLAB, ++}; ++ + #ifdef CONFIG_SYSFS + static ssize_t enabled_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +@@ -487,13 +563,19 @@ static int __init hugepage_init(void) + if (err) + goto err_slab; + ++ schedule_delayed_work(&thp_utilization_work, HZ); ++ err = register_shrinker(&huge_low_util_page_shrinker, "thp-low-util"); ++ if (err) ++ goto err_low_util_shrinker; + err = register_shrinker(&huge_zero_page_shrinker, "thp-zero"); + if (err) + goto err_hzp_shrinker; + err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split"); + if (err) + goto err_split_shrinker; +- ++ err = list_lru_init_memcg(&huge_low_util_page_lru, &huge_low_util_page_shrinker); ++ if (err) ++ goto err_low_util_list_lru; + /* + * By default disable transparent hugepages on smaller systems, + * where the extra memory used could hurt more than TLB overhead +@@ -509,11 +591,16 @@ static int __init hugepage_init(void) + goto err_khugepaged; + + return 0; ++ + err_khugepaged: ++ list_lru_destroy(&huge_low_util_page_lru); ++err_low_util_list_lru: + unregister_shrinker(&deferred_split_shrinker); + err_split_shrinker: + unregister_shrinker(&huge_zero_page_shrinker); + err_hzp_shrinker: ++ unregister_shrinker(&huge_low_util_page_shrinker); ++err_low_util_shrinker: + khugepaged_destroy(); + err_slab: + hugepage_exit_sysfs(hugepage_kobj); +@@ -588,6 +675,7 @@ void prep_transhuge_page(struct page *page) + */ + + INIT_LIST_HEAD(page_deferred_list(page)); ++ INIT_LIST_HEAD(page_underutilized_thp_list(page)); + set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); + } + +@@ -601,6 +689,11 @@ static inline bool is_transparent_hugepage(struct page *page) + page[1].compound_dtor == TRANSHUGE_PAGE_DTOR; + } + ++static inline bool is_anon_transparent_hugepage(struct page *page) ++{ ++ return PageAnon(page) && is_transparent_hugepage(page); ++} ++ + static unsigned long __thp_get_unmapped_area(struct file *filp, + unsigned long addr, unsigned long len, + loff_t off, unsigned long flags, unsigned long size) +@@ -651,6 +744,49 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, + } + EXPORT_SYMBOL_GPL(thp_get_unmapped_area); + ++int thp_number_utilized_pages(struct page *page) ++{ ++ struct folio *folio; ++ unsigned long page_offset, value; ++ int thp_nr_utilized_pages = HPAGE_PMD_NR; ++ int step_size = sizeof(unsigned long); ++ bool is_all_zeroes; ++ void *kaddr; ++ int i; ++ ++ if (!page || !is_anon_transparent_hugepage(page)) ++ return -1; ++ ++ folio = page_folio(page); ++ for (i = 0; i < folio_nr_pages(folio); i++) { ++ kaddr = kmap_local_folio(folio, i); ++ is_all_zeroes = true; ++ for (page_offset = 0; page_offset < PAGE_SIZE; page_offset += step_size) { ++ value = *(unsigned long *)(kaddr + page_offset); ++ if (value != 0) { ++ is_all_zeroes = false; ++ break; ++ } ++ } ++ if (is_all_zeroes) ++ thp_nr_utilized_pages--; ++ ++ kunmap_local(kaddr); ++ } ++ return thp_nr_utilized_pages; ++} ++ ++int thp_utilization_bucket(int num_utilized_pages) ++{ ++ int bucket; ++ ++ if (num_utilized_pages < 0 || num_utilized_pages > HPAGE_PMD_NR) ++ return -1; ++ /* Group THPs into utilization buckets */ ++ bucket = num_utilized_pages * THP_UTIL_BUCKET_NR / HPAGE_PMD_NR; ++ return min(bucket, THP_UTIL_BUCKET_NR - 1); ++} ++ + static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, + struct page *page, gfp_t gfp) + { +@@ -2365,7 +2501,7 @@ static void unmap_page(struct page *page) + try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); + } + +-static void remap_page(struct folio *folio, unsigned long nr) ++static void remap_page(struct folio *folio, unsigned long nr, bool unmap_clean) + { + int i = 0; + +@@ -2373,7 +2509,7 @@ static void remap_page(struct folio *folio, unsigned long nr) + if (!folio_test_anon(folio)) + return; + for (;;) { +- remove_migration_ptes(folio, folio, true); ++ remove_migration_ptes(folio, folio, true, unmap_clean); + i += folio_nr_pages(folio); + if (i >= nr) + break; +@@ -2443,8 +2579,7 @@ static void __split_huge_page_tail(struct page *head, int tail, + LRU_GEN_MASK | LRU_REFS_MASK)); + + /* ->mapping in first tail page is compound_mapcount */ +- VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, +- page_tail); ++ VM_BUG_ON_PAGE(tail > 3 && page_tail->mapping != TAIL_MAPPING, page_tail); + page_tail->mapping = head->mapping; + page_tail->index = head->index + tail; + if (!PageSwapCache(page_tail)) +@@ -2489,6 +2624,8 @@ static void __split_huge_page(struct page *page, struct list_head *list, + struct address_space *swap_cache = NULL; + unsigned long offset = 0; + unsigned int nr = thp_nr_pages(head); ++ LIST_HEAD(pages_to_free); ++ int nr_pages_to_free = 0; + int i; + + /* complete memcg works before add pages to LRU */ +@@ -2551,7 +2688,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, + } + local_irq_enable(); + +- remap_page(folio, nr); ++ remap_page(folio, nr, PageAnon(head)); + + if (PageSwapCache(head)) { + swp_entry_t entry = { .val = page_private(head) }; +@@ -2565,6 +2702,33 @@ static void __split_huge_page(struct page *page, struct list_head *list, + continue; + unlock_page(subpage); + ++ /* ++ * If a tail page has only two references left, one inherited ++ * from the isolation of its head and the other from ++ * lru_add_page_tail() which we are about to drop, it means this ++ * tail page was concurrently zapped. Then we can safely free it ++ * and save page reclaim or migration the trouble of trying it. ++ */ ++ if (list && page_ref_freeze(subpage, 2)) { ++ VM_BUG_ON_PAGE(PageLRU(subpage), subpage); ++ VM_BUG_ON_PAGE(PageCompound(subpage), subpage); ++ VM_BUG_ON_PAGE(page_mapped(subpage), subpage); ++ ++ ClearPageActive(subpage); ++ ClearPageUnevictable(subpage); ++ list_move(&subpage->lru, &pages_to_free); ++ nr_pages_to_free++; ++ continue; ++ } ++ /* ++ * If a tail page has only one reference left, it will be freed ++ * by the call to free_page_and_swap_cache below. Since zero ++ * subpages are no longer remapped, there will only be one ++ * reference left in cases outside of reclaim or migration. ++ */ ++ if (page_ref_count(subpage) == 1) ++ nr_pages_to_free++; ++ + /* + * Subpages may be freed if there wasn't any mapping + * like if add_to_swap() is running on a lru page that +@@ -2574,6 +2738,13 @@ static void __split_huge_page(struct page *page, struct list_head *list, + */ + free_page_and_swap_cache(subpage); + } ++ ++ if (!nr_pages_to_free) ++ return; ++ ++ mem_cgroup_uncharge_list(&pages_to_free); ++ free_unref_page_list(&pages_to_free); ++ count_vm_events(THP_SPLIT_FREE, nr_pages_to_free); + } + + /* Racy check whether the huge page can be split */ +@@ -2616,6 +2787,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + struct folio *folio = page_folio(page); + struct page *head = &folio->page; + struct deferred_split *ds_queue = get_deferred_split_queue(head); ++ struct list_head *underutilized_thp_list = page_underutilized_thp_list(head); + XA_STATE(xas, &head->mapping->i_pages, head->index); + struct anon_vma *anon_vma = NULL; + struct address_space *mapping = NULL; +@@ -2714,6 +2886,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + list_del(page_deferred_list(head)); + } + spin_unlock(&ds_queue->split_queue_lock); ++ if (!list_empty(underutilized_thp_list)) ++ list_lru_del_page(&huge_low_util_page_lru, head, underutilized_thp_list); + if (mapping) { + int nr = thp_nr_pages(head); + +@@ -2736,7 +2910,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + if (mapping) + xas_unlock(&xas); + local_irq_enable(); +- remap_page(folio, folio_nr_pages(folio)); ++ remap_page(folio, folio_nr_pages(folio), false); + ret = -EBUSY; + } + +@@ -2756,6 +2930,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) + void free_transhuge_page(struct page *page) + { + struct deferred_split *ds_queue = get_deferred_split_queue(page); ++ struct list_head *underutilized_thp_list = page_underutilized_thp_list(page); + unsigned long flags; + + spin_lock_irqsave(&ds_queue->split_queue_lock, flags); +@@ -2764,6 +2939,12 @@ void free_transhuge_page(struct page *page) + list_del(page_deferred_list(page)); + } + spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); ++ if (!list_empty(underutilized_thp_list)) ++ list_lru_del_page(&huge_low_util_page_lru, page, underutilized_thp_list); ++ ++ if (PageLRU(page)) ++ __clear_page_lru_flags(page); ++ + free_compound_page(page); + } + +@@ -2804,6 +2985,26 @@ void deferred_split_huge_page(struct page *page) + spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); + } + ++void add_underutilized_thp(struct page *page) ++{ ++ VM_BUG_ON_PAGE(!PageTransHuge(page), page); ++ ++ if (PageSwapCache(page)) ++ return; ++ ++ /* ++ * Need to take a reference on the page to prevent the page from getting free'd from ++ * under us while we are adding the THP to the shrinker. ++ */ ++ if (!get_page_unless_zero(page)) ++ return; ++ ++ if (!is_huge_zero_page(page) && is_anon_transparent_hugepage(page)) ++ list_lru_add_page(&huge_low_util_page_lru, page, page_underutilized_thp_list(page)); ++ ++ put_page(page); ++} ++ + static unsigned long deferred_split_count(struct shrinker *shrink, + struct shrink_control *sc) + { +@@ -3158,6 +3359,42 @@ static int __init split_huge_pages_debugfs(void) + return 0; + } + late_initcall(split_huge_pages_debugfs); ++ ++static int thp_utilization_show(struct seq_file *seqf, void *pos) ++{ ++ int i; ++ int start; ++ int end; ++ ++ for (i = 0; i < THP_UTIL_BUCKET_NR; i++) { ++ start = i * HPAGE_PMD_NR / THP_UTIL_BUCKET_NR; ++ end = (i + 1 == THP_UTIL_BUCKET_NR) ++ ? HPAGE_PMD_NR ++ : ((i + 1) * HPAGE_PMD_NR / THP_UTIL_BUCKET_NR - 1); ++ /* The last bucket will need to contain 100 */ ++ seq_printf(seqf, "Utilized[%d-%d]: %d %d\n", start, end, ++ thp_scan_debugfs.buckets[i].nr_thps, ++ thp_scan_debugfs.buckets[i].nr_zero_pages); ++ } ++ seq_printf(seqf, "Last Scan Time: %lu.%02lus\n", ++ (unsigned long)thp_scan_debugfs.last_scan_time.tv_sec, ++ (thp_scan_debugfs.last_scan_time.tv_nsec / (NSEC_PER_SEC / 100))); ++ ++ seq_printf(seqf, "Last Scan Duration: %lu.%02lus\n", ++ (unsigned long)thp_scan_debugfs.last_scan_duration.tv_sec, ++ (thp_scan_debugfs.last_scan_duration.tv_nsec / (NSEC_PER_SEC / 100))); ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(thp_utilization); ++ ++static int __init thp_utilization_debugfs(void) ++{ ++ debugfs_create_file("thp_utilization", 0200, NULL, NULL, ++ &thp_utilization_fops); ++ return 0; ++} ++late_initcall(thp_utilization_debugfs); + #endif + + #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +@@ -3243,3 +3480,94 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) + trace_remove_migration_pmd(address, pmd_val(pmde)); + } + #endif ++ ++static void thp_scan_next_zone(void) ++{ ++ struct timespec64 current_time; ++ int i; ++ bool update_debugfs; ++ /* ++ * THP utilization worker thread has reached the end ++ * of the memory zone. Proceed to the next zone. ++ */ ++ thp_scan.scan_zone = next_zone(thp_scan.scan_zone); ++ update_debugfs = !thp_scan.scan_zone; ++ thp_scan.scan_zone = update_debugfs ? (first_online_pgdat())->node_zones ++ : thp_scan.scan_zone; ++ thp_scan.pfn = (thp_scan.scan_zone->zone_start_pfn + HPAGE_PMD_NR - 1) ++ & ~(HPAGE_PMD_SIZE - 1); ++ if (!update_debugfs) ++ return; ++ /* ++ * If the worker has scanned through all of physical ++ * memory. Then update information displayed in /sys/kernel/debug/thp_utilization ++ */ ++ ktime_get_ts64(¤t_time); ++ thp_scan_debugfs.last_scan_duration = timespec64_sub(current_time, ++ thp_scan_debugfs.last_scan_time); ++ thp_scan_debugfs.last_scan_time = current_time; ++ ++ for (i = 0; i < THP_UTIL_BUCKET_NR; i++) { ++ thp_scan_debugfs.buckets[i].nr_thps = thp_scan.buckets[i].nr_thps; ++ thp_scan_debugfs.buckets[i].nr_zero_pages = thp_scan.buckets[i].nr_zero_pages; ++ thp_scan.buckets[i].nr_thps = 0; ++ thp_scan.buckets[i].nr_zero_pages = 0; ++ } ++} ++ ++static void thp_util_scan(unsigned long pfn_end) ++{ ++ struct page *page = NULL; ++ int bucket, num_utilized_pages, current_pfn; ++ int i; ++ /* ++ * Scan through each memory zone in chunks of THP_UTIL_SCAN_SIZE ++ * PFNs every second looking for anonymous THPs. ++ */ ++ for (i = 0; i < THP_UTIL_SCAN_SIZE; i++) { ++ current_pfn = thp_scan.pfn; ++ thp_scan.pfn += HPAGE_PMD_NR; ++ if (current_pfn >= pfn_end) ++ return; ++ ++ if (!pfn_valid(current_pfn)) ++ continue; ++ ++ page = pfn_to_page(current_pfn); ++ num_utilized_pages = thp_number_utilized_pages(page); ++ bucket = thp_utilization_bucket(num_utilized_pages); ++ if (bucket < 0) ++ continue; ++ ++ if (bucket < THP_UTIL_BUCKET_NR - 1) ++ add_underutilized_thp(page); ++ ++ thp_scan.buckets[bucket].nr_thps++; ++ thp_scan.buckets[bucket].nr_zero_pages += (HPAGE_PMD_NR - num_utilized_pages); ++ } ++} ++ ++static void thp_utilization_workfn(struct work_struct *work) ++{ ++ unsigned long pfn_end; ++ ++ if (!thp_scan.scan_zone) ++ thp_scan.scan_zone = (first_online_pgdat())->node_zones; ++ /* ++ * Worker function that scans through all of physical memory ++ * for anonymous THPs. ++ */ ++ pfn_end = (thp_scan.scan_zone->zone_start_pfn + ++ thp_scan.scan_zone->spanned_pages + HPAGE_PMD_NR - 1) ++ & ~(HPAGE_PMD_SIZE - 1); ++ /* If we have reached the end of the zone or end of physical memory ++ * move on to the next zone. Otherwise, scan the next PFNs in the ++ * current zone. ++ */ ++ if (!populated_zone(thp_scan.scan_zone) || thp_scan.pfn >= pfn_end) ++ thp_scan_next_zone(); ++ else ++ thp_util_scan(pfn_end); ++ ++ schedule_delayed_work(&thp_utilization_work, HZ); ++} +diff --git a/mm/list_lru.c b/mm/list_lru.c +index a05e5bef3b40..7e8b324cc840 100644 +--- a/mm/list_lru.c ++++ b/mm/list_lru.c +@@ -140,6 +140,32 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item) + } + EXPORT_SYMBOL_GPL(list_lru_add); + ++bool list_lru_add_page(struct list_lru *lru, struct page *page, struct list_head *item) ++{ ++ int nid = page_to_nid(page); ++ struct list_lru_node *nlru = &lru->node[nid]; ++ struct list_lru_one *l; ++ struct mem_cgroup *memcg; ++ ++ spin_lock(&nlru->lock); ++ if (list_empty(item)) { ++ memcg = page_memcg(page); ++ memcg_list_lru_alloc(memcg, lru, GFP_KERNEL); ++ l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); ++ list_add_tail(item, &l->list); ++ /* Set shrinker bit if the first element was added */ ++ if (!l->nr_items++) ++ set_shrinker_bit(memcg, nid, ++ lru_shrinker_id(lru)); ++ nlru->nr_items++; ++ spin_unlock(&nlru->lock); ++ return true; ++ } ++ spin_unlock(&nlru->lock); ++ return false; ++} ++EXPORT_SYMBOL_GPL(list_lru_add_page); ++ + bool list_lru_del(struct list_lru *lru, struct list_head *item) + { + int nid = page_to_nid(virt_to_page(item)); +@@ -160,6 +186,29 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item) + } + EXPORT_SYMBOL_GPL(list_lru_del); + ++bool list_lru_del_page(struct list_lru *lru, struct page *page, struct list_head *item) ++{ ++ int nid = page_to_nid(page); ++ struct list_lru_node *nlru = &lru->node[nid]; ++ struct list_lru_one *l; ++ struct mem_cgroup *memcg; ++ ++ spin_lock(&nlru->lock); ++ if (!list_empty(item)) { ++ memcg = page_memcg(page); ++ memcg_list_lru_alloc(memcg, lru, GFP_KERNEL); ++ l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); ++ list_del_init(item); ++ l->nr_items--; ++ nlru->nr_items--; ++ spin_unlock(&nlru->lock); ++ return true; ++ } ++ spin_unlock(&nlru->lock); ++ return false; ++} ++EXPORT_SYMBOL_GPL(list_lru_del_page); ++ + void list_lru_isolate(struct list_lru_one *list, struct list_head *item) + { + list_del_init(item); +diff --git a/mm/migrate.c b/mm/migrate.c +index 55e7718cfe45..57908d680276 100644 +--- a/mm/migrate.c ++++ b/mm/migrate.c +@@ -168,13 +168,62 @@ void putback_movable_pages(struct list_head *l) + } + } + ++static bool try_to_unmap_clean(struct page_vma_mapped_walk *pvmw, struct page *page) ++{ ++ void *addr; ++ bool dirty; ++ pte_t newpte; ++ ++ VM_BUG_ON_PAGE(PageCompound(page), page); ++ VM_BUG_ON_PAGE(!PageAnon(page), page); ++ VM_BUG_ON_PAGE(!PageLocked(page), page); ++ VM_BUG_ON_PAGE(pte_present(*pvmw->pte), page); ++ ++ if (PageMlocked(page) || (pvmw->vma->vm_flags & VM_LOCKED)) ++ return false; ++ ++ /* ++ * The pmd entry mapping the old thp was flushed and the pte mapping ++ * this subpage has been non present. Therefore, this subpage is ++ * inaccessible. We don't need to remap it if it contains only zeros. ++ */ ++ addr = kmap_local_page(page); ++ dirty = memchr_inv(addr, 0, PAGE_SIZE); ++ kunmap_local(addr); ++ ++ if (dirty) ++ return false; ++ ++ pte_clear_not_present_full(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, false); ++ ++ if (userfaultfd_armed(pvmw->vma)) { ++ newpte = pte_mkspecial(pfn_pte(page_to_pfn(ZERO_PAGE(pvmw->address)), ++ pvmw->vma->vm_page_prot)); ++ ptep_clear_flush(pvmw->vma, pvmw->address, pvmw->pte); ++ set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); ++ dec_mm_counter(pvmw->vma->vm_mm, MM_ANONPAGES); ++ count_vm_event(THP_SPLIT_REMAP_READONLY_ZERO_PAGE); ++ return true; ++ } ++ ++ dec_mm_counter(pvmw->vma->vm_mm, mm_counter(page)); ++ count_vm_event(THP_SPLIT_UNMAP); ++ return true; ++} ++ ++struct rmap_walk_arg { ++ struct folio *folio; ++ bool unmap_clean; ++}; ++ + /* + * Restore a potential migration pte to a working pte entry + */ + static bool remove_migration_pte(struct folio *folio, +- struct vm_area_struct *vma, unsigned long addr, void *old) ++ struct vm_area_struct *vma, unsigned long addr, void *arg) + { +- DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION); ++ struct rmap_walk_arg *rmap_walk_arg = arg; ++ DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + + while (page_vma_mapped_walk(&pvmw)) { + rmap_t rmap_flags = RMAP_NONE; +@@ -197,6 +246,8 @@ static bool remove_migration_pte(struct folio *folio, + continue; + } + #endif ++ if (rmap_walk_arg->unmap_clean && try_to_unmap_clean(&pvmw, new)) ++ continue; + + folio_get(folio); + pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); +@@ -268,13 +319,20 @@ static bool remove_migration_pte(struct folio *folio, + * Get rid of all migration entries and replace them by + * references to the indicated page. + */ +-void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked) ++void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked, bool unmap_clean) + { ++ struct rmap_walk_arg rmap_walk_arg = { ++ .folio = src, ++ .unmap_clean = unmap_clean, ++ }; ++ + struct rmap_walk_control rwc = { + .rmap_one = remove_migration_pte, +- .arg = src, ++ .arg = &rmap_walk_arg, + }; + ++ VM_BUG_ON_FOLIO(unmap_clean && src != dst, src); ++ + if (locked) + rmap_walk_locked(dst, &rwc); + else +@@ -850,7 +908,7 @@ static int writeout(struct address_space *mapping, struct folio *folio) + * At this point we know that the migration attempt cannot + * be successful. + */ +- remove_migration_ptes(folio, folio, false); ++ remove_migration_ptes(folio, folio, false, false); + + rc = mapping->a_ops->writepage(&folio->page, &wbc); + +@@ -1109,7 +1167,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, + + if (page_was_mapped) + remove_migration_ptes(folio, +- rc == MIGRATEPAGE_SUCCESS ? dst : folio, false); ++ rc == MIGRATEPAGE_SUCCESS ? dst : folio, false, false); + + out_unlock_both: + unlock_page(newpage); +@@ -1319,7 +1377,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, + + if (page_was_mapped) + remove_migration_ptes(src, +- rc == MIGRATEPAGE_SUCCESS ? dst : src, false); ++ rc == MIGRATEPAGE_SUCCESS ? dst : src, false, false); + + unlock_put_anon: + unlock_page(new_hpage); +diff --git a/mm/migrate_device.c b/mm/migrate_device.c +index dbf6c7a7a7c9..518aacc914c9 100644 +--- a/mm/migrate_device.c ++++ b/mm/migrate_device.c +@@ -413,7 +413,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) + continue; + + folio = page_folio(page); +- remove_migration_ptes(folio, folio, false); ++ remove_migration_ptes(folio, folio, false, false); + + migrate->src[i] = 0; + folio_unlock(folio); +@@ -789,7 +789,7 @@ void migrate_vma_finalize(struct migrate_vma *migrate) + + src = page_folio(page); + dst = page_folio(newpage); +- remove_migration_ptes(src, dst, false); ++ remove_migration_ptes(src, dst, false, false); + folio_unlock(src); + + if (is_zone_device_page(page)) +diff --git a/mm/page_alloc.c b/mm/page_alloc.c +index cc6179d3a7dc..ed2eddb2fbf4 100644 +--- a/mm/page_alloc.c ++++ b/mm/page_alloc.c +@@ -1327,6 +1327,12 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) + * deferred_list.next -- ignore value. + */ + break; ++ case 3: ++ /* ++ * the third tail page: ->mapping is ++ * underutilized_thp_list.next -- ignore value. ++ */ ++ break; + default: + if (page->mapping != TAIL_MAPPING) { + bad_page(page, "corrupted mapping in tail page"); +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 33091a67627e..f6c5d0e97499 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -1369,6 +1369,9 @@ const char * const vmstat_text[] = { + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD + "thp_split_pud", + #endif ++ "thp_split_free", ++ "thp_split_unmap", ++ "thp_split_remap_readonly_zero_page", + "thp_zero_page_alloc", + "thp_zero_page_alloc_failed", + "thp_swpout", +diff --git a/tools/testing/selftests/vm/split_huge_page_test.c b/tools/testing/selftests/vm/split_huge_page_test.c +index 6aa2b8253aed..2c669aadbfd0 100644 +--- a/tools/testing/selftests/vm/split_huge_page_test.c ++++ b/tools/testing/selftests/vm/split_huge_page_test.c +@@ -16,6 +16,9 @@ + #include + #include + #include ++#include /* Definition of SYS_* constants */ ++#include ++#include + #include "vm_util.h" + + uint64_t pagesize; +@@ -88,6 +91,113 @@ static void write_debugfs(const char *fmt, ...) + } + } + ++static char *allocate_zero_filled_hugepage(size_t len) ++{ ++ char *result; ++ size_t i; ++ ++ result = memalign(pmd_pagesize, len); ++ if (!result) { ++ printf("Fail to allocate memory\n"); ++ exit(EXIT_FAILURE); ++ } ++ madvise(result, len, MADV_HUGEPAGE); ++ ++ for (i = 0; i < len; i++) ++ result[i] = (char)0; ++ ++ return result; ++} ++ ++static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, size_t len) ++{ ++ uint64_t thp_size, rss_anon_before, rss_anon_after; ++ size_t i; ++ ++ thp_size = check_huge(one_page); ++ if (!thp_size) { ++ printf("No THP is allocated\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ rss_anon_before = rss_anon(); ++ if (!rss_anon_before) { ++ printf("No RssAnon is allocated before split\n"); ++ exit(EXIT_FAILURE); ++ } ++ /* split all THPs */ ++ write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, ++ (uint64_t)one_page + len); ++ ++ for (i = 0; i < len; i++) ++ if (one_page[i] != (char)0) { ++ printf("%ld byte corrupted\n", i); ++ exit(EXIT_FAILURE); ++ } ++ ++ thp_size = check_huge(one_page); ++ if (thp_size) { ++ printf("Still %ld kB AnonHugePages not split\n", thp_size); ++ exit(EXIT_FAILURE); ++ } ++ ++ rss_anon_after = rss_anon(); ++ if (rss_anon_after >= rss_anon_before) { ++ printf("Incorrect RssAnon value. Before: %ld After: %ld\n", ++ rss_anon_before, rss_anon_after); ++ exit(EXIT_FAILURE); ++ } ++} ++ ++void split_pmd_zero_pages(void) ++{ ++ char *one_page; ++ size_t len = 4 * pmd_pagesize; ++ ++ one_page = allocate_zero_filled_hugepage(len); ++ verify_rss_anon_split_huge_page_all_zeroes(one_page, len); ++ printf("Split zero filled huge pages successful\n"); ++ free(one_page); ++} ++ ++void split_pmd_zero_pages_uffd(void) ++{ ++ char *one_page; ++ size_t len = 4 * pmd_pagesize; ++ long uffd; /* userfaultfd file descriptor */ ++ struct uffdio_api uffdio_api; ++ struct uffdio_register uffdio_register; ++ ++ /* Create and enable userfaultfd object. */ ++ ++ uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); ++ if (uffd == -1) { ++ perror("userfaultfd"); ++ exit(1); ++ } ++ ++ uffdio_api.api = UFFD_API; ++ uffdio_api.features = 0; ++ if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) { ++ perror("ioctl-UFFDIO_API"); ++ exit(1); ++ } ++ ++ one_page = allocate_zero_filled_hugepage(len); ++ ++ uffdio_register.range.start = (unsigned long)one_page; ++ uffdio_register.range.len = len; ++ uffdio_register.mode = UFFDIO_REGISTER_MODE_WP; ++ if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) { ++ perror("ioctl-UFFDIO_REGISTER"); ++ exit(1); ++ } ++ ++ verify_rss_anon_split_huge_page_all_zeroes(one_page, len); ++ printf("Split zero filled huge pages with uffd successful\n"); ++ free(one_page); ++} ++ + void split_pmd_thp(void) + { + char *one_page; +@@ -123,7 +233,6 @@ void split_pmd_thp(void) + exit(EXIT_FAILURE); + } + +- + thp_size = check_huge(one_page); + if (thp_size) { + printf("Still %ld kB AnonHugePages not split\n", thp_size); +@@ -305,6 +414,8 @@ int main(int argc, char **argv) + pageshift = ffs(pagesize) - 1; + pmd_pagesize = read_pmd_pagesize(); + ++ split_pmd_zero_pages(); ++ split_pmd_zero_pages_uffd(); + split_pmd_thp(); + split_pte_mapped_thp(); + split_file_backed_thp(); +diff --git a/tools/testing/selftests/vm/vm_util.c b/tools/testing/selftests/vm/vm_util.c +index b58ab11a7a30..c6a785a67fc9 100644 +--- a/tools/testing/selftests/vm/vm_util.c ++++ b/tools/testing/selftests/vm/vm_util.c +@@ -6,6 +6,7 @@ + + #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" + #define SMAP_FILE_PATH "/proc/self/smaps" ++#define STATUS_FILE_PATH "/proc/self/status" + #define MAX_LINE_LENGTH 500 + + uint64_t pagemap_get_entry(int fd, char *start) +@@ -72,6 +73,28 @@ uint64_t read_pmd_pagesize(void) + return strtoul(buf, NULL, 10); + } + ++uint64_t rss_anon(void) ++{ ++ uint64_t rss_anon = 0; ++ int ret; ++ FILE *fp; ++ char buffer[MAX_LINE_LENGTH]; ++ ++ fp = fopen(STATUS_FILE_PATH, "r"); ++ if (!fp) ++ ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); ++ ++ if (!check_for_pattern(fp, "RssAnon:", buffer)) ++ goto err_out; ++ ++ if (sscanf(buffer, "RssAnon:%10ld kB", &rss_anon) != 1) ++ ksft_exit_fail_msg("Reading status error\n"); ++ ++err_out: ++ fclose(fp); ++ return rss_anon; ++} ++ + uint64_t check_huge(void *addr) + { + uint64_t thp = 0; +diff --git a/tools/testing/selftests/vm/vm_util.h b/tools/testing/selftests/vm/vm_util.h +index 2e512bd57ae1..00b92ccef20d 100644 +--- a/tools/testing/selftests/vm/vm_util.h ++++ b/tools/testing/selftests/vm/vm_util.h +@@ -6,4 +6,5 @@ uint64_t pagemap_get_entry(int fd, char *start); + bool pagemap_is_softdirty(int fd, char *start); + void clear_softdirty(void); + uint64_t read_pmd_pagesize(void); ++uint64_t rss_anon(void); + uint64_t check_huge(void *addr); +-- +2.38.0 +